diff --git a/src/common/annotation-manager.js b/src/common/annotation-manager.js
index 46672e0cd..32ca6f5c8 100644
--- a/src/common/annotation-manager.js
+++ b/src/common/annotation-manager.js
@@ -49,11 +49,18 @@ class AnnotationManager {
 
 	// Called when changes come from the client side
 	async setAnnotations(annotations) {
+		let changed = false;
 		for (let annotation of annotations) {
+			if (this._unsavedAnnotations.has(annotation.id)) {
+				continue;
+			}
 			this._annotations = this._annotations.filter(x => x.id !== annotation.id);
 			this._annotations.push(annotation);
+			changed = true;
+		}
+		if (changed) {
+			this._annotations.sort((a, b) => (a.sortIndex > b.sortIndex) - (a.sortIndex < b.sortIndex));
 		}
-		this._annotations.sort((a, b) => (a.sortIndex > b.sortIndex) - (a.sortIndex < b.sortIndex));
 		this._clearInterferingHistory(annotations.map(x => x.id));
 		this.render();
 	}