diff --git a/src/display/annotation_storage.js b/src/display/annotation_storage.js
index 3729192ee..515869c13 100644
--- a/src/display/annotation_storage.js
+++ b/src/display/annotation_storage.js
@@ -146,7 +146,11 @@ class AnnotationStorage {
     const clone = new Map();
 
     for (const [key, val] of this._storage) {
-      clone.set(key, val instanceof AnnotationEditor ? val.serialize() : val);
+      const serialized =
+        val instanceof AnnotationEditor ? val.serialize() : val;
+      if (serialized) {
+        clone.set(key, serialized);
+      }
     }
     return clone;
   }
diff --git a/src/display/editor/annotation_editor_layer.js b/src/display/editor/annotation_editor_layer.js
index 6b5380842..ecc29b3f7 100644
--- a/src/display/editor/annotation_editor_layer.js
+++ b/src/display/editor/annotation_editor_layer.js
@@ -42,10 +42,10 @@ import { InkEditor } from "./ink.js";
 class AnnotationEditorLayer {
   #boundClick;
 
-  #boundMouseover;
-
   #editors = new Map();
 
+  #isCleaningUp = false;
+
   #uiManager;
 
   static _initialized = false;
@@ -92,7 +92,6 @@ class AnnotationEditorLayer {
     this.pageIndex = options.pageIndex;
     this.div = options.div;
     this.#boundClick = this.click.bind(this);
-    this.#boundMouseover = this.mouseover.bind(this);
 
     for (const editor of this.#uiManager.getEditors(options.pageIndex)) {
       this.add(editor);
@@ -114,24 +113,36 @@ class AnnotationEditorLayer {
    * The mode has changed: it must be updated.
    * @param {number} mode
    */
-  updateMode(mode) {
-    switch (mode) {
-      case AnnotationEditorType.INK:
-        // We want to have the ink editor covering all of the page without
-        // having to click to create it: it must be here when we start to draw.
-        this.div.addEventListener("mouseover", this.#boundMouseover);
-        this.div.removeEventListener("click", this.#boundClick);
-        break;
-      case AnnotationEditorType.FREETEXT:
-        this.div.removeEventListener("mouseover", this.#boundMouseover);
-        this.div.addEventListener("click", this.#boundClick);
-        break;
-      default:
-        this.div.removeEventListener("mouseover", this.#boundMouseover);
-        this.div.removeEventListener("click", this.#boundClick);
+  updateMode(mode = this.#uiManager.getMode()) {
+    this.#cleanup();
+    if (mode === AnnotationEditorType.INK) {
+      // We always want to an ink editor ready to draw in.
+      this.addInkEditorIfNeeded(false);
+    }
+    this.setActiveEditor(null);
+  }
+
+  addInkEditorIfNeeded(isCommitting) {
+    if (
+      !isCommitting &&
+      this.#uiManager.getMode() !== AnnotationEditorType.INK
+    ) {
+      return;
     }
 
-    this.setActiveEditor(null);
+    if (!isCommitting) {
+      // We're removing an editor but an empty one can already exist so in this
+      // case we don't need to create a new one.
+      for (const editor of this.#editors.values()) {
+        if (editor.isEmpty()) {
+          editor.setInBackground();
+          return;
+        }
+      }
+    }
+
+    const editor = this.#createAndAddNewEditor({ offsetX: 0, offsetY: 0 });
+    editor.setInBackground();
   }
 
   /**
@@ -142,25 +153,6 @@ class AnnotationEditorLayer {
     this.#uiManager.setEditingState(isEditing);
   }
 
-  /**
-   * Mouseover callback.
-   * @param {MouseEvent} event
-   */
-  mouseover(event) {
-    if (
-      event.target === this.div &&
-      event.buttons === 0 &&
-      !this.#uiManager.hasActive()
-    ) {
-      // The div is the target so there is no ink editor, hence we can
-      // create a new one.
-      // event.buttons === 0 is here to avoid adding a new ink editor
-      // when we drop an editor.
-      const editor = this.#createAndAddNewEditor(event);
-      editor.setInBackground();
-    }
-  }
-
   /**
    * Add some commands into the CommandManager (undo/redo stuff).
    * @param {Object} params
@@ -258,14 +250,12 @@ class AnnotationEditorLayer {
       currentActive.commitOrRemove();
     }
 
+    this.#uiManager.allowClick =
+      this.#uiManager.getMode() === AnnotationEditorType.INK;
     if (editor) {
       this.unselectAll();
       this.div.removeEventListener("click", this.#boundClick);
     } else {
-      // When in Ink mode, setting the editor to null allows the
-      // user to have to make one click in order to start drawing.
-      this.#uiManager.allowClick =
-        this.#uiManager.getMode() === AnnotationEditorType.INK;
       this.div.addEventListener("click", this.#boundClick);
     }
   }
@@ -295,6 +285,10 @@ class AnnotationEditorLayer {
       this.setActiveEditor(null);
       this.#uiManager.allowClick = true;
     }
+
+    if (!this.#isCleaningUp) {
+      this.addInkEditorIfNeeded(/* isCommitting = */ false);
+    }
   }
 
   /**
@@ -496,6 +490,19 @@ class AnnotationEditorLayer {
     this.#uiManager.removeLayer(this);
   }
 
+  #cleanup() {
+    // When we're cleaning up, some editors are removed but we don't want
+    // to add a new one which will induce an addition in this.#editors, hence
+    // an infinite loop.
+    this.#isCleaningUp = true;
+    for (const editor of this.#editors.values()) {
+      if (editor.isEmpty()) {
+        editor.remove();
+      }
+    }
+    this.#isCleaningUp = false;
+  }
+
   /**
    * Render the main editor.
    * @param {Object} parameters
@@ -505,6 +512,7 @@ class AnnotationEditorLayer {
     bindEvents(this, this.div, ["dragover", "drop", "keydown"]);
     this.div.addEventListener("click", this.#boundClick);
     this.setDimensions();
+    this.updateMode();
   }
 
   /**
@@ -515,6 +523,7 @@ class AnnotationEditorLayer {
     this.setActiveEditor(null);
     this.viewport = parameters.viewport;
     this.setDimensions();
+    this.updateMode();
   }
 
   /**
diff --git a/src/display/editor/editor.js b/src/display/editor/editor.js
index eb91a39b5..110e8464d 100644
--- a/src/display/editor/editor.js
+++ b/src/display/editor/editor.js
@@ -16,8 +16,12 @@
 // eslint-disable-next-line max-len
 /** @typedef {import("./annotation_editor_layer.js").AnnotationEditorLayer} AnnotationEditorLayer */
 
+import {
+  AnnotationEditorPrefix,
+  shadow,
+  unreachable,
+} from "../../shared/util.js";
 import { bindEvents, ColorManager } from "./tools.js";
-import { shadow, unreachable } from "../../shared/util.js";
 
 /**
  * @typedef {Object} AnnotationEditorParameters
@@ -109,7 +113,10 @@ class AnnotationEditor {
     event.preventDefault();
 
     this.commitOrRemove();
-    this.parent.setActiveEditor(null);
+
+    if (!target?.id?.startsWith(AnnotationEditorPrefix)) {
+      this.parent.setActiveEditor(null);
+    }
   }
 
   commitOrRemove() {
diff --git a/src/display/editor/freetext.js b/src/display/editor/freetext.js
index 85f99ec7c..0bada5621 100644
--- a/src/display/editor/freetext.js
+++ b/src/display/editor/freetext.js
@@ -372,6 +372,10 @@ class FreeTextEditor extends AnnotationEditor {
 
   /** @inheritdoc */
   serialize() {
+    if (this.isEmpty()) {
+      return null;
+    }
+
     const padding = FreeTextEditor._internalPadding * this.parent.scaleFactor;
     const rect = this.getRect(padding, padding);
 
diff --git a/src/display/editor/ink.js b/src/display/editor/ink.js
index 559cf679d..3c60a65a1 100644
--- a/src/display/editor/ink.js
+++ b/src/display/editor/ink.js
@@ -41,6 +41,8 @@ class InkEditor extends AnnotationEditor {
 
   #disableEditing = false;
 
+  #isCanvasInitialized = false;
+
   #observer = null;
 
   #realWidth = 0;
@@ -53,11 +55,8 @@ class InkEditor extends AnnotationEditor {
 
   constructor(params) {
     super({ ...params, name: "inkEditor" });
-    this.color =
-      params.color ||
-      InkEditor._defaultColor ||
-      AnnotationEditor._defaultLineColor;
-    this.thickness = params.thickness || InkEditor._defaultThickness;
+    this.color = params.color || null;
+    this.thickness = params.thickness || null;
     this.paths = [];
     this.bezierPath2D = [];
     this.currentPath = [];
@@ -255,7 +254,6 @@ class InkEditor extends AnnotationEditor {
   /** @inheritdoc */
   onceAdded() {
     this.div.draggable = !this.isEmpty();
-    this.div.focus();
   }
 
   /** @inheritdoc */
@@ -298,6 +296,13 @@ class InkEditor extends AnnotationEditor {
    * @param {number} y
    */
   #startDrawing(x, y) {
+    if (!this.#isCanvasInitialized) {
+      this.#isCanvasInitialized = true;
+      this.#setCanvasDims();
+      this.thickness ||= InkEditor._defaultThickness;
+      this.color ||=
+        InkEditor._defaultColor || AnnotationEditor._defaultLineColor;
+    }
     this.currentPath.push([x, y]);
     this.#setStroke();
     this.ctx.beginPath();
@@ -406,6 +411,8 @@ class InkEditor extends AnnotationEditor {
     this.div.classList.add("disabled");
 
     this.#fitToContent();
+
+    this.parent.addInkEditorIfNeeded(/* isCommitting = */ true);
   }
 
   /** @inheritdoc */
@@ -491,6 +498,7 @@ class InkEditor extends AnnotationEditor {
    */
   #createCanvas() {
     this.canvas = document.createElement("canvas");
+    this.canvas.width = this.canvas.height = 0;
     this.canvas.className = "inkEditorCanvas";
     this.div.append(this.canvas);
     this.ctx = this.canvas.getContext("2d");
@@ -522,7 +530,6 @@ class InkEditor extends AnnotationEditor {
     }
 
     super.render();
-    this.div.classList.add("editing");
     const [x, y, w, h] = this.#getInitialBBox();
     this.setAt(x, y, 0, 0);
     this.setDims(w, h);
@@ -531,6 +538,7 @@ class InkEditor extends AnnotationEditor {
 
     if (this.width) {
       // This editor was created in using copy (ctrl+c).
+      this.#isCanvasInitialized = true;
       const [parentWidth, parentHeight] = this.parent.viewportBaseDimensions;
       this.setAt(
         baseX * parentWidth,
@@ -542,6 +550,9 @@ class InkEditor extends AnnotationEditor {
       this.#setCanvasDims();
       this.#redraw();
       this.div.classList.add("disabled");
+    } else {
+      this.div.classList.add("editing");
+      this.enableEditMode();
     }
 
     this.#createObserver();
@@ -550,6 +561,9 @@ class InkEditor extends AnnotationEditor {
   }
 
   #setCanvasDims() {
+    if (!this.#isCanvasInitialized) {
+      return;
+    }
     const [parentWidth, parentHeight] = this.parent.viewportBaseDimensions;
     this.canvas.width = this.width * parentWidth;
     this.canvas.height = this.height * parentHeight;
@@ -861,6 +875,10 @@ class InkEditor extends AnnotationEditor {
 
   /** @inheritdoc */
   serialize() {
+    if (this.isEmpty()) {
+      return null;
+    }
+
     const rect = this.getRect(0, 0);
     const height =
       this.rotation % 180 === 0 ? rect[3] - rect[1] : rect[2] - rect[0];
diff --git a/src/display/editor/tools.js b/src/display/editor/tools.js
index 22f92ec9e..83d8696c6 100644
--- a/src/display/editor/tools.js
+++ b/src/display/editor/tools.js
@@ -779,7 +779,9 @@ class AnnotationEditorUIManager {
       const editors = Array.from(this.#allEditors.values());
       cmd = () => {
         for (const editor of editors) {
-          editor.remove();
+          if (!editor.isEmpty()) {
+            editor.remove();
+          }
         }
       };