Implement caret browsing mode (bug 807730)

The users will be able to navigate within the pdf in using the arrows
and they'll be able to select some text, for example in order to
highlight it.

test/integration-boot.mjs

@@ -26,6 +26,7 @@ async function runTests(results) {
     spec_files: [
       "accessibility_spec.mjs",
       "annotation_spec.mjs",
+      "caret_browsing_spec.mjs",
       "copy_paste_spec.mjs",
       "find_spec.mjs",
       "freetext_editor_spec.mjs",

test/integration/caret_browsing_spec.mjs

@@ -0,0 +1,100 @@
+/* Copyright 2021 Mozilla Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { closePages, loadAndWait } from "./test_utils.mjs";
+
+const waitForSelectionChange = (page, selection) =>
+  page.waitForFunction(
+    // We need to replace EOL on Windows to make the test pass.
+    sel => window.getSelection().toString().replaceAll("\r\n", "\n") === sel,
+    {},
+    selection
+  );
+
+describe("Caret browsing", () => {
+  describe("Selection", () => {
+    let pages;
+
+    beforeAll(async () => {
+      pages = await loadAndWait("tracemonkey.pdf", ".textLayer .endOfContent");
+    });
+
+    afterAll(async () => {
+      await closePages(pages);
+    });
+
+    it("must move the caret down and check the selection", async () => {
+      await Promise.all(
+        pages.map(async ([browserName, page]) => {
+          const spanRect = await page.evaluate(() => {
+            const span = document.querySelector(
+              `.page[data-page-number="1"] > .textLayer > span`
+            );
+            const { x, y, width, height } = span.getBoundingClientRect();
+            return { x, y, width, height };
+          });
+          await page.mouse.click(
+            spanRect.x + 1,
+            spanRect.y + spanRect.height / 2,
+            { count: 2 }
+          );
+          await page.keyboard.down("Shift");
+          for (let i = 0; i < 6; i++) {
+            await page.keyboard.press("ArrowRight");
+          }
+          await page.keyboard.up("Shift");
+          await waitForSelectionChange(page, "Trace-based");
+
+          await page.keyboard.down("Shift");
+          await page.keyboard.press("ArrowDown");
+          await page.keyboard.up("Shift");
+
+          // The caret is just before Languages.
+          await waitForSelectionChange(
+            page,
+            "Trace-based Just-in-Time Type Specialization for Dynamic\n"
+          );
+
+          await page.keyboard.down("Shift");
+          await page.keyboard.press("ArrowDown");
+          await page.keyboard.up("Shift");
+
+          // The caret is just before Mike Shaver.
+          await waitForSelectionChange(
+            page,
+            "Trace-based Just-in-Time Type Specialization for Dynamic\nLanguages\nAndreas Gal∗+, Brendan Eich∗, "
+          );
+
+          await page.keyboard.down("Shift");
+          await page.keyboard.press("ArrowUp");
+          await page.keyboard.up("Shift");
+
+          // The caret is just before Languages.
+          await waitForSelectionChange(
+            page,
+            "Trace-based Just-in-Time Type Specialization for Dynamic\n"
+          );
+
+          await page.keyboard.down("Shift");
+          await page.keyboard.press("ArrowUp");
+          await page.keyboard.up("Shift");
+
+          // The caret is in the middle of Time.
+          await waitForSelectionChange(page, "Trace-based Just-in-Tim");
+        })
+      );
+    });
+  });
+});

web/app.js

@@ -56,6 +56,7 @@ import { AutomationEventBus, EventBus } from "./event_utils.js";
 import { LinkTarget, PDFLinkService } from "./pdf_link_service.js";
 import { AltTextManager } from "web-alt_text_manager";
 import { AnnotationEditorParams } from "web-annotation_editor_params";
+import { CaretBrowsingMode } from "./caret_browsing.js";
 import { DownloadManager } from "web-download_manager";
 import { ExternalServices } from "web-external_services";
 import { OverlayManager } from "./overlay_manager.js";
@@ -162,6 +163,7 @@ const PDFViewerApplication = {
   _touchInfo: null,
   _isCtrlKeyDown: false,
   _nimbusDataPromise: null,
+  _caretBrowsing: null,
 
   // Called once when the document is loaded.
   async initialize(appConfig) {
@@ -774,6 +776,23 @@ const PDFViewerApplication = {
     );
   },
 
+  get supportsCaretBrowsingMode() {
+    return shadow(
+      this,
+      "supportsCaretBrowsingMode",
+      AppOptions.get("supportsCaretBrowsingMode")
+    );
+  },
+
+  moveCaret(isUp, select) {
+    this._caretBrowsing ||= new CaretBrowsingMode(
+      this.appConfig.mainContainer,
+      this.appConfig.viewerContainer,
+      this.appConfig.toolbar?.container
+    );
+    this._caretBrowsing.moveCaret(isUp, select);
+  },
+
   initPassiveLoading(file) {
     if (
       typeof PDFJSDev === "undefined" ||
@@ -3014,6 +3033,15 @@ function webViewerKeyDown(evt) {
       turnOnlyIfPageFit = false;
     switch (evt.keyCode) {
       case 38: // up arrow
+        if (PDFViewerApplication.supportsCaretBrowsingMode) {
+          PDFViewerApplication.moveCaret(
+            /* isUp = */ true,
+            /* select = */ false
+          );
+          handled = true;
+          break;
+        }
+      /* falls through */
       case 33: // pg up
         // vertical scrolling using arrow/pg keys
         if (pdfViewer.isVerticalScrollbarEnabled) {
@@ -3028,6 +3056,9 @@ function webViewerKeyDown(evt) {
         turnPage = -1;
         break;
       case 37: // left arrow
+        if (PDFViewerApplication.supportsCaretBrowsingMode) {
+          return;
+        }
         // horizontal scrolling using arrow keys
         if (pdfViewer.isHorizontalScrollbarEnabled) {
           turnOnlyIfPageFit = true;
@@ -3051,6 +3082,15 @@ function webViewerKeyDown(evt) {
         }
         break;
       case 40: // down arrow
+        if (PDFViewerApplication.supportsCaretBrowsingMode) {
+          PDFViewerApplication.moveCaret(
+            /* isUp = */ false,
+            /* select = */ false
+          );
+          handled = true;
+          break;
+        }
+      /* falls through */
       case 34: // pg down
         // vertical scrolling using arrow/pg keys
         if (pdfViewer.isVerticalScrollbarEnabled) {
@@ -3066,6 +3106,9 @@ function webViewerKeyDown(evt) {
         turnPage = 1;
         break;
       case 39: // right arrow
+        if (PDFViewerApplication.supportsCaretBrowsingMode) {
+          return;
+        }
         // horizontal scrolling using arrow keys
         if (pdfViewer.isHorizontalScrollbarEnabled) {
           turnOnlyIfPageFit = true;
@@ -3139,6 +3182,14 @@ function webViewerKeyDown(evt) {
         handled = true;
         break;
 
+      case 38: // up arrow
+        PDFViewerApplication.moveCaret(/* isUp = */ true, /* select = */ true);
+        handled = true;
+        break;
+      case 40: // down arrow
+        PDFViewerApplication.moveCaret(/* isUp = */ false, /* select = */ true);
+        handled = true;
+        break;
       case 82: // 'r'
         PDFViewerApplication.rotatePages(-90);
         break;

web/app_options.js

@@ -64,6 +64,11 @@ const defaultOptions = {
     value: false,
     kind: OptionKind.BROWSER,
   },
+  supportsCaretBrowsingMode: {
+    /** @type {boolean} */
+    value: false,
+    kind: OptionKind.BROWSER,
+  },
   supportsDocumentFonts: {
     /** @type {boolean} */
     value: true,

web/caret_browsing.js

@@ -0,0 +1,329 @@
+/* Copyright 2024 Mozilla Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Used to compare floats: there is no exact equality due to rounding errors.
+const PRECISION = 1e-1;
+
+class CaretBrowsingMode {
+  #mainContainer;
+
+  #toolBarHeight;
+
+  #viewerContainer;
+
+  constructor(mainContainer, viewerContainer, toolbarContainer) {
+    this.#mainContainer = mainContainer;
+    this.#viewerContainer = viewerContainer;
+    this.#toolBarHeight = toolbarContainer?.getBoundingClientRect().height ?? 0;
+  }
+
+  /**
+   * Return true if the two rectangles are on the same line.
+   * @param {DOMRect} rect1
+   * @param {DOMRect} rect2
+   * @returns {boolean}
+   */
+  #isOnSameLine(rect1, rect2) {
+    const top1 = rect1.y;
+    const bot1 = rect1.bottom;
+    const mid1 = rect1.y + rect1.height / 2;
+
+    const top2 = rect2.y;
+    const bot2 = rect2.bottom;
+    const mid2 = rect2.y + rect2.height / 2;
+
+    return (top1 <= mid2 && mid2 <= bot1) || (top2 <= mid1 && mid1 <= bot2);
+  }
+
+  /**
+   * Return `true` if the rectangle is:
+   *  - under the caret when `isUp === false`.
+   *  - over the caret when `isUp === true`.
+   * @param {DOMRect} rect
+   * @param {number} x
+   * @param {number} y
+   * @param {boolean} isUp
+   * @returns {boolean}
+   */
+  #isUnderOver(rect, x, y, isUp) {
+    const midY = rect.y + rect.height / 2;
+    return (
+      (isUp ? y >= midY : y <= midY) &&
+      rect.x - PRECISION <= x &&
+      x <= rect.right + PRECISION
+    );
+  }
+
+  /**
+   * Check if the rectangle is visible.
+   * @param {DOMRect} rect
+   * @returns {boolean}
+   */
+  #isVisible(rect) {
+    return (
+      rect.top >= this.#toolBarHeight &&
+      rect.left >= 0 &&
+      rect.bottom <=
+        (window.innerHeight || document.documentElement.clientHeight) &&
+      rect.right <= (window.innerWidth || document.documentElement.clientWidth)
+    );
+  }
+
+  /**
+   * Get the position of the caret.
+   * @param {Selection} selection
+   * @param {boolean} isUp
+   * @returns {Array<number>}
+   */
+  #getCaretPosition(selection, isUp) {
+    const { focusNode, focusOffset } = selection;
+    const range = document.createRange();
+    range.setStart(focusNode, focusOffset);
+    range.setEnd(focusNode, focusOffset);
+    const rect = range.getBoundingClientRect();
+
+    return [rect.x, isUp ? rect.top : rect.bottom];
+  }
+
+  static #caretPositionFromPoint(x, y) {
+    if (
+      (typeof PDFJSDev === "undefined" || !PDFJSDev.test("MOZCENTRAL")) &&
+      !document.caretPositionFromPoint
+    ) {
+      const { startContainer: offsetNode, startOffset: offset } =
+        document.caretRangeFromPoint(x, y);
+      return { offsetNode, offset };
+    }
+    return document.caretPositionFromPoint(x, y);
+  }
+
+  #setCaretPositionHelper(selection, caretX, select, element, rect) {
+    rect ||= element.getBoundingClientRect();
+    if (caretX <= rect.x + PRECISION) {
+      if (select) {
+        selection.extend(element.firstChild, 0);
+      } else {
+        selection.setPosition(element.firstChild, 0);
+      }
+      return;
+    }
+    if (rect.right - PRECISION <= caretX) {
+      const { lastChild } = element;
+      if (select) {
+        selection.extend(lastChild, lastChild.length);
+      } else {
+        selection.setPosition(lastChild, lastChild.length);
+      }
+      return;
+    }
+
+    const midY = rect.y + rect.height / 2;
+    const caretPosition = CaretBrowsingMode.#caretPositionFromPoint(
+      caretX,
+      midY
+    );
+    if (caretPosition.offsetNode?.parentElement !== element) {
+      // The element targeted by caretPositionFromPoint isn't in the text
+      // layer.
+      if (select) {
+        selection.extend(element.firstChild, 0);
+      } else {
+        selection.setPosition(element.firstChild, 0);
+      }
+      return;
+    }
+    if (select) {
+      selection.extend(caretPosition.offsetNode, caretPosition.offset);
+    } else {
+      selection.setPosition(caretPosition.offsetNode, caretPosition.offset);
+    }
+  }
+
+  /**
+   * Set the caret position or extend the selection (it depends on the select
+   * parameter).
+   * @param {boolean} select
+   * @param {Selection} selection
+   * @param {Element} newLineElement
+   * @param {DOMRect} newLineElementRect
+   * @param {number} caretX
+   */
+  #setCaretPosition(
+    select,
+    selection,
+    newLineElement,
+    newLineElementRect,
+    caretX
+  ) {
+    if (this.#isVisible(newLineElementRect)) {
+      this.#setCaretPositionHelper(
+        selection,
+        caretX,
+        select,
+        newLineElement,
+        newLineElementRect
+      );
+      return;
+    }
+    this.#mainContainer.addEventListener(
+      "scrollend",
+      this.#setCaretPositionHelper.bind(
+        this,
+        selection,
+        caretX,
+        select,
+        newLineElement,
+        null
+      ),
+      { once: true }
+    );
+    newLineElement.scrollIntoView();
+  }
+
+  /**
+   * Get the node on the next page.
+   * @param {Element} textLayer
+   * @param {boolean} isUp
+   * @returns {Node}
+   */
+  #getNodeOnNextPage(textLayer, isUp) {
+    while (true) {
+      const page = textLayer.closest(".page");
+      const pageNumber = parseInt(page.getAttribute("data-page-number"));
+      const nextPage = isUp ? pageNumber - 1 : pageNumber + 1;
+      textLayer = this.#viewerContainer.querySelector(
+        `.page[data-page-number="${nextPage}"] .textLayer`
+      );
+      if (!textLayer) {
+        return null;
+      }
+      const walker = document.createTreeWalker(textLayer, NodeFilter.SHOW_TEXT);
+      const node = isUp ? walker.lastChild() : walker.firstChild();
+      if (node) {
+        return node;
+      }
+    }
+  }
+
+  /**
+   * Move the caret in the given direction.
+   * @param {boolean} isUp
+   * @param {boolean} select
+   */
+  moveCaret(isUp, select) {
+    const selection = document.getSelection();
+    if (selection.rangeCount === 0) {
+      return;
+    }
+    const { focusNode } = selection;
+    const focusElement =
+      focusNode.nodeType !== Node.ELEMENT_NODE
+        ? focusNode.parentElement
+        : focusNode;
+    const root = focusElement.closest(".textLayer");
+    if (!root) {
+      return;
+    }
+    const walker = document.createTreeWalker(root, NodeFilter.SHOW_TEXT);
+    walker.currentNode = focusNode;
+
+    // Move to the next element which is not on the same line as the focus
+    // element.
+    const focusRect = focusElement.getBoundingClientRect();
+    let newLineElement = null;
+    const nodeIterator = (
+      isUp ? walker.previousSibling : walker.nextSibling
+    ).bind(walker);
+    while (nodeIterator()) {
+      const element = walker.currentNode.parentElement;
+      if (!this.#isOnSameLine(focusRect, element.getBoundingClientRect())) {
+        newLineElement = element;
+        break;
+      }
+    }
+
+    if (!newLineElement) {
+      // Need to find the next line on the next page.
+      const node = this.#getNodeOnNextPage(root, isUp);
+      if (!node) {
+        return;
+      }
+      if (select) {
+        const lastNode =
+          (isUp ? walker.firstChild() : walker.lastChild()) || focusNode;
+        selection.extend(lastNode, isUp ? 0 : lastNode.length);
+        const range = document.createRange();
+        range.setStart(node, isUp ? node.length : 0);
+        range.setEnd(node, isUp ? node.length : 0);
+        selection.addRange(range);
+        return;
+      }
+      const [caretX] = this.#getCaretPosition(selection, isUp);
+      const { parentElement } = node;
+      this.#setCaretPosition(
+        select,
+        selection,
+        parentElement,
+        parentElement.getBoundingClientRect(),
+        caretX
+      );
+      return;
+    }
+
+    // We've a candidate for the next line now we want to find the first element
+    // which is under/over the caret.
+    const [caretX, caretY] = this.#getCaretPosition(selection, isUp);
+    const newLineElementRect = newLineElement.getBoundingClientRect();
+
+    // Maybe the element on the new line is a valid candidate.
+    if (this.#isUnderOver(newLineElementRect, caretX, caretY, isUp)) {
+      this.#setCaretPosition(
+        select,
+        selection,
+        newLineElement,
+        newLineElementRect,
+        caretX
+      );
+      return;
+    }
+
+    while (nodeIterator()) {
+      // Search an element on the same line as newLineElement
+      // which could be under/over the caret.
+      const element = walker.currentNode.parentElement;
+      const elementRect = element.getBoundingClientRect();
+      if (!this.#isOnSameLine(newLineElementRect, elementRect)) {
+        break;
+      }
+      if (this.#isUnderOver(elementRect, caretX, caretY, isUp)) {
+        // We found the element.
+        this.#setCaretPosition(select, selection, element, elementRect, caretX);
+        return;
+      }
+    }
+
+    // No element has been found so just put the caret on the element on the new
+    // line.
+    this.#setCaretPosition(
+      select,
+      selection,
+      newLineElement,
+      newLineElementRect,
+      caretX
+    );
+  }
+}
+
+export { CaretBrowsingMode };