Merge pull request #17611 from calixteman/caret_browsing_mode

Implement caret browsing mode (bug 807730)

test/integration-boot.mjs

@@ -26,6 +26,7 @@ async function runTests(results) {
     spec_files: [
       "accessibility_spec.mjs",
       "annotation_spec.mjs",
+      "caret_browsing_spec.mjs",
       "copy_paste_spec.mjs",
       "find_spec.mjs",
       "freetext_editor_spec.mjs",

test/integration/caret_browsing_spec.mjs

@@ -0,0 +1,100 @@
+/* Copyright 2021 Mozilla Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { closePages, loadAndWait } from "./test_utils.mjs";
+
+const waitForSelectionChange = (page, selection) =>
+  page.waitForFunction(
+    // We need to replace EOL on Windows to make the test pass.
+    sel => window.getSelection().toString().replaceAll("\r\n", "\n") === sel,
+    {},
+    selection
+  );
+
+describe("Caret browsing", () => {
+  describe("Selection", () => {
+    let pages;
+
+    beforeAll(async () => {
+      pages = await loadAndWait("tracemonkey.pdf", ".textLayer .endOfContent");
+    });
+
+    afterAll(async () => {
+      await closePages(pages);
+    });
+
+    it("must move the caret down and check the selection", async () => {
+      await Promise.all(
+        pages.map(async ([browserName, page]) => {
+          const spanRect = await page.evaluate(() => {
+            const span = document.querySelector(
+              `.page[data-page-number="1"] > .textLayer > span`
+            );
+            const { x, y, width, height } = span.getBoundingClientRect();
+            return { x, y, width, height };
+          });
+          await page.mouse.click(
+            spanRect.x + 1,
+            spanRect.y + spanRect.height / 2,
+            { count: 2 }
+          );
+          await page.keyboard.down("Shift");
+          for (let i = 0; i < 6; i++) {
+            await page.keyboard.press("ArrowRight");
+          }
+          await page.keyboard.up("Shift");
+          await waitForSelectionChange(page, "Trace-based");
+
+          await page.keyboard.down("Shift");
+          await page.keyboard.press("ArrowDown");
+          await page.keyboard.up("Shift");
+
+          // The caret is just before Languages.
+          await waitForSelectionChange(
+            page,
+            "Trace-based Just-in-Time Type Specialization for Dynamic\n"
+          );
+
+          await page.keyboard.down("Shift");
+          await page.keyboard.press("ArrowDown");
+          await page.keyboard.up("Shift");
+
+          // The caret is just before Mike Shaver.
+          await waitForSelectionChange(
+            page,
+            "Trace-based Just-in-Time Type Specialization for Dynamic\nLanguages\nAndreas Gal∗+, Brendan Eich∗, "
+          );
+
+          await page.keyboard.down("Shift");
+          await page.keyboard.press("ArrowUp");
+          await page.keyboard.up("Shift");
+
+          // The caret is just before Languages.
+          await waitForSelectionChange(
+            page,
+            "Trace-based Just-in-Time Type Specialization for Dynamic\n"
+          );
+
+          await page.keyboard.down("Shift");
+          await page.keyboard.press("ArrowUp");
+          await page.keyboard.up("Shift");
+
+          // The caret is in the middle of Time.
+          await waitForSelectionChange(page, "Trace-based Just-in-Tim");
+        })
+      );
+    });
+  });
+});

web/app.js

@@ -56,6 +56,7 @@ import { AutomationEventBus, EventBus } from "./event_utils.js";
 import { LinkTarget, PDFLinkService } from "./pdf_link_service.js";
 import { AltTextManager } from "web-alt_text_manager";
 import { AnnotationEditorParams } from "web-annotation_editor_params";
+import { CaretBrowsingMode } from "./caret_browsing.js";
 import { DownloadManager } from "web-download_manager";
 import { ExternalServices } from "web-external_services";
 import { OverlayManager } from "./overlay_manager.js";
@@ -162,6 +163,7 @@ const PDFViewerApplication = {
   _touchInfo: null,
   _isCtrlKeyDown: false,
   _nimbusDataPromise: null,
+  _caretBrowsing: null,
 
   // Called once when the document is loaded.
   async initialize(appConfig) {
@@ -774,6 +776,23 @@ const PDFViewerApplication = {
     );
   },
 
+  get supportsCaretBrowsingMode() {
+    return shadow(
+      this,
+      "supportsCaretBrowsingMode",
+      AppOptions.get("supportsCaretBrowsingMode")
+    );
+  },
+
+  moveCaret(isUp, select) {
+    this._caretBrowsing ||= new CaretBrowsingMode(
+      this.appConfig.mainContainer,
+      this.appConfig.viewerContainer,
+      this.appConfig.toolbar?.container
+    );
+    this._caretBrowsing.moveCaret(isUp, select);
+  },
+
   initPassiveLoading(file) {
     if (
       typeof PDFJSDev === "undefined" ||
@@ -3014,6 +3033,15 @@ function webViewerKeyDown(evt) {
       turnOnlyIfPageFit = false;
     switch (evt.keyCode) {
       case 38: // up arrow
+        if (PDFViewerApplication.supportsCaretBrowsingMode) {
+          PDFViewerApplication.moveCaret(
+            /* isUp = */ true,
+            /* select = */ false
+          );
+          handled = true;
+          break;
+        }
+      /* falls through */
       case 33: // pg up
         // vertical scrolling using arrow/pg keys
         if (pdfViewer.isVerticalScrollbarEnabled) {
@@ -3028,6 +3056,9 @@ function webViewerKeyDown(evt) {
         turnPage = -1;
         break;
       case 37: // left arrow
+        if (PDFViewerApplication.supportsCaretBrowsingMode) {
+          return;
+        }
         // horizontal scrolling using arrow keys
         if (pdfViewer.isHorizontalScrollbarEnabled) {
           turnOnlyIfPageFit = true;
@@ -3051,6 +3082,15 @@ function webViewerKeyDown(evt) {
         }
         break;
       case 40: // down arrow
+        if (PDFViewerApplication.supportsCaretBrowsingMode) {
+          PDFViewerApplication.moveCaret(
+            /* isUp = */ false,
+            /* select = */ false
+          );
+          handled = true;
+          break;
+        }
+      /* falls through */
       case 34: // pg down
         // vertical scrolling using arrow/pg keys
         if (pdfViewer.isVerticalScrollbarEnabled) {
@@ -3066,6 +3106,9 @@ function webViewerKeyDown(evt) {
         turnPage = 1;
         break;
       case 39: // right arrow
+        if (PDFViewerApplication.supportsCaretBrowsingMode) {
+          return;
+        }
         // horizontal scrolling using arrow keys
         if (pdfViewer.isHorizontalScrollbarEnabled) {
           turnOnlyIfPageFit = true;
@@ -3139,6 +3182,14 @@ function webViewerKeyDown(evt) {
         handled = true;
         break;
 
+      case 38: // up arrow
+        PDFViewerApplication.moveCaret(/* isUp = */ true, /* select = */ true);
+        handled = true;
+        break;
+      case 40: // down arrow
+        PDFViewerApplication.moveCaret(/* isUp = */ false, /* select = */ true);
+        handled = true;
+        break;
       case 82: // 'r'
         PDFViewerApplication.rotatePages(-90);
         break;

web/app_options.js

@@ -64,6 +64,11 @@ const defaultOptions = {
     value: false,
     kind: OptionKind.BROWSER,
   },
+  supportsCaretBrowsingMode: {
+    /** @type {boolean} */
+    value: false,
+    kind: OptionKind.BROWSER,
+  },
   supportsDocumentFonts: {
     /** @type {boolean} */
     value: true,

web/caret_browsing.js

@@ -0,0 +1,329 @@
+/* Copyright 2024 Mozilla Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Used to compare floats: there is no exact equality due to rounding errors.
+const PRECISION = 1e-1;
+
+class CaretBrowsingMode {
+  #mainContainer;
+
+  #toolBarHeight;
+
+  #viewerContainer;
+
+  constructor(mainContainer, viewerContainer, toolbarContainer) {
+    this.#mainContainer = mainContainer;
+    this.#viewerContainer = viewerContainer;
+    this.#toolBarHeight = toolbarContainer?.getBoundingClientRect().height ?? 0;
+  }
+
+  /**
+   * Return true if the two rectangles are on the same line.
+   * @param {DOMRect} rect1
+   * @param {DOMRect} rect2
+   * @returns {boolean}
+   */
+  #isOnSameLine(rect1, rect2) {
+    const top1 = rect1.y;
+    const bot1 = rect1.bottom;
+    const mid1 = rect1.y + rect1.height / 2;
+
+    const top2 = rect2.y;
+    const bot2 = rect2.bottom;
+    const mid2 = rect2.y + rect2.height / 2;
+
+    return (top1 <= mid2 && mid2 <= bot1) || (top2 <= mid1 && mid1 <= bot2);
+  }
+
+  /**
+   * Return `true` if the rectangle is:
+   *  - under the caret when `isUp === false`.
+   *  - over the caret when `isUp === true`.
+   * @param {DOMRect} rect
+   * @param {number} x
+   * @param {number} y
+   * @param {boolean} isUp
+   * @returns {boolean}
+   */
+  #isUnderOver(rect, x, y, isUp) {
+    const midY = rect.y + rect.height / 2;
+    return (
+      (isUp ? y >= midY : y <= midY) &&
+      rect.x - PRECISION <= x &&
+      x <= rect.right + PRECISION
+    );
+  }
+
+  /**
+   * Check if the rectangle is visible.
+   * @param {DOMRect} rect
+   * @returns {boolean}
+   */
+  #isVisible(rect) {
+    return (
+      rect.top >= this.#toolBarHeight &&
+      rect.left >= 0 &&
+      rect.bottom <=
+        (window.innerHeight || document.documentElement.clientHeight) &&
+      rect.right <= (window.innerWidth || document.documentElement.clientWidth)
+    );
+  }
+
+  /**
+   * Get the position of the caret.
+   * @param {Selection} selection
+   * @param {boolean} isUp
+   * @returns {Array<number>}
+   */
+  #getCaretPosition(selection, isUp) {
+    const { focusNode, focusOffset } = selection;
+    const range = document.createRange();
+    range.setStart(focusNode, focusOffset);
+    range.setEnd(focusNode, focusOffset);
+    const rect = range.getBoundingClientRect();
+
+    return [rect.x, isUp ? rect.top : rect.bottom];
+  }
+
+  static #caretPositionFromPoint(x, y) {
+    if (
+      (typeof PDFJSDev === "undefined" || !PDFJSDev.test("MOZCENTRAL")) &&
+      !document.caretPositionFromPoint
+    ) {
+      const { startContainer: offsetNode, startOffset: offset } =
+        document.caretRangeFromPoint(x, y);
+      return { offsetNode, offset };
+    }
+    return document.caretPositionFromPoint(x, y);
+  }
+
+  #setCaretPositionHelper(selection, caretX, select, element, rect) {
+    rect ||= element.getBoundingClientRect();
+    if (caretX <= rect.x + PRECISION) {
+      if (select) {
+        selection.extend(element.firstChild, 0);
+      } else {
+        selection.setPosition(element.firstChild, 0);
+      }
+      return;
+    }
+    if (rect.right - PRECISION <= caretX) {
+      const { lastChild } = element;
+      if (select) {
+        selection.extend(lastChild, lastChild.length);
+      } else {
+        selection.setPosition(lastChild, lastChild.length);
+      }
+      return;
+    }
+
+    const midY = rect.y + rect.height / 2;
+    const caretPosition = CaretBrowsingMode.#caretPositionFromPoint(
+      caretX,
+      midY
+    );
+    if (caretPosition.offsetNode?.parentElement !== element) {
+      // The element targeted by caretPositionFromPoint isn't in the text
+      // layer.
+      if (select) {
+        selection.extend(element.firstChild, 0);
+      } else {
+        selection.setPosition(element.firstChild, 0);
+      }
+      return;
+    }
+    if (select) {
+      selection.extend(caretPosition.offsetNode, caretPosition.offset);
+    } else {
+      selection.setPosition(caretPosition.offsetNode, caretPosition.offset);
+    }
+  }
+
+  /**
+   * Set the caret position or extend the selection (it depends on the select
+   * parameter).
+   * @param {boolean} select
+   * @param {Selection} selection
+   * @param {Element} newLineElement
+   * @param {DOMRect} newLineElementRect
+   * @param {number} caretX
+   */
+  #setCaretPosition(
+    select,
+    selection,
+    newLineElement,
+    newLineElementRect,
+    caretX
+  ) {
+    if (this.#isVisible(newLineElementRect)) {
+      this.#setCaretPositionHelper(
+        selection,
+        caretX,
+        select,
+        newLineElement,
+        newLineElementRect
+      );
+      return;
+    }
+    this.#mainContainer.addEventListener(
+      "scrollend",
+      this.#setCaretPositionHelper.bind(
+        this,
+        selection,
+        caretX,
+        select,
+        newLineElement,
+        null
+      ),
+      { once: true }
+    );
+    newLineElement.scrollIntoView();
+  }
+
+  /**
+   * Get the node on the next page.
+   * @param {Element} textLayer
+   * @param {boolean} isUp
+   * @returns {Node}
+   */
+  #getNodeOnNextPage(textLayer, isUp) {
+    while (true) {
+      const page = textLayer.closest(".page");
+      const pageNumber = parseInt(page.getAttribute("data-page-number"));
+      const nextPage = isUp ? pageNumber - 1 : pageNumber + 1;
+      textLayer = this.#viewerContainer.querySelector(
+        `.page[data-page-number="${nextPage}"] .textLayer`
+      );
+      if (!textLayer) {
+        return null;
+      }
+      const walker = document.createTreeWalker(textLayer, NodeFilter.SHOW_TEXT);
+      const node = isUp ? walker.lastChild() : walker.firstChild();
+      if (node) {
+        return node;
+      }
+    }
+  }
+
+  /**
+   * Move the caret in the given direction.
+   * @param {boolean} isUp
+   * @param {boolean} select
+   */
+  moveCaret(isUp, select) {
+    const selection = document.getSelection();
+    if (selection.rangeCount === 0) {
+      return;
+    }
+    const { focusNode } = selection;
+    const focusElement =
+      focusNode.nodeType !== Node.ELEMENT_NODE
+        ? focusNode.parentElement
+        : focusNode;
+    const root = focusElement.closest(".textLayer");
+    if (!root) {
+      return;
+    }
+    const walker = document.createTreeWalker(root, NodeFilter.SHOW_TEXT);
+    walker.currentNode = focusNode;
+
+    // Move to the next element which is not on the same line as the focus
+    // element.
+    const focusRect = focusElement.getBoundingClientRect();
+    let newLineElement = null;
+    const nodeIterator = (
+      isUp ? walker.previousSibling : walker.nextSibling
+    ).bind(walker);
+    while (nodeIterator()) {
+      const element = walker.currentNode.parentElement;
+      if (!this.#isOnSameLine(focusRect, element.getBoundingClientRect())) {
+        newLineElement = element;
+        break;
+      }
+    }
+
+    if (!newLineElement) {
+      // Need to find the next line on the next page.
+      const node = this.#getNodeOnNextPage(root, isUp);
+      if (!node) {
+        return;
+      }
+      if (select) {
+        const lastNode =
+          (isUp ? walker.firstChild() : walker.lastChild()) || focusNode;
+        selection.extend(lastNode, isUp ? 0 : lastNode.length);
+        const range = document.createRange();
+        range.setStart(node, isUp ? node.length : 0);
+        range.setEnd(node, isUp ? node.length : 0);
+        selection.addRange(range);
+        return;
+      }
+      const [caretX] = this.#getCaretPosition(selection, isUp);
+      const { parentElement } = node;
+      this.#setCaretPosition(
+        select,
+        selection,
+        parentElement,
+        parentElement.getBoundingClientRect(),
+        caretX
+      );
+      return;
+    }
+
+    // We've a candidate for the next line now we want to find the first element
+    // which is under/over the caret.
+    const [caretX, caretY] = this.#getCaretPosition(selection, isUp);
+    const newLineElementRect = newLineElement.getBoundingClientRect();
+
+    // Maybe the element on the new line is a valid candidate.
+    if (this.#isUnderOver(newLineElementRect, caretX, caretY, isUp)) {
+      this.#setCaretPosition(
+        select,
+        selection,
+        newLineElement,
+        newLineElementRect,
+        caretX
+      );
+      return;
+    }
+
+    while (nodeIterator()) {
+      // Search an element on the same line as newLineElement
+      // which could be under/over the caret.
+      const element = walker.currentNode.parentElement;
+      const elementRect = element.getBoundingClientRect();
+      if (!this.#isOnSameLine(newLineElementRect, elementRect)) {
+        break;
+      }
+      if (this.#isUnderOver(elementRect, caretX, caretY, isUp)) {
+        // We found the element.
+        this.#setCaretPosition(select, selection, element, elementRect, caretX);
+        return;
+      }
+    }
+
+    // No element has been found so just put the caret on the element on the new
+    // line.
+    this.#setCaretPosition(
+      select,
+      selection,
+      newLineElement,
+      newLineElementRect,
+      caretX
+    );
+  }
+}
+
+export { CaretBrowsingMode };