Improve performance with image masks (bug 857031)

- it aims to partially fix performance issue reported: https://bugzilla.mozilla.org/show_bug.cgi?id=857031;
- the idea is too avoid to use byte arrays but use ImageBitmap which are a way faster to draw:
  * an ImageBitmap is Transferable which means that it can be built in the worker instead of in the main thread:
    - this is achieved in using an OffscreenCanvas when it's available, there is a bug to enable them
      for pdf.js: https://bugzilla.mozilla.org/show_bug.cgi?id=1763330;
    - or in using createImageBitmap: in Firefox a task is sent to the main thread to build the bitmap so
      it's slightly slower than using an OffscreenCanvas.
  * it's transfered from the worker to the main thread by "reference";
  * the byte buffers used to create the image data have a very short lifetime and ergo the memory used is globally
    less than before.
- Use the localImageCache for the mask;
- Fix the pdf issue4436r.pdf: it was expected to have a binary stream for the image;
- Move the singlePixel trick from operator_list to image: this way we can use this trick even if it isn't in a set
  as defined in operator_list.

src/core/evaluator.js

@@ -540,7 +540,7 @@ class PartialEvaluator {
   }
 
   _sendImgData(objId, imgData, cacheGlobally = false) {
-    const transfers = imgData ? [imgData.data.buffer] : null;
+    const transfers = imgData ? [imgData.bitmap || imgData.data.buffer] : null;
 
     if (this.parsingType3Font || cacheGlobally) {
       return this.handler.send(
@@ -612,6 +612,33 @@ class PartialEvaluator {
       );
       const decode = dict.getArray("D", "Decode");
 
+      if (this.parsingType3Font) {
+        imgData = PDFImage.createRawMask({
+          imgArray,
+          width: w,
+          height: h,
+          imageIsFromDecodeStream: image instanceof DecodeStream,
+          inverseDecode: !!decode && decode[0] > 0,
+          interpolate,
+        });
+
+        imgData.cached = !!cacheKey;
+        args = [imgData];
+
+        operatorList.addOp(OPS.paintImageMaskXObject, args);
+        if (cacheKey) {
+          localImageCache.set(cacheKey, imageRef, {
+            fn: OPS.paintImageMaskXObject,
+            args,
+          });
+        }
+
+        if (optionalContent !== undefined) {
+          operatorList.addOp(OPS.endMarkedContent, []);
+        }
+        return;
+      }
+
       imgData = PDFImage.createMask({
         imgArray,
         width: w,
@@ -620,8 +647,36 @@ class PartialEvaluator {
         inverseDecode: !!decode && decode[0] > 0,
         interpolate,
       });
-      imgData.cached = !!cacheKey;
+
-      args = [imgData];
+      if (imgData.isSingleOpaquePixel) {
+        // Handles special case of mainly LaTeX documents which use image
+        // masks to draw lines with the current fill style.
+        operatorList.addOp(OPS.paintSolidColorImageMask, []);
+        if (cacheKey) {
+          localImageCache.set(cacheKey, imageRef, {
+            fn: OPS.paintSolidColorImageMask,
+            args: [],
+          });
+        }
+
+        if (optionalContent !== undefined) {
+          operatorList.addOp(OPS.endMarkedContent, []);
+        }
+        return;
+      }
+
+      const objId = `mask_${this.idFactory.createObjId()}`;
+      operatorList.addDependency(objId);
+      this._sendImgData(objId, imgData);
+
+      args = [
+        {
+          data: objId,
+          width: imgData.width,
+          height: imgData.height,
+          interpolate: imgData.interpolate,
+        },
+      ];
 
       operatorList.addOp(OPS.paintImageMaskXObject, args);
       if (cacheKey) {

src/core/image.js

@@ -13,7 +13,15 @@
  * limitations under the License.
  */
 
-import { assert, FormatError, ImageKind, info, warn } from "../shared/util.js";
+import {
+  assert,
+  FeatureTest,
+  FormatError,
+  ImageKind,
+  info,
+  warn,
+} from "../shared/util.js";
+import { applyMaskImageData } from "../shared/image_utils.js";
 import { BaseStream } from "./base_stream.js";
 import { ColorSpace } from "./colorspace.js";
 import { DecodeStream } from "./decode_stream.js";
@@ -288,7 +296,7 @@ class PDFImage {
     });
   }
 
-  static createMask({
+  static createRawMask({
     imgArray,
     width,
     height,
@@ -302,7 +310,7 @@ class PDFImage {
     ) {
       assert(
         imgArray instanceof Uint8ClampedArray,
-        'PDFImage.createMask: Unsupported "imgArray" type.'
+        'PDFImage.createRawMask: Unsupported "imgArray" type.'
       );
     }
     // |imgArray| might not contain full data for every pixel of the mask, so
@@ -343,6 +351,69 @@ class PDFImage {
     return { data, width, height, interpolate };
   }
 
+  static createMask({
+    imgArray,
+    width,
+    height,
+    imageIsFromDecodeStream,
+    inverseDecode,
+    interpolate,
+  }) {
+    if (
+      typeof PDFJSDev === "undefined" ||
+      PDFJSDev.test("!PRODUCTION || TESTING")
+    ) {
+      assert(
+        imgArray instanceof Uint8ClampedArray,
+        'PDFImage.createMask: Unsupported "imgArray" type.'
+      );
+    }
+
+    const isSingleOpaquePixel =
+      width === 1 &&
+      height === 1 &&
+      inverseDecode === (imgArray.length === 0 || !!(imgArray[0] & 128));
+
+    if (isSingleOpaquePixel) {
+      return { isSingleOpaquePixel };
+    }
+
+    if (FeatureTest.isOffscreenCanvasSupported) {
+      const canvas = new OffscreenCanvas(width, height);
+      const ctx = canvas.getContext("2d");
+      const imgData = ctx.createImageData(width, height);
+      applyMaskImageData({
+        src: imgArray,
+        dest: imgData.data,
+        width,
+        height,
+        inverseDecode,
+      });
+
+      ctx.putImageData(imgData, 0, 0);
+      const bitmap = canvas.transferToImageBitmap();
+
+      return {
+        data: null,
+        width,
+        height,
+        interpolate,
+        bitmap,
+      };
+    }
+
+    // Get the data almost as they're and they'll be decoded
+    // just before being drawn.
+    return this.createRawMask({
+      imgArray,
+      width,
+      height,
+      inverseDecode,
+      imageIsFromDecodeStream,
+      interpolate,
+    });
+  }
+
   get drawWidth() {
     return Math.max(
       this.width,

src/core/operator_list.js

@@ -35,31 +35,6 @@ function addState(parentState, pattern, checkFn, iterateFn, processFn) {
   };
 }
 
-function handlePaintSolidColorImageMask(iFirstSave, count, fnArray, argsArray) {
-  // Handles special case of mainly LaTeX documents which use image masks to
-  // draw lines with the current fill style.
-  // 'count' groups of (save, transform, paintImageMaskXObject, restore)+
-  // have been found at iFirstSave.
-  const iFirstPIMXO = iFirstSave + 2;
-  let i;
-  for (i = 0; i < count; i++) {
-    const arg = argsArray[iFirstPIMXO + 4 * i];
-    const imageMask = arg.length === 1 && arg[0];
-    if (
-      imageMask &&
-      imageMask.width === 1 &&
-      imageMask.height === 1 &&
-      (!imageMask.data.length ||
-        (imageMask.data.length === 1 && imageMask.data[0] === 0))
-    ) {
-      fnArray[iFirstPIMXO + 4 * i] = OPS.paintSolidColorImageMask;
-      continue;
-    }
-    break;
-  }
-  return count - i;
-}
-
 const InitialState = [];
 
 // This replaces (save, transform, paintInlineImageXObject, restore)+
@@ -216,12 +191,6 @@ addState(
     // At this point, i is the index of the first op past the last valid
     // quartet.
     let count = Math.floor((i - iFirstSave) / 4);
-    count = handlePaintSolidColorImageMask(
-      iFirstSave,
-      count,
-      fnArray,
-      argsArray
-    );
     if (count < MIN_IMAGES_IN_MASKS_BLOCK) {
       return i - ((i - iFirstSave) % 4);
     }
@@ -701,11 +670,16 @@ class OperatorList {
             PDFJSDev.test("!PRODUCTION || TESTING")
           ) {
             assert(
-              arg.data instanceof Uint8ClampedArray,
+              arg.data instanceof Uint8ClampedArray ||
+                typeof arg.data === "string",
               'OperatorList._transfers: Unsupported "arg.data" type.'
             );
           }
-          if (!arg.cached) {
+          if (
+            !arg.cached &&
+            arg.data &&
+            arg.data.buffer instanceof ArrayBuffer
+          ) {
             transfers.push(arg.data.buffer);
           }
           break;

src/display/api.js

@@ -1241,6 +1241,8 @@ class PDFPageProxy {
     this.commonObjs = transport.commonObjs;
     this.objs = new PDFObjects();
 
+    this._bitmaps = new Set();
+
     this.cleanupAfterRender = false;
     this.pendingCleanup = false;
     this._intentStates = new Map();
@@ -1696,6 +1698,10 @@ class PDFPageProxy {
       }
     }
     this.objs.clear();
+    for (const bitmap of this._bitmaps) {
+      bitmap.close();
+    }
+    this._bitmaps.clear();
     this._annotationPromises.clear();
     this._jsActionsPromise = null;
     this._structTreePromise = null;
@@ -1737,6 +1743,10 @@ class PDFPageProxy {
     if (resetStats && this._stats) {
       this._stats = new StatTimer();
     }
+    for (const bitmap of this._bitmaps) {
+      bitmap.close();
+    }
+    this._bitmaps.clear();
     this.pendingCleanup = false;
     return true;
   }
@@ -2778,8 +2788,19 @@ class WorkerTransport {
 
           // Heuristic that will allow us not to store large data.
           const MAX_IMAGE_SIZE_TO_STORE = 8000000;
-          if (imageData?.data?.length > MAX_IMAGE_SIZE_TO_STORE) {
+          if (imageData) {
-            pageProxy.cleanupAfterRender = true;
+            let length;
+            if (imageData.bitmap) {
+              const { bitmap, width, height } = imageData;
+              length = width * height * 4;
+              pageProxy._bitmaps.add(bitmap);
+            } else {
+              length = imageData.data?.length || 0;
+            }
+
+            if (length > MAX_IMAGE_SIZE_TO_STORE) {
+              pageProxy.cleanupAfterRender = true;
+            }
           }
           break;
         case "Pattern":

src/display/canvas.js

@@ -31,6 +31,7 @@ import {
   PathType,
   TilingPattern,
 } from "./pattern_helper.js";
+import { applyMaskImageData } from "../shared/image_utils.js";
 import { PixelsPerInch } from "./display_utils.js";
 
 // <canvas> contexts store most of the state we need natively.
@@ -845,6 +846,13 @@ function putBinaryImageData(ctx, imgData, transferMaps = null) {
 }
 
 function putBinaryImageMask(ctx, imgData) {
+  if (imgData.bitmap) {
+    // The bitmap has been created in the worker.
+    ctx.drawImage(imgData.bitmap, 0, 0);
+    return;
+  }
+
+  // Slow path: OffscreenCanvas isn't available in the worker.
   const height = imgData.height,
     width = imgData.width;
   const partialChunkHeight = height % FULL_CHUNK_HEIGHT;
@@ -862,20 +870,15 @@ function putBinaryImageMask(ctx, imgData) {
 
     // Expand the mask so it can be used by the canvas.  Any required
     // inversion has already been handled.
-    let destPos = 3; // alpha component offset
+
-    for (let j = 0; j < thisChunkHeight; j++) {
+    ({ srcPos } = applyMaskImageData({
-      let elem,
+      src,
-        mask = 0;
+      srcPos,
-      for (let k = 0; k < width; k++) {
+      dest,
-        if (!mask) {
+      width,
-          elem = src[srcPos++];
+      height: thisChunkHeight,
-          mask = 128;
+    }));
-        }
+
-        dest[destPos] = elem & mask ? 0 : 255;
-        destPos += 4;
-        mask >>= 1;
-      }
-    }
     ctx.putImageData(chunkImgData, 0, i * FULL_CHUNK_HEIGHT);
   }
 }
@@ -1120,6 +1123,15 @@ class CanvasGraphics {
     this._cachedGetSinglePixelWidth = null;
   }
 
+  getObject(data, fallback = null) {
+    if (typeof data === "string") {
+      return data.startsWith("g_")
+        ? this.commonObjs.get(data)
+        : this.objs.get(data);
+    }
+    return fallback;
+  }
+
   beginDrawing({
     transform,
     viewport,
@@ -2754,6 +2766,9 @@ class CanvasGraphics {
     if (!this.contentVisible) {
       return;
     }
+
+    img = this.getObject(img.data, img);
+
     const ctx = this.ctx;
     const width = img.width,
       height = img.height;
@@ -2785,7 +2800,7 @@ class CanvasGraphics {
   }
 
   paintImageMaskXObjectRepeat(
-    imgData,
+    img,
     scaleX,
     skewX = 0,
     skewY = 0,
@@ -2795,11 +2810,14 @@ class CanvasGraphics {
     if (!this.contentVisible) {
       return;
     }
+
+    img = this.getObject(img.data, img);
+
     const ctx = this.ctx;
     ctx.save();
     const currentTransform = ctx.mozCurrentTransform;
     ctx.transform(scaleX, skewX, skewY, scaleY, 0, 0);
-    const mask = this._createMaskCanvas(imgData);
+    const mask = this._createMaskCanvas(img);
 
     ctx.setTransform(1, 0, 0, 1, 0, 0);
     for (let i = 0, ii = positions.length; i < ii; i += 2) {
@@ -2869,9 +2887,7 @@ class CanvasGraphics {
     if (!this.contentVisible) {
       return;
     }
-    const imgData = objId.startsWith("g_")
+    const imgData = this.getObject(objId);
-      ? this.commonObjs.get(objId)
-      : this.objs.get(objId);
     if (!imgData) {
       warn("Dependent image isn't ready yet");
       return;
@@ -2884,9 +2900,7 @@ class CanvasGraphics {
     if (!this.contentVisible) {
       return;
     }
-    const imgData = objId.startsWith("g_")
+    const imgData = this.getObject(objId);
-      ? this.commonObjs.get(objId)
-      : this.objs.get(objId);
     if (!imgData) {
       warn("Dependent image isn't ready yet");
       return;

src/shared/image_utils.js

@@ -0,0 +1,46 @@
+/* Copyright 2022 Mozilla Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+function applyMaskImageData({
+  src,
+  srcPos = 0,
+  dest,
+  destPos = 3,
+  width,
+  height,
+  inverseDecode = false,
+}) {
+  const srcLength = src.byteLength;
+  const zeroMapping = inverseDecode ? 0 : 255;
+  const oneMapping = inverseDecode ? 255 : 0;
+
+  for (let j = 0; j < height; j++) {
+    let elem,
+      mask = 0;
+    for (let k = 0; k < width; k++) {
+      if (mask === 0) {
+        elem = srcPos < srcLength ? src[srcPos++] : 255;
+        mask = 128;
+      }
+      dest[destPos] = elem & mask ? oneMapping : zeroMapping;
+      destPos += 4;
+      mask >>= 1;
+    }
+  }
+
+  return { srcPos, destPos };
+}
+
+export { applyMaskImageData };

src/shared/util.js

@@ -701,6 +701,14 @@ class FeatureTest {
   static get isEvalSupported() {
     return shadow(this, "isEvalSupported", isEvalSupported());
   }
+
+  static get isOffscreenCanvasSupported() {
+    return shadow(
+      this,
+      "isOffscreenCanvasSupported",
+      typeof OffscreenCanvas !== "undefined"
+    );
+  }
 }
 
 const hexNumbers = [...Array(256).keys()].map(n =>

test/test.js

@@ -959,6 +959,8 @@ async function startBrowser(browserName, startUrl = "") {
       print_printer: "PDF",
       "print.printer_PDF.print_to_file": true,
       "print.printer_PDF.print_to_filename": printFile,
+      // Enable OffscreenCanvas
+      "gfx.offscreencanvas.enabled": true,
     };
   }
 

test/test_manifest.json

@@ -5576,7 +5576,7 @@
     },
     {  "id": "issue4436",
        "file": "pdfs/issue4436r.pdf",
-       "md5": "4e43d692d213f56674fcac92110c7364",
+       "md5": "f5dc60cce342ac8d165069a80d23b92e",
        "rounds": 1,
        "link": false,
        "type": "eq"

test/unit/api_spec.js

@@ -1666,7 +1666,7 @@ describe("api", function () {
 
       expect(fingerprints1).not.toEqual(fingerprints2);
 
-      expect(fingerprints1).toEqual(["2f695a83d6e7553c24fc08b7ac69712d", null]);
+      expect(fingerprints1).toEqual(["657428c0628e329f9a281fb6d2d092d4", null]);
       expect(fingerprints2).toEqual(["04c7126b34a46b6d4d6e7a1eff7edcb6", null]);
 
       await Promise.all([loadingTask1.destroy(), loadingTask2.destroy()]);