From 040fcae5ab08d1116ccfdd0cee45c1ffefff6c0e Mon Sep 17 00:00:00 2001 From: Calixte Denizet Date: Wed, 6 Apr 2022 15:34:08 +0200 Subject: [PATCH] Improve performance with image masks (bug 857031) - it aims to partially fix performance issue reported: https://bugzilla.mozilla.org/show_bug.cgi?id=857031; - the idea is too avoid to use byte arrays but use ImageBitmap which are a way faster to draw: * an ImageBitmap is Transferable which means that it can be built in the worker instead of in the main thread: - this is achieved in using an OffscreenCanvas when it's available, there is a bug to enable them for pdf.js: https://bugzilla.mozilla.org/show_bug.cgi?id=1763330; - or in using createImageBitmap: in Firefox a task is sent to the main thread to build the bitmap so it's slightly slower than using an OffscreenCanvas. * it's transfered from the worker to the main thread by "reference"; * the byte buffers used to create the image data have a very short lifetime and ergo the memory used is globally less than before. - Use the localImageCache for the mask; - Fix the pdf issue4436r.pdf: it was expected to have a binary stream for the image; - Move the singlePixel trick from operator_list to image: this way we can use this trick even if it isn't in a set as defined in operator_list. --- src/core/evaluator.js | 61 ++++++++++++++++++++++++++++-- src/core/image.js | 77 ++++++++++++++++++++++++++++++++++++-- src/core/operator_list.js | 40 ++++---------------- src/display/api.js | 25 ++++++++++++- src/display/canvas.js | 58 +++++++++++++++++----------- src/shared/image_utils.js | 46 +++++++++++++++++++++++ src/shared/util.js | 8 ++++ test/pdfs/issue4436r.pdf | Bin 777 -> 777 bytes test/test.js | 2 + test/test_manifest.json | 2 +- test/unit/api_spec.js | 2 +- 11 files changed, 256 insertions(+), 65 deletions(-) create mode 100644 src/shared/image_utils.js diff --git a/src/core/evaluator.js b/src/core/evaluator.js index 4caa568eb..ec4157374 100644 --- a/src/core/evaluator.js +++ b/src/core/evaluator.js @@ -540,7 +540,7 @@ class PartialEvaluator { } _sendImgData(objId, imgData, cacheGlobally = false) { - const transfers = imgData ? [imgData.data.buffer] : null; + const transfers = imgData ? [imgData.bitmap || imgData.data.buffer] : null; if (this.parsingType3Font || cacheGlobally) { return this.handler.send( @@ -612,6 +612,33 @@ class PartialEvaluator { ); const decode = dict.getArray("D", "Decode"); + if (this.parsingType3Font) { + imgData = PDFImage.createRawMask({ + imgArray, + width: w, + height: h, + imageIsFromDecodeStream: image instanceof DecodeStream, + inverseDecode: !!decode && decode[0] > 0, + interpolate, + }); + + imgData.cached = !!cacheKey; + args = [imgData]; + + operatorList.addOp(OPS.paintImageMaskXObject, args); + if (cacheKey) { + localImageCache.set(cacheKey, imageRef, { + fn: OPS.paintImageMaskXObject, + args, + }); + } + + if (optionalContent !== undefined) { + operatorList.addOp(OPS.endMarkedContent, []); + } + return; + } + imgData = PDFImage.createMask({ imgArray, width: w, @@ -620,8 +647,36 @@ class PartialEvaluator { inverseDecode: !!decode && decode[0] > 0, interpolate, }); - imgData.cached = !!cacheKey; - args = [imgData]; + + if (imgData.isSingleOpaquePixel) { + // Handles special case of mainly LaTeX documents which use image + // masks to draw lines with the current fill style. + operatorList.addOp(OPS.paintSolidColorImageMask, []); + if (cacheKey) { + localImageCache.set(cacheKey, imageRef, { + fn: OPS.paintSolidColorImageMask, + args: [], + }); + } + + if (optionalContent !== undefined) { + operatorList.addOp(OPS.endMarkedContent, []); + } + return; + } + + const objId = `mask_${this.idFactory.createObjId()}`; + operatorList.addDependency(objId); + this._sendImgData(objId, imgData); + + args = [ + { + data: objId, + width: imgData.width, + height: imgData.height, + interpolate: imgData.interpolate, + }, + ]; operatorList.addOp(OPS.paintImageMaskXObject, args); if (cacheKey) { diff --git a/src/core/image.js b/src/core/image.js index 322903b58..1c3c721c7 100644 --- a/src/core/image.js +++ b/src/core/image.js @@ -13,7 +13,15 @@ * limitations under the License. */ -import { assert, FormatError, ImageKind, info, warn } from "../shared/util.js"; +import { + assert, + FeatureTest, + FormatError, + ImageKind, + info, + warn, +} from "../shared/util.js"; +import { applyMaskImageData } from "../shared/image_utils.js"; import { BaseStream } from "./base_stream.js"; import { ColorSpace } from "./colorspace.js"; import { DecodeStream } from "./decode_stream.js"; @@ -288,7 +296,7 @@ class PDFImage { }); } - static createMask({ + static createRawMask({ imgArray, width, height, @@ -302,7 +310,7 @@ class PDFImage { ) { assert( imgArray instanceof Uint8ClampedArray, - 'PDFImage.createMask: Unsupported "imgArray" type.' + 'PDFImage.createRawMask: Unsupported "imgArray" type.' ); } // |imgArray| might not contain full data for every pixel of the mask, so @@ -343,6 +351,69 @@ class PDFImage { return { data, width, height, interpolate }; } + static createMask({ + imgArray, + width, + height, + imageIsFromDecodeStream, + inverseDecode, + interpolate, + }) { + if ( + typeof PDFJSDev === "undefined" || + PDFJSDev.test("!PRODUCTION || TESTING") + ) { + assert( + imgArray instanceof Uint8ClampedArray, + 'PDFImage.createMask: Unsupported "imgArray" type.' + ); + } + + const isSingleOpaquePixel = + width === 1 && + height === 1 && + inverseDecode === (imgArray.length === 0 || !!(imgArray[0] & 128)); + + if (isSingleOpaquePixel) { + return { isSingleOpaquePixel }; + } + + if (FeatureTest.isOffscreenCanvasSupported) { + const canvas = new OffscreenCanvas(width, height); + const ctx = canvas.getContext("2d"); + const imgData = ctx.createImageData(width, height); + applyMaskImageData({ + src: imgArray, + dest: imgData.data, + width, + height, + inverseDecode, + }); + + ctx.putImageData(imgData, 0, 0); + const bitmap = canvas.transferToImageBitmap(); + + return { + data: null, + width, + height, + interpolate, + bitmap, + }; + } + + // Get the data almost as they're and they'll be decoded + // just before being drawn. + return this.createRawMask({ + imgArray, + width, + height, + inverseDecode, + imageIsFromDecodeStream, + interpolate, + }); + } + get drawWidth() { return Math.max( this.width, diff --git a/src/core/operator_list.js b/src/core/operator_list.js index 6f0c3f73d..0ffa33ab4 100644 --- a/src/core/operator_list.js +++ b/src/core/operator_list.js @@ -35,31 +35,6 @@ function addState(parentState, pattern, checkFn, iterateFn, processFn) { }; } -function handlePaintSolidColorImageMask(iFirstSave, count, fnArray, argsArray) { - // Handles special case of mainly LaTeX documents which use image masks to - // draw lines with the current fill style. - // 'count' groups of (save, transform, paintImageMaskXObject, restore)+ - // have been found at iFirstSave. - const iFirstPIMXO = iFirstSave + 2; - let i; - for (i = 0; i < count; i++) { - const arg = argsArray[iFirstPIMXO + 4 * i]; - const imageMask = arg.length === 1 && arg[0]; - if ( - imageMask && - imageMask.width === 1 && - imageMask.height === 1 && - (!imageMask.data.length || - (imageMask.data.length === 1 && imageMask.data[0] === 0)) - ) { - fnArray[iFirstPIMXO + 4 * i] = OPS.paintSolidColorImageMask; - continue; - } - break; - } - return count - i; -} - const InitialState = []; // This replaces (save, transform, paintInlineImageXObject, restore)+ @@ -216,12 +191,6 @@ addState( // At this point, i is the index of the first op past the last valid // quartet. let count = Math.floor((i - iFirstSave) / 4); - count = handlePaintSolidColorImageMask( - iFirstSave, - count, - fnArray, - argsArray - ); if (count < MIN_IMAGES_IN_MASKS_BLOCK) { return i - ((i - iFirstSave) % 4); } @@ -701,11 +670,16 @@ class OperatorList { PDFJSDev.test("!PRODUCTION || TESTING") ) { assert( - arg.data instanceof Uint8ClampedArray, + arg.data instanceof Uint8ClampedArray || + typeof arg.data === "string", 'OperatorList._transfers: Unsupported "arg.data" type.' ); } - if (!arg.cached) { + if ( + !arg.cached && + arg.data && + arg.data.buffer instanceof ArrayBuffer + ) { transfers.push(arg.data.buffer); } break; diff --git a/src/display/api.js b/src/display/api.js index e065eb3e1..dea887e43 100644 --- a/src/display/api.js +++ b/src/display/api.js @@ -1241,6 +1241,8 @@ class PDFPageProxy { this.commonObjs = transport.commonObjs; this.objs = new PDFObjects(); + this._bitmaps = new Set(); + this.cleanupAfterRender = false; this.pendingCleanup = false; this._intentStates = new Map(); @@ -1696,6 +1698,10 @@ class PDFPageProxy { } } this.objs.clear(); + for (const bitmap of this._bitmaps) { + bitmap.close(); + } + this._bitmaps.clear(); this._annotationPromises.clear(); this._jsActionsPromise = null; this._structTreePromise = null; @@ -1737,6 +1743,10 @@ class PDFPageProxy { if (resetStats && this._stats) { this._stats = new StatTimer(); } + for (const bitmap of this._bitmaps) { + bitmap.close(); + } + this._bitmaps.clear(); this.pendingCleanup = false; return true; } @@ -2778,8 +2788,19 @@ class WorkerTransport { // Heuristic that will allow us not to store large data. const MAX_IMAGE_SIZE_TO_STORE = 8000000; - if (imageData?.data?.length > MAX_IMAGE_SIZE_TO_STORE) { - pageProxy.cleanupAfterRender = true; + if (imageData) { + let length; + if (imageData.bitmap) { + const { bitmap, width, height } = imageData; + length = width * height * 4; + pageProxy._bitmaps.add(bitmap); + } else { + length = imageData.data?.length || 0; + } + + if (length > MAX_IMAGE_SIZE_TO_STORE) { + pageProxy.cleanupAfterRender = true; + } } break; case "Pattern": diff --git a/src/display/canvas.js b/src/display/canvas.js index 3dc0ad1de..2f3b5ea77 100644 --- a/src/display/canvas.js +++ b/src/display/canvas.js @@ -31,6 +31,7 @@ import { PathType, TilingPattern, } from "./pattern_helper.js"; +import { applyMaskImageData } from "../shared/image_utils.js"; import { PixelsPerInch } from "./display_utils.js"; // contexts store most of the state we need natively. @@ -845,6 +846,13 @@ function putBinaryImageData(ctx, imgData, transferMaps = null) { } function putBinaryImageMask(ctx, imgData) { + if (imgData.bitmap) { + // The bitmap has been created in the worker. + ctx.drawImage(imgData.bitmap, 0, 0); + return; + } + + // Slow path: OffscreenCanvas isn't available in the worker. const height = imgData.height, width = imgData.width; const partialChunkHeight = height % FULL_CHUNK_HEIGHT; @@ -862,20 +870,15 @@ function putBinaryImageMask(ctx, imgData) { // Expand the mask so it can be used by the canvas. Any required // inversion has already been handled. - let destPos = 3; // alpha component offset - for (let j = 0; j < thisChunkHeight; j++) { - let elem, - mask = 0; - for (let k = 0; k < width; k++) { - if (!mask) { - elem = src[srcPos++]; - mask = 128; - } - dest[destPos] = elem & mask ? 0 : 255; - destPos += 4; - mask >>= 1; - } - } + + ({ srcPos } = applyMaskImageData({ + src, + srcPos, + dest, + width, + height: thisChunkHeight, + })); + ctx.putImageData(chunkImgData, 0, i * FULL_CHUNK_HEIGHT); } } @@ -1120,6 +1123,15 @@ class CanvasGraphics { this._cachedGetSinglePixelWidth = null; } + getObject(data, fallback = null) { + if (typeof data === "string") { + return data.startsWith("g_") + ? this.commonObjs.get(data) + : this.objs.get(data); + } + return fallback; + } + beginDrawing({ transform, viewport, @@ -2754,6 +2766,9 @@ class CanvasGraphics { if (!this.contentVisible) { return; } + + img = this.getObject(img.data, img); + const ctx = this.ctx; const width = img.width, height = img.height; @@ -2785,7 +2800,7 @@ class CanvasGraphics { } paintImageMaskXObjectRepeat( - imgData, + img, scaleX, skewX = 0, skewY = 0, @@ -2795,11 +2810,14 @@ class CanvasGraphics { if (!this.contentVisible) { return; } + + img = this.getObject(img.data, img); + const ctx = this.ctx; ctx.save(); const currentTransform = ctx.mozCurrentTransform; ctx.transform(scaleX, skewX, skewY, scaleY, 0, 0); - const mask = this._createMaskCanvas(imgData); + const mask = this._createMaskCanvas(img); ctx.setTransform(1, 0, 0, 1, 0, 0); for (let i = 0, ii = positions.length; i < ii; i += 2) { @@ -2869,9 +2887,7 @@ class CanvasGraphics { if (!this.contentVisible) { return; } - const imgData = objId.startsWith("g_") - ? this.commonObjs.get(objId) - : this.objs.get(objId); + const imgData = this.getObject(objId); if (!imgData) { warn("Dependent image isn't ready yet"); return; @@ -2884,9 +2900,7 @@ class CanvasGraphics { if (!this.contentVisible) { return; } - const imgData = objId.startsWith("g_") - ? this.commonObjs.get(objId) - : this.objs.get(objId); + const imgData = this.getObject(objId); if (!imgData) { warn("Dependent image isn't ready yet"); return; diff --git a/src/shared/image_utils.js b/src/shared/image_utils.js new file mode 100644 index 000000000..69d843467 --- /dev/null +++ b/src/shared/image_utils.js @@ -0,0 +1,46 @@ +/* Copyright 2022 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +function applyMaskImageData({ + src, + srcPos = 0, + dest, + destPos = 3, + width, + height, + inverseDecode = false, +}) { + const srcLength = src.byteLength; + const zeroMapping = inverseDecode ? 0 : 255; + const oneMapping = inverseDecode ? 255 : 0; + + for (let j = 0; j < height; j++) { + let elem, + mask = 0; + for (let k = 0; k < width; k++) { + if (mask === 0) { + elem = srcPos < srcLength ? src[srcPos++] : 255; + mask = 128; + } + dest[destPos] = elem & mask ? oneMapping : zeroMapping; + destPos += 4; + mask >>= 1; + } + } + + return { srcPos, destPos }; +} + +export { applyMaskImageData }; diff --git a/src/shared/util.js b/src/shared/util.js index ee0a7eceb..67ad8c388 100644 --- a/src/shared/util.js +++ b/src/shared/util.js @@ -701,6 +701,14 @@ class FeatureTest { static get isEvalSupported() { return shadow(this, "isEvalSupported", isEvalSupported()); } + + static get isOffscreenCanvasSupported() { + return shadow( + this, + "isOffscreenCanvasSupported", + typeof OffscreenCanvas !== "undefined" + ); + } } const hexNumbers = [...Array(256).keys()].map(n => diff --git a/test/pdfs/issue4436r.pdf b/test/pdfs/issue4436r.pdf index bf8504c6c04c8bde82f0da5f0890b1f0f1a0d6d3..cbb0db7b85450eff00e7e24e912c0242a178b35e 100644 GIT binary patch delta 16 WcmeBV>tx$t#>Bz^1e2|p&I14+T?45A delta 16 XcmeBV>tx$t#>8S!VPG)Xis?K6CN%`H diff --git a/test/test.js b/test/test.js index 9cb619932..5bc020252 100644 --- a/test/test.js +++ b/test/test.js @@ -959,6 +959,8 @@ async function startBrowser(browserName, startUrl = "") { print_printer: "PDF", "print.printer_PDF.print_to_file": true, "print.printer_PDF.print_to_filename": printFile, + // Enable OffscreenCanvas + "gfx.offscreencanvas.enabled": true, }; } diff --git a/test/test_manifest.json b/test/test_manifest.json index c067b7f4e..8607b9560 100644 --- a/test/test_manifest.json +++ b/test/test_manifest.json @@ -5576,7 +5576,7 @@ }, { "id": "issue4436", "file": "pdfs/issue4436r.pdf", - "md5": "4e43d692d213f56674fcac92110c7364", + "md5": "f5dc60cce342ac8d165069a80d23b92e", "rounds": 1, "link": false, "type": "eq" diff --git a/test/unit/api_spec.js b/test/unit/api_spec.js index 081947830..9f44d5205 100644 --- a/test/unit/api_spec.js +++ b/test/unit/api_spec.js @@ -1666,7 +1666,7 @@ describe("api", function () { expect(fingerprints1).not.toEqual(fingerprints2); - expect(fingerprints1).toEqual(["2f695a83d6e7553c24fc08b7ac69712d", null]); + expect(fingerprints1).toEqual(["657428c0628e329f9a281fb6d2d092d4", null]); expect(fingerprints2).toEqual(["04c7126b34a46b6d4d6e7a1eff7edcb6", null]); await Promise.all([loadingTask1.destroy(), loadingTask2.destroy()]);