Output JavaScript modules for the LIB build-target (PR 17055 follow-up)

This *finally* allows us to mark the entire PDF.js library as a "module", which should thus conclude the (multi-year) effort to re-factor and improve how we import files/resources in the code-base.

This also means that the `gulp ci-test` target, which is what's run in GitHub Actions, now uses JavaScript modules since that's supported in modern Node.js versions.

extensions/firefox/.eslintrc

@@ -6,10 +6,6 @@
     "plugin:mozilla/recommended",
   ],
 
-  "parserOptions": {
-    "sourceType": "module",
-  },
-
   "plugins": [
     "mozilla"
   ],

gulpfile.mjs

@@ -222,9 +222,7 @@ function createWebpackConfig(
           },
         ],
       ];
-  const babelPlugins = isModule
+  const babelPlugins = [];
-    ? []
-    : ["@babel/plugin-transform-modules-commonjs"];
 
   const plugins = [];
   if (!disableLicenseHeader) {
@@ -1522,18 +1520,11 @@ gulp.task("types", function (done) {
 });
 
 function buildLibHelper(bundleDefines, inputStream, outputDir) {
-  // When we create a bundle, webpack is run on the source and it will replace
+  function babelPluginReplaceNonWebpackImport(b) {
-  // require with __webpack_require__. When we want to use the real require,
-  // __non_webpack_require__ has to be used.
-  // In this target, we don't create a bundle, so we have to replace the
-  // occurrences of __non_webpack_require__ ourselves.
-  function babelPluginReplaceNonWebpackImports(b) {
     return {
       visitor: {
         Identifier(curPath, state) {
-          if (curPath.node.name === "__non_webpack_require__") {
+          if (curPath.node.name === "__non_webpack_import__") {
-            curPath.replaceWith(b.types.identifier("require"));
-          } else if (curPath.node.name === "__non_webpack_import__") {
             curPath.replaceWith(b.types.identifier("import"));
           }
         },
@@ -1545,18 +1536,15 @@ function buildLibHelper(bundleDefines, inputStream, outputDir) {
     content = preprocessPDFJSCode(ctx, content);
     content = babel.transform(content, {
       sourceType: "module",
-      presets: skipBabel ? undefined : ["@babel/preset-env"],
+      presets: skipBabel
-      plugins: [
+        ? undefined
-        "@babel/plugin-transform-modules-commonjs",
+        : [["@babel/preset-env", { loose: false, modules: false }]],
-        babelPluginReplaceNonWebpackImports,
+      plugins: [babelPluginReplaceNonWebpackImport],
-      ],
       targets: BABEL_TARGETS,
     }).code;
-    const removeCjsSrc =
-      /^(var\s+\w+\s*=\s*(_interopRequireDefault\()?require\(".*?)(?:\/src)(\/[^"]*"\)\)?;)$/gm;
     content = content.replaceAll(
-      removeCjsSrc,
+      /(\sfrom\s".*?)(?:\/src)(\/[^"]*"?;)$/gm,
-      (all, prefix, interop, suffix) => prefix + suffix
+      (all, prefix, suffix) => prefix + suffix
     );
     return licenseHeaderLibre + content;
   }
@@ -1565,12 +1553,12 @@ function buildLibHelper(bundleDefines, inputStream, outputDir) {
     saveComments: false,
     defines: bundleDefines,
     map: {
-      "pdfjs-lib": "../pdf",
+      "pdfjs-lib": "../pdf.js",
-      "display-fetch_stream": "./fetch_stream",
+      "display-fetch_stream": "./fetch_stream.js",
-      "display-l10n_utils": "../web/l10n_utils",
+      "display-l10n_utils": "../web/l10n_utils.js",
-      "display-network": "./network",
+      "display-network": "./network.js",
-      "display-node_stream": "./node_stream",
+      "display-node_stream": "./node_stream.js",
-      "display-node_utils": "./node_utils",
+      "display-node_utils": "./node_utils.js",
     },
   };
   const licenseHeaderLibre = fs

package-lock.json

@@ -9,7 +9,6 @@
       "license": "Apache-2.0",
       "devDependencies": {
         "@babel/core": "^7.22.20",
-        "@babel/plugin-transform-modules-commonjs": "^7.22.15",
         "@babel/preset-env": "^7.22.20",
         "@babel/runtime": "^7.22.15",
         "@javascript-obfuscator/escodegen": "2.3.0",
@@ -13727,7 +13726,6 @@
     },
     "node_modules/npm/node_modules/lodash._baseindexof": {
       "version": "3.1.0",
-      "dev": true,
       "inBundle": true,
       "license": "MIT"
     },
@@ -13743,19 +13741,16 @@
     },
     "node_modules/npm/node_modules/lodash._bindcallback": {
       "version": "3.0.1",
-      "dev": true,
       "inBundle": true,
       "license": "MIT"
     },
     "node_modules/npm/node_modules/lodash._cacheindexof": {
       "version": "3.0.2",
-      "dev": true,
       "inBundle": true,
       "license": "MIT"
     },
     "node_modules/npm/node_modules/lodash._createcache": {
       "version": "3.1.2",
-      "dev": true,
       "inBundle": true,
       "license": "MIT",
       "dependencies": {
@@ -13770,7 +13765,6 @@
     },
     "node_modules/npm/node_modules/lodash._getnative": {
       "version": "3.9.1",
-      "dev": true,
       "inBundle": true,
       "license": "MIT"
     },
@@ -13788,7 +13782,6 @@
     },
     "node_modules/npm/node_modules/lodash.restparam": {
       "version": "3.6.1",
-      "dev": true,
       "inBundle": true,
       "license": "MIT"
     },

package.json

@@ -1,8 +1,8 @@
 {
   "name": "pdf.js",
+  "type": "module",
   "devDependencies": {
     "@babel/core": "^7.22.20",
-    "@babel/plugin-transform-modules-commonjs": "^7.22.15",
     "@babel/preset-env": "^7.22.20",
     "@babel/runtime": "^7.22.15",
     "@javascript-obfuscator/escodegen": "2.3.0",

src/display/node_stream.js

@@ -12,10 +12,12 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+/* globals __non_webpack_import__ */
 
 import {
   AbortException,
   assert,
+  isNodeJS,
   MissingPDFException,
   PromiseCapability,
 } from "../shared/util.js";
@@ -30,10 +32,18 @@ if (typeof PDFJSDev !== "undefined" && PDFJSDev.test("MOZCENTRAL")) {
   );
 }
 
+let fs, http, https, url;
+if (isNodeJS) {
+  // Native packages.
+  fs = await __non_webpack_import__("fs");
+  http = await __non_webpack_import__("http");
+  https = await __non_webpack_import__("https");
+  url = await __non_webpack_import__("url");
+}
+
 const fileUriRegex = /^file:\/\/\/[a-zA-Z]:\//;
 
 function parseUrl(sourceUrl) {
-  const { url } = globalThis.__pdfjsPackages__;
   const parsedUrl = url.parse(sourceUrl);
   if (parsedUrl.protocol === "file:" || parsedUrl.host) {
     return parsedUrl;
@@ -339,13 +349,11 @@ class PDFNodeStreamFullReader extends BaseFullReader {
 
     this._request = null;
     if (this._url.protocol === "http:") {
-      const { http } = globalThis.__pdfjsPackages__;
       this._request = http.request(
         createRequestOptions(this._url, stream.httpHeaders),
         handleResponse
       );
     } else {
-      const { https } = globalThis.__pdfjsPackages__;
       this._request = https.request(
         createRequestOptions(this._url, stream.httpHeaders),
         handleResponse
@@ -388,13 +396,11 @@ class PDFNodeStreamRangeReader extends BaseRangeReader {
 
     this._request = null;
     if (this._url.protocol === "http:") {
-      const { http } = globalThis.__pdfjsPackages__;
       this._request = http.request(
         createRequestOptions(this._url, this._httpHeaders),
         handleResponse
       );
     } else {
-      const { https } = globalThis.__pdfjsPackages__;
       this._request = https.request(
         createRequestOptions(this._url, this._httpHeaders),
         handleResponse
@@ -419,7 +425,6 @@ class PDFNodeStreamFsFullReader extends BaseFullReader {
       path = path.replace(/^\//, "");
     }
 
-    const { fs } = globalThis.__pdfjsPackages__;
     fs.lstat(path, (error, stat) => {
       if (error) {
         if (error.code === "ENOENT") {
@@ -449,7 +454,6 @@ class PDFNodeStreamFsRangeReader extends BaseRangeReader {
       path = path.replace(/^\//, "");
     }
 
-    const { fs } = globalThis.__pdfjsPackages__;
     this._setReadableStream(fs.createReadStream(path, { start, end: end - 1 }));
   }
 }

src/display/node_utils.js

@@ -12,7 +12,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-/* globals __non_webpack_import__, __non_webpack_require__ */
+/* globals __non_webpack_import__ */
 
 import {
   BaseCanvasFactory,
@@ -28,46 +28,17 @@ if (typeof PDFJSDev !== "undefined" && PDFJSDev.test("MOZCENTRAL")) {
   );
 }
 
-if (isNodeJS && !globalThis.__pdfjsPackages__) {
+let fs, canvas, path2d_polyfill;
-  let fs, http, https, url, canvas, path2d_polyfill;
+if (isNodeJS) {
-
+  // Native packages.
-  if (typeof PDFJSDev !== "undefined" && PDFJSDev.test("LIB")) {
+  fs = await __non_webpack_import__("fs");
-    // Native packages.
+  // Optional, third-party, packages.
-    fs = __non_webpack_require__("fs");
+  try {
-    http = __non_webpack_require__("http");
+    canvas = await __non_webpack_import__("canvas");
-    https = __non_webpack_require__("https");
+  } catch {}
-    url = __non_webpack_require__("url");
+  try {
-    // Optional, third-party, packages.
+    path2d_polyfill = await __non_webpack_import__("path2d-polyfill");
-    try {
+  } catch {}
-      canvas = __non_webpack_require__("canvas");
-    } catch {}
-    try {
-      path2d_polyfill = __non_webpack_require__("path2d-polyfill");
-    } catch {}
-  } else {
-    // Native packages.
-    fs = await __non_webpack_import__("fs");
-    http = await __non_webpack_import__("http");
-    https = await __non_webpack_import__("https");
-    url = await __non_webpack_import__("url");
-    // Optional, third-party, packages.
-    try {
-      canvas = await __non_webpack_import__("canvas");
-    } catch {}
-    try {
-      path2d_polyfill = await __non_webpack_import__("path2d-polyfill");
-    } catch {}
-  }
-  globalThis.__pdfjsPackages__ = {
-    CanvasRenderingContext2D: canvas?.CanvasRenderingContext2D,
-    createCanvas: canvas?.createCanvas,
-    DOMMatrix: canvas?.DOMMatrix,
-    fs,
-    http,
-    https,
-    polyfillPath2D: path2d_polyfill?.polyfillPath2D,
-    url,
-  };
 }
 
 if (typeof PDFJSDev !== "undefined" && !PDFJSDev.test("SKIP_BABEL")) {
@@ -75,7 +46,7 @@ if (typeof PDFJSDev !== "undefined" && !PDFJSDev.test("SKIP_BABEL")) {
     if (globalThis.DOMMatrix || !isNodeJS) {
       return;
     }
-    const { DOMMatrix } = globalThis.__pdfjsPackages__;
+    const DOMMatrix = canvas?.DOMMatrix;
 
     if (DOMMatrix) {
       globalThis.DOMMatrix = DOMMatrix;
@@ -88,8 +59,8 @@ if (typeof PDFJSDev !== "undefined" && !PDFJSDev.test("SKIP_BABEL")) {
     if (globalThis.Path2D || !isNodeJS) {
       return;
     }
-    const { CanvasRenderingContext2D, polyfillPath2D } =
+    const CanvasRenderingContext2D = canvas?.CanvasRenderingContext2D;
-      globalThis.__pdfjsPackages__;
+    const polyfillPath2D = path2d_polyfill?.polyfillPath2D;
 
     if (CanvasRenderingContext2D && polyfillPath2D) {
       globalThis.CanvasRenderingContext2D = CanvasRenderingContext2D;
@@ -102,7 +73,6 @@ if (typeof PDFJSDev !== "undefined" && !PDFJSDev.test("SKIP_BABEL")) {
 
 const fetchData = function (url) {
   return new Promise((resolve, reject) => {
-    const { fs } = globalThis.__pdfjsPackages__;
     fs.readFile(url, (error, data) => {
       if (error || !data) {
         reject(new Error(error));
@@ -120,8 +90,7 @@ class NodeCanvasFactory extends BaseCanvasFactory {
    * @ignore
    */
   _createCanvas(width, height) {
-    const { createCanvas } = globalThis.__pdfjsPackages__;
+    return canvas.createCanvas(width, height);
-    return createCanvas(width, height);
   }
 }
 

test/stats/statcmp.js

@@ -1,6 +1,7 @@
-"use strict";
+import { createRequire } from "module";
+import fs from "fs";
 
-const fs = require("fs");
+const require = createRequire(import.meta.url);
 const ttest = require("ttest");
 
 const VALID_GROUP_BYS = ["browser", "pdf", "page", "round", "stat"];

test/unit/node_stream_spec.js

@@ -12,7 +12,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-/* globals __non_webpack_require__ */
+/* globals __non_webpack_import__ */
 
 import { AbortException, isNodeJS } from "../../src/shared/util.js";
 import { PDFNodeStream } from "../../src/display/node_stream.js";
@@ -24,10 +24,10 @@ if (!isNodeJS) {
   );
 }
 
-const path = __non_webpack_require__("path");
+const path = await __non_webpack_import__("path");
-const url = __non_webpack_require__("url");
+const url = await __non_webpack_import__("url");
-const http = __non_webpack_require__("http");
+const http = await __non_webpack_import__("http");
-const fs = __non_webpack_require__("fs");
+const fs = await __non_webpack_import__("fs");
 
 describe("node_stream", function () {
   let server = null;

test/unit/test_utils.js

@@ -12,12 +12,19 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+/* globals __non_webpack_import__ */
 
 import { NullStream, StringStream } from "../../src/core/stream.js";
 import { Page, PDFDocument } from "../../src/core/document.js";
 import { isNodeJS } from "../../src/shared/util.js";
 import { Ref } from "../../src/core/primitives.js";
 
+let fs;
+if (isNodeJS) {
+  // Native packages.
+  fs = await __non_webpack_import__("fs");
+}
+
 const TEST_PDFS_PATH = isNodeJS ? "./test/pdfs/" : "../pdfs/";
 
 const CMAP_URL = isNodeJS ? "./external/bcmaps/" : "../../external/bcmaps/";
@@ -38,8 +45,6 @@ class DOMFileReaderFactory {
 
 class NodeFileReaderFactory {
   static async fetch(params) {
-    const fs = require("fs");
-
     return new Promise((resolve, reject) => {
       fs.readFile(params.path, (error, data) => {
         if (error || !data) {