diff --git a/src/core/evaluator.js b/src/core/evaluator.js
index 4a307f142..05e033244 100644
--- a/src/core/evaluator.js
+++ b/src/core/evaluator.js
@@ -79,6 +79,7 @@ import { DecodeStream } from "./stream.js";
 import { getGlyphsUnicode } from "./glyphlist.js";
 import { getMetrics } from "./metrics.js";
 import { isPDFFunction } from "./function.js";
+import { LocalImageCache } from "./image_utils.js";
 import { MurmurHash3_64 } from "./murmurhash3.js";
 import { OperatorList } from "./operator_list.js";
 import { PDFImage } from "./image.js";
@@ -444,7 +445,7 @@ var PartialEvaluator = (function PartialEvaluatorClosure() {
       isInline = false,
       operatorList,
       cacheKey,
-      imageCache,
+      localImageCache,
     }) {
       var dict = image.dict;
       const imageRef = dict.objId;
@@ -491,10 +492,10 @@ var PartialEvaluator = (function PartialEvaluatorClosure() {
 
         operatorList.addOp(OPS.paintImageMaskXObject, args);
         if (cacheKey) {
-          imageCache[cacheKey] = {
+          localImageCache.set(cacheKey, imageRef, {
             fn: OPS.paintImageMaskXObject,
             args,
-          };
+          });
         }
         return undefined;
       }
@@ -598,10 +599,10 @@ var PartialEvaluator = (function PartialEvaluatorClosure() {
 
       operatorList.addOp(OPS.paintImageXObject, args);
       if (cacheKey) {
-        imageCache[cacheKey] = {
+        localImageCache.set(cacheKey, imageRef, {
           fn: OPS.paintImageXObject,
           args,
-        };
+        });
 
         if (imageRef) {
           this.globalImageCache.addPageIndex(imageRef, this.pageIndex);
@@ -1201,7 +1202,7 @@ var PartialEvaluator = (function PartialEvaluatorClosure() {
       var self = this;
       var xref = this.xref;
       let parsingText = false;
-      var imageCache = Object.create(null);
+      const localImageCache = new LocalImageCache();
 
       var xobjs = resources.get("XObject") || Dict.empty;
       var patterns = resources.get("Pattern") || Dict.empty;
@@ -1248,10 +1249,13 @@ var PartialEvaluator = (function PartialEvaluatorClosure() {
             case OPS.paintXObject:
               // eagerly compile XForm objects
               var name = args[0].name;
-              if (name && imageCache[name] !== undefined) {
-                operatorList.addOp(imageCache[name].fn, imageCache[name].args);
-                args = null;
-                continue;
+              if (name) {
+                const localImage = localImageCache.getByName(name);
+                if (localImage) {
+                  operatorList.addOp(localImage.fn, localImage.args);
+                  args = null;
+                  continue;
+                }
               }
 
               next(
@@ -1264,11 +1268,18 @@ var PartialEvaluator = (function PartialEvaluatorClosure() {
 
                   let xobj = xobjs.getRaw(name);
                   if (xobj instanceof Ref) {
+                    const localImage = localImageCache.getByRef(xobj);
+                    if (localImage) {
+                      operatorList.addOp(localImage.fn, localImage.args);
+
+                      resolveXObject();
+                      return;
+                    }
+
                     const globalImage = self.globalImageCache.getData(
                       xobj,
                       self.pageIndex
                     );
-
                     if (globalImage) {
                       operatorList.addDependency(globalImage.objId);
                       operatorList.addOp(globalImage.fn, globalImage.args);
@@ -1276,6 +1287,7 @@ var PartialEvaluator = (function PartialEvaluatorClosure() {
                       resolveXObject();
                       return;
                     }
+
                     xobj = xref.fetch(xobj);
                   }
 
@@ -1316,7 +1328,7 @@ var PartialEvaluator = (function PartialEvaluatorClosure() {
                         image: xobj,
                         operatorList,
                         cacheKey: name,
-                        imageCache,
+                        localImageCache,
                       })
                       .then(resolveXObject, rejectXObject);
                     return;
@@ -1375,9 +1387,9 @@ var PartialEvaluator = (function PartialEvaluatorClosure() {
             case OPS.endInlineImage:
               var cacheKey = args[0].cacheKey;
               if (cacheKey) {
-                var cacheEntry = imageCache[cacheKey];
-                if (cacheEntry !== undefined) {
-                  operatorList.addOp(cacheEntry.fn, cacheEntry.args);
+                const localImage = localImageCache.getByName(cacheKey);
+                if (localImage) {
+                  operatorList.addOp(localImage.fn, localImage.args);
                   args = null;
                   continue;
                 }
@@ -1389,7 +1401,7 @@ var PartialEvaluator = (function PartialEvaluatorClosure() {
                   isInline: true,
                   operatorList,
                   cacheKey,
-                  imageCache,
+                  localImageCache,
                 })
               );
               return;
@@ -1712,7 +1724,7 @@ var PartialEvaluator = (function PartialEvaluatorClosure() {
 
       // The xobj is parsed iff it's needed, e.g. if there is a `DO` cmd.
       var xobjs = null;
-      var skipEmptyXObjs = Object.create(null);
+      const emptyXObjectCache = new LocalImageCache();
 
       var preprocessor = new EvaluatorPreprocessor(stream, xref, stateManager);
 
@@ -2188,7 +2200,7 @@ var PartialEvaluator = (function PartialEvaluatorClosure() {
               }
 
               var name = args[0].name;
-              if (name && skipEmptyXObjs[name] !== undefined) {
+              if (name && emptyXObjectCache.getByName(name)) {
                 break;
               }
 
@@ -2200,7 +2212,16 @@ var PartialEvaluator = (function PartialEvaluatorClosure() {
                     );
                   }
 
-                  const xobj = xobjs.get(name);
+                  let xobj = xobjs.getRaw(name);
+                  if (xobj instanceof Ref) {
+                    if (emptyXObjectCache.getByRef(xobj)) {
+                      resolveXObject();
+                      return;
+                    }
+
+                    xobj = xref.fetch(xobj);
+                  }
+
                   if (!xobj) {
                     resolveXObject();
                     return;
@@ -2215,7 +2236,8 @@ var PartialEvaluator = (function PartialEvaluatorClosure() {
                   }
 
                   if (type.name !== "Form") {
-                    skipEmptyXObjs[name] = true;
+                    emptyXObjectCache.set(name, xobj.dict.objId, true);
+
                     resolveXObject();
                     return;
                   }
@@ -2266,7 +2288,7 @@ var PartialEvaluator = (function PartialEvaluatorClosure() {
                     })
                     .then(function () {
                       if (!sinkWrapper.enqueueInvoked) {
-                        skipEmptyXObjs[name] = true;
+                        emptyXObjectCache.set(name, xobj.dict.objId, true);
                       }
                       resolveXObject();
                     }, rejectXObject);
diff --git a/src/core/image_utils.js b/src/core/image_utils.js
index 0cbdc2c61..ea1090dff 100644
--- a/src/core/image_utils.js
+++ b/src/core/image_utils.js
@@ -17,6 +17,45 @@
 import { assert, info, shadow } from "../shared/util.js";
 import { RefSetCache } from "./primitives.js";
 
+class LocalImageCache {
+  constructor() {
+    this._nameRefMap = new Map();
+    this._imageMap = new Map();
+    this._imageCache = new RefSetCache();
+  }
+
+  getByName(name) {
+    const ref = this._nameRefMap.get(name);
+    if (ref) {
+      return this.getByRef(ref);
+    }
+    return this._imageMap.get(name) || null;
+  }
+
+  getByRef(ref) {
+    return this._imageCache.get(ref) || null;
+  }
+
+  set(name, ref = null, data) {
+    if (!name) {
+      throw new Error('LocalImageCache.set - expected "name" argument.');
+    }
+    if (ref) {
+      if (this._imageCache.has(ref)) {
+        return;
+      }
+      this._nameRefMap.set(name, ref);
+      this._imageCache.put(ref, data);
+      return;
+    }
+    // name
+    if (this._imageMap.has(name)) {
+      return;
+    }
+    this._imageMap.set(name, data);
+  }
+}
+
 class GlobalImageCache {
   static get NUM_PAGES_THRESHOLD() {
     return shadow(this, "NUM_PAGES_THRESHOLD", 2);
@@ -111,4 +150,4 @@ class GlobalImageCache {
   }
 }
 
-export { GlobalImageCache };
+export { LocalImageCache, GlobalImageCache };