diff --git a/src/core/document.js b/src/core/document.js
index f2d80c07b..1c7772936 100644
--- a/src/core/document.js
+++ b/src/core/document.js
@@ -844,8 +844,10 @@ class PDFDocument {
     return this.catalog.fontFallback(id, handler);
   }
 
-  async cleanup() {
-    return this.catalog ? this.catalog.cleanup() : clearPrimitiveCaches();
+  async cleanup(manuallyTriggered = false) {
+    return this.catalog
+      ? this.catalog.cleanup(manuallyTriggered)
+      : clearPrimitiveCaches();
   }
 }
 
diff --git a/src/core/image_utils.js b/src/core/image_utils.js
index 1a6a8bd87..1ae7bf6cc 100644
--- a/src/core/image_utils.js
+++ b/src/core/image_utils.js
@@ -199,8 +199,10 @@ class GlobalImageCache {
     this._imageCache.put(ref, data);
   }
 
-  clear() {
-    this._refCache.clear();
+  clear(onlyData = false) {
+    if (!onlyData) {
+      this._refCache.clear();
+    }
     this._imageCache.clear();
   }
 }
diff --git a/src/core/obj.js b/src/core/obj.js
index 939377633..fdaa1bd17 100644
--- a/src/core/obj.js
+++ b/src/core/obj.js
@@ -716,9 +716,9 @@ class Catalog {
     });
   }
 
-  cleanup() {
+  cleanup(manuallyTriggered = false) {
     clearPrimitiveCaches();
-    this.globalImageCache.clear();
+    this.globalImageCache.clear(/* onlyData = */ manuallyTriggered);
     this.pageKidsCountCache.clear();
 
     const promises = [];
diff --git a/src/core/pdf_manager.js b/src/core/pdf_manager.js
index 0ae629c63..1cdcc03fa 100644
--- a/src/core/pdf_manager.js
+++ b/src/core/pdf_manager.js
@@ -76,8 +76,8 @@ class BasePdfManager {
     return this.pdfDocument.fontFallback(id, handler);
   }
 
-  cleanup() {
-    return this.pdfDocument.cleanup();
+  cleanup(manuallyTriggered = false) {
+    return this.pdfDocument.cleanup(manuallyTriggered);
   }
 
   async ensure(obj, prop, args) {
diff --git a/src/core/worker.js b/src/core/worker.js
index 91a70db63..e03c98da2 100644
--- a/src/core/worker.js
+++ b/src/core/worker.js
@@ -625,7 +625,7 @@ var WorkerMessageHandler = {
     });
 
     handler.on("Cleanup", function wphCleanup(data) {
-      return pdfManager.cleanup();
+      return pdfManager.cleanup(/* manuallyTriggered = */ true);
     });
 
     handler.on("Terminate", function wphTerminate(data) {