Sakurai/pdf.js
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Implement streaming using moz-chunk-arraybuffer

Browse Source
This commit is contained in:
Yury Delendik 2014-09-05 20:02:54 -05:00
parent 477efd52bc
commit c3f191a27c
10 changed files with 274 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
extensions/chromium/preferences_schema.json
View File

@ -44,6 +44,12 @@
"type": "boolean",
"default": false
},
"disableStream": {
"title": "Disable streaming for requests",
"description": "Whether to disable streaming for requests (not recommended).",
"type": "boolean",
"default": false
},
"disableAutoFetch": {
"type": "boolean",
"default": false

59
extensions/firefox/content/PdfStreamConverter.jsm
View File

@ -178,6 +178,7 @@ function makeContentReadable(obj, window) {
function PdfDataListener(length) {
this.length = length; // less than 0, if length is unknown
this.data = new Uint8Array(length >= 0 ? length : 0x10000);
this.position = 0;
this.loaded = 0;
}
@ -200,6 +201,11 @@ PdfDataListener.prototype = {
this.loaded = willBeLoaded;
this.onprogress(this.loaded, this.length >= 0 ? this.length : void(0));
},
readData: function PdfDataListener_readData() {
var data = this.data.subarray(this.position, this.loaded);
this.position = this.loaded;
return data;
},
getData: function PdfDataListener_getData() {
var data = this.data;
if (this.loaded != data.length)
@ -523,11 +529,13 @@ var RangedChromeActions = (function RangedChromeActionsClosure() {
*/
function RangedChromeActions(
domWindow, contentDispositionFilename, originalRequest,
dataListener) {
rangeEnabled, streamingEnabled, dataListener) {
ChromeActions.call(this, domWindow, contentDispositionFilename);
this.dataListener = dataListener;
this.originalRequest = originalRequest;
this.rangeEnabled = rangeEnabled;
this.streamingEnabled = streamingEnabled;
this.pdfUrl = originalRequest.URI.spec;
this.contentLength = originalRequest.contentLength;
@ -585,20 +593,46 @@ var RangedChromeActions = (function RangedChromeActionsClosure() {
proto.constructor = RangedChromeActions;
proto.initPassiveLoading = function RangedChromeActions_initPassiveLoading() {
this.originalRequest.cancel(Cr.NS_BINDING_ABORTED);
this.originalRequest = null;
var self = this;
var data;
if (!this.streamingEnabled) {
this.originalRequest.cancel(Cr.NS_BINDING_ABORTED);
this.originalRequest = null;
data = this.dataListener.getData();
this.dataListener = null;
} else {
data = this.dataListener.readData();
this.dataListener.onprogress = function (loaded, total) {
self.domWindow.postMessage({
pdfjsLoadAction: 'progressiveRead',
loaded: loaded,
total: total,
chunk: self.dataListener.readData()
}, '*');
};
this.dataListener.oncomplete = function () {
delete self.dataListener;
};
}
this.domWindow.postMessage({
pdfjsLoadAction: 'supportsRangedLoading',
rangeEnabled: this.rangeEnabled,
streamingEnabled: this.streamingEnabled,
pdfUrl: this.pdfUrl,
length: this.contentLength,
data: this.dataListener.getData()
data: data
}, '*');
this.dataListener = null;
return true;
};
proto.requestDataRange = function RangedChromeActions_requestDataRange(args) {
if (!this.rangeEnabled) {
return;
}
var begin = args.begin;
var end = args.end;
var domWindow = this.domWindow;
@ -840,7 +874,8 @@ PdfStreamConverter.prototype = {
} catch (e) {}
var rangeRequest = false;
if (isHttpRequest) {
var hash = aRequest.URI.ref;
if (isHttpRequest && !getBoolPref(PREF_PREFIX + '.disableRange', false)) {
var contentEncoding = 'identity';
try {
contentEncoding = aRequest.getResponseHeader('Content-Encoding');
@ -851,12 +886,13 @@ PdfStreamConverter.prototype = {
acceptRanges = aRequest.getResponseHeader('Accept-Ranges');
} catch (e) {}
var hash = aRequest.URI.ref;
rangeRequest = contentEncoding === 'identity' &&
acceptRanges === 'bytes' &&
aRequest.contentLength >= 0 &&
hash.indexOf('disableRange=true') < 0;
hash.toLowerCase().indexOf('disablerange=true') < 0;
}
var streamRequest = !getBoolPref(PREF_PREFIX + '.disableStream', false) &&
hash.toLowerCase().indexOf('disablestream=true') < 0;
aRequest.QueryInterface(Ci.nsIChannel);
@ -914,12 +950,13 @@ PdfStreamConverter.prototype = {
// may have changed during a redirect.
var domWindow = getDOMWindow(channel);
var actions;
if (rangeRequest) {
if (rangeRequest || streamRequest) {
actions = new RangedChromeActions(
domWindow, contentDispositionFilename, aRequest, dataListener);
domWindow, contentDispositionFilename, aRequest,
rangeRequest, streamRequest, dataListener);
} else {
actions = new StandardChromeActions(
domWindow, contentDispositionFilename, dataListener);
domWindow, contentDispositionFilename, dataListener);
}
var requestListener = new RequestListener(actions);
domWindow.addEventListener(PDFJS_EVENT_ID, function(event) {

60
src/core/chunked_stream.js
View File

@ -30,7 +30,7 @@ var ChunkedStream = (function ChunkedStreamClosure() {
this.numChunksLoaded = 0;
this.numChunks = Math.ceil(length / chunkSize);
this.manager = manager;
this.initialDataLength = 0;
this.progressiveDataLength = 0;
this.lastSuccessfulEnsureByteChunk = -1; // a single-entry cache
}
@ -80,14 +80,22 @@ var ChunkedStream = (function ChunkedStreamClosure() {
}
},
onReceiveInitialData: function ChunkedStream_onReceiveInitialData(data) {
this.bytes.set(data);
this.initialDataLength = data.length;
var endChunk = (this.end === data.length ?
this.numChunks : Math.floor(data.length / this.chunkSize));
for (var i = 0; i < endChunk; i++) {
this.loadedChunks[i] = true;
++this.numChunksLoaded;
onReceiveProgressiveData:
function ChunkedStream_onReceiveProgressiveData(data) {
var position = this.progressiveDataLength;
var beginChunk = Math.floor(position / this.chunkSize);
this.bytes.set(new Uint8Array(data), position);
position += data.byteLength;
this.progressiveDataLength = position;
var endChunk = position >= this.end ? this.numChunks :
Math.floor(position / this.chunkSize);
var curChunk;
for (curChunk = beginChunk; curChunk < endChunk; ++curChunk) {
if (!(curChunk in this.loadedChunks)) {
this.loadedChunks[curChunk] = true;
++this.numChunksLoaded;
}
}
},
@ -108,7 +116,7 @@ var ChunkedStream = (function ChunkedStreamClosure() {
return;
}
if (end <= this.initialDataLength) {
if (end <= this.progressiveDataLength) {
return;
}
@ -300,28 +308,16 @@ var ChunkedStreamManager = (function ChunkedStreamManagerClosure() {
this.chunksNeededByRequest = {};
this.requestsByChunk = {};
this.callbacksByRequest = {};
this.progressiveDataLength = 0;
this._loadedStreamCapability = createPromiseCapability();
if (args.initialData) {
this.setInitialData(args.initialData);
this.onReceiveData({chunk: args.initialData});
}
}
ChunkedStreamManager.prototype = {
setInitialData: function ChunkedStreamManager_setInitialData(data) {
this.stream.onReceiveInitialData(data);
if (this.stream.allChunksLoaded()) {
this._loadedStreamCapability.resolve(this.stream);
} else if (this.msgHandler) {
this.msgHandler.send('DocProgress', {
loaded: data.length,
total: this.length
});
}
},
onLoadedStream: function ChunkedStreamManager_getLoadedStream() {
return this._loadedStreamCapability.promise;
},
@ -459,13 +455,21 @@ var ChunkedStreamManager = (function ChunkedStreamManagerClosure() {
onReceiveData: function ChunkedStreamManager_onReceiveData(args) {
var chunk = args.chunk;
var begin = args.begin;
var isProgressive = args.begin === undefined;
var begin = isProgressive ? this.progressiveDataLength : args.begin;
var end = begin + chunk.byteLength;
var beginChunk = this.getBeginChunk(begin);
var endChunk = this.getEndChunk(end);
var beginChunk = Math.floor(begin / this.chunkSize);
var endChunk = end < this.length ? Math.floor(end / this.chunkSize) :
Math.ceil(end / this.chunkSize);
if (isProgressive) {
this.stream.onReceiveProgressiveData(chunk);
this.progressiveDataLength = end;
} else {
this.stream.onReceiveData(begin, chunk);
}
this.stream.onReceiveData(begin, chunk);
if (this.stream.allChunksLoaded()) {
this._loadedStreamCapability.resolve(this.stream);
}

51
src/core/network.js
View File

@ -68,11 +68,11 @@ var NetworkManager = (function NetworkManagerClosure() {
return data;
}
var length = data.length;
var buffer = new Uint8Array(length);
var array = new Uint8Array(length);
for (var i = 0; i < length; i++) {
buffer[i] = data.charCodeAt(i) & 0xFF;
array[i] = data.charCodeAt(i) & 0xFF;
}
return buffer;
return array.buffer;
}
NetworkManager.prototype = {
@ -87,11 +87,11 @@ var NetworkManager = (function NetworkManagerClosure() {
return this.request(args);
},
requestFull: function NetworkManager_requestRange(listeners) {
requestFull: function NetworkManager_requestFull(listeners) {
return this.request(listeners);
},
request: function NetworkManager_requestRange(args) {
request: function NetworkManager_request(args) {
var xhr = this.getXhr();
var xhrId = this.currXhrId++;
var pendingRequest = this.pendingRequests[xhrId] = {
@ -115,27 +115,54 @@ var NetworkManager = (function NetworkManagerClosure() {
pendingRequest.expectedStatus = 200;
}
xhr.responseType = 'arraybuffer';
if (args.onProgress) {
xhr.onprogress = args.onProgress;
if (args.onProgressiveData) {
xhr.responseType = 'moz-chunked-arraybuffer';
if (xhr.responseType === 'moz-chunked-arraybuffer') {
pendingRequest.onProgressiveData = args.onProgressiveData;
pendingRequest.mozChunked = true;
} else {
xhr.responseType = 'arraybuffer';
}
} else {
xhr.responseType = 'arraybuffer';
}
if (args.onError) {
xhr.onerror = function(evt) {
args.onError(xhr.status);
};
}
xhr.onreadystatechange = this.onStateChange.bind(this, xhrId);
xhr.onprogress = this.onProgress.bind(this, xhrId);
pendingRequest.onHeadersReceived = args.onHeadersReceived;
pendingRequest.onDone = args.onDone;
pendingRequest.onError = args.onError;
pendingRequest.onProgress = args.onProgress;
xhr.send(null);
return xhrId;
},
onProgress: function NetworkManager_onProgress(xhrId, evt) {
var pendingRequest = this.pendingRequests[xhrId];
if (!pendingRequest) {
// Maybe abortRequest was called...
return;
}
if (pendingRequest.mozChunked) {
var chunk = getArrayBuffer(pendingRequest.xhr);
pendingRequest.onProgressiveData(chunk);
}
var onProgress = pendingRequest.onProgress;
if (onProgress) {
onProgress(evt);
}
},
onStateChange: function NetworkManager_onStateChange(xhrId, evt) {
var pendingRequest = this.pendingRequests[xhrId];
if (!pendingRequest) {
@ -196,6 +223,8 @@ var NetworkManager = (function NetworkManagerClosure() {
begin: begin,
chunk: chunk
});
} else if (pendingRequest.onProgressiveData) {
pendingRequest.onDone(null);
} else {
pendingRequest.onDone({
begin: 0,
@ -215,6 +244,10 @@ var NetworkManager = (function NetworkManagerClosure() {
return this.pendingRequests[xhrId].xhr;
},
isStreamingRequest: function NetworkManager_isStreamingRequest(xhrId) {
return !!(this.pendingRequests[xhrId].onProgressiveData);
},
isPendingRequest: function NetworkManager_isPendingRequest(xhrId) {
return xhrId in this.pendingRequests;
},

9
src/core/pdf_manager.js
View File

@ -65,6 +65,10 @@ var BasePdfManager = (function BasePdfManagerClosure() {
return new NotImplementedException();
},
sendProgressiveData: function BasePdfManager_sendProgressiveData(chunk) {
return new NotImplementedException();
},
updatePassword: function BasePdfManager_updatePassword(password) {
this.pdfDocument.xref.password = this.password = password;
if (this._passwordChangedCapability) {
@ -201,6 +205,11 @@ var NetworkPdfManager = (function NetworkPdfManagerClosure() {
this.streamManager.requestAllChunks();
};
NetworkPdfManager.prototype.sendProgressiveData =
function NetworkPdfManager_sendProgressiveData(chunk) {
this.streamManager.onReceiveData({ chunk: chunk });
};
NetworkPdfManager.prototype.onLoadedStream =
function NetworkPdfManager_getLoadedStream() {
return this.streamManager.onLoadedStream();

57
src/core/worker.js
View File

@ -16,7 +16,7 @@
*/
/* globals PDFJS, createPromiseCapability, LocalPdfManager, NetworkPdfManager,
NetworkManager, isInt, RANGE_CHUNK_SIZE, MissingPDFException,
UnexpectedResponseException, PasswordException, Promise,
UnexpectedResponseException, PasswordException, Promise, warn,
PasswordResponses, InvalidPDFException, UnknownErrorException,
XRefParseException, Ref, info, globalScope, error, MessageHandler */
@ -86,6 +86,7 @@ var WorkerMessageHandler = PDFJS.WorkerMessageHandler = {
httpHeaders: source.httpHeaders,
withCredentials: source.withCredentials
});
var cachedChunks = [];
var fullRequestXhrId = networkManager.requestFull({
onHeadersReceived: function onHeadersReceived() {
if (disableRange) {
@ -116,11 +117,18 @@ var WorkerMessageHandler = PDFJS.WorkerMessageHandler = {
return;
}
// NOTE: by cancelling the full request, and then issuing range
// requests, there will be an issue for sites where you can only
// request the pdf once. However, if this is the case, then the
// server should not be returning that it can support range requests.
networkManager.abortRequest(fullRequestXhrId);
if (networkManager.isStreamingRequest(fullRequestXhrId)) {
// We can continue fetching when progressive loading is enabled,
// and we don't need the autoFetch feature.
source.disableAutoFetch = true;
} else {
// NOTE: by cancelling the full request, and then issuing range
// requests, there will be an issue for sites where you can only
// request the pdf once. However, if this is the case, then the
// server should not be returning that it can support range
// requests.
networkManager.abortRequest(fullRequestXhrId);
}
try {
pdfManager = new NetworkPdfManager(source, handler);
@ -130,10 +138,44 @@ var WorkerMessageHandler = PDFJS.WorkerMessageHandler = {
}
},
onProgressiveData: PDFJS.disableStream ? null :
function onProgressiveData(chunk) {
if (!pdfManager) {
cachedChunks.push(chunk);
return;
}
pdfManager.sendProgressiveData(chunk);
},
onDone: function onDone(args) {
if (pdfManager) {
return; // already processed
}
var pdfFile;
if (args === null) {
// TODO add some streaming manager, e.g. for unknown length files.
// The data was returned in the onProgressiveData, combining...
var pdfFileLength = 0, pos = 0;
cachedChunks.forEach(function (chunk) {
pdfFileLength += chunk.byteLength;
});
if (source.length && pdfFileLength !== source.length) {
warn('reported HTTP length is different from actual');
}
var pdfFileArray = new Uint8Array(pdfFileLength);
cachedChunks.forEach(function (chunk) {
pdfFileArray.set(new Uint8Array(chunk), pos);
pos += chunk.byteLength;
});
pdfFile = pdfFileArray.buffer;
} else {
pdfFile = args.chunk;
}
// the data is array, instantiating directly from it
try {
pdfManager = new LocalPdfManager(args.chunk, source.password);
pdfManager = new LocalPdfManager(pdfFile, source.password);
pdfManagerCapability.resolve();
} catch (ex) {
pdfManagerCapability.reject(ex);
@ -228,6 +270,7 @@ var WorkerMessageHandler = PDFJS.WorkerMessageHandler = {
PDFJS.cMapPacked = data.cMapPacked === true;
getPdfManager(data).then(function () {
handler.send('PDFManagerReady', null);
pdfManager.onLoadedStream().then(function(stream) {
handler.send('DataLoaded', { length: stream.bytes.byteLength });
});

20
src/display/api.js
View File

@ -86,6 +86,14 @@ PDFJS.workerSrc = (PDFJS.workerSrc === undefined ? null : PDFJS.workerSrc);
PDFJS.disableRange = (PDFJS.disableRange === undefined ?
false : PDFJS.disableRange);
/**
* Disable streaming of PDF file data. By default PDF.js attempts to load PDF
* in chunks. This default behavior can be disabled.
* @var {boolean}
*/
PDFJS.disableStream = (PDFJS.disableStream === undefined ?
false : PDFJS.disableStream);
/**
* Disable pre-fetching of PDF file data. When range requests are enabled PDF.js
* will automatically keep fetching more data even if it isn't needed to display
@ -851,6 +859,12 @@ var WorkerTransport = (function WorkerTransportClosure() {
});
});
pdfDataRangeTransport.addProgressiveReadListener(function(chunk) {
messageHandler.send('OnDataRange', {
chunk: chunk
});
});
messageHandler.on('RequestDataRange',
function transportDataRange(data) {
pdfDataRangeTransport.requestDataRange(data.begin, data.end);
@ -911,6 +925,12 @@ var WorkerTransport = (function WorkerTransportClosure() {
this.downloadInfoCapability.resolve(data);
}, this);
messageHandler.on('PDFManagerReady', function transportPage(data) {
if (this.pdfDataRangeTransport) {
this.pdfDataRangeTransport.transportReady();
}
}, this);
messageHandler.on('StartRenderPage', function transportRender(data) {
var page = this.pageCache[data.pageIndex];

47
web/compatibility.js
View File

@ -167,6 +167,21 @@ if (typeof PDFJS === 'undefined') {
// The worker will be using XHR, so we can save time and disable worker.
PDFJS.disableWorker = true;
Object.defineProperty(xhrPrototype, 'responseType', {
get: function xmlHttpRequestGetResponseType() {
return this._responseType || 'text';
},
set: function xmlHttpRequestSetResponseType(value) {
if (value === 'text' || value === 'arraybuffer') {
this._responseType = value;
if (value === 'arraybuffer' &&
typeof this.overrideMimeType === 'function') {
this.overrideMimeType('text/plain; charset=x-user-defined');
}
}
}
});
// Support: IE9
if (typeof VBArray !== 'undefined') {
Object.defineProperty(xhrPrototype, 'response', {
@ -181,25 +196,20 @@ if (typeof PDFJS === 'undefined') {
return;
}
// other browsers
function responseTypeSetter() {
// will be only called to set "arraybuffer"
this.overrideMimeType('text/plain; charset=x-user-defined');
}
if (typeof xhr.overrideMimeType === 'function') {
Object.defineProperty(xhrPrototype, 'responseType',
{ set: responseTypeSetter });
}
function responseGetter() {
var text = this.responseText;
var i, n = text.length;
var result = new Uint8Array(n);
for (i = 0; i < n; ++i) {
result[i] = text.charCodeAt(i) & 0xFF;
Object.defineProperty(xhrPrototype, 'response', {
get: function xmlHttpRequestResponseGet() {
if (this.responseType !== 'arraybuffer') {
return this.responseText;
}
var text = this.responseText;
var i, n = text.length;
var result = new Uint8Array(n);
for (i = 0; i < n; ++i) {
result[i] = text.charCodeAt(i) & 0xFF;
}
return result.buffer;
}
return result.buffer;
}
Object.defineProperty(xhrPrototype, 'response', { get: responseGetter });
});
})();
// window.btoa (base64 encode function) ?
@ -471,6 +481,7 @@ if (typeof PDFJS === 'undefined') {
if (isSafari || isOldAndroid) {
PDFJS.disableRange = true;
PDFJS.disableStream = true;
}
})();

1
web/default_preferences.js
View File

@ -28,6 +28,7 @@ var DEFAULT_PREFERENCES = {
enableWebGL: false,
pdfBugEnabled: false,
disableRange: false,
disableStream: false,
disableAutoFetch: false,
disableFontFace: false,
//#if B2G

41
web/viewer.js
View File

@ -516,9 +516,15 @@ var PDFView = {
//#if (FIREFOX || MOZCENTRAL)
initPassiveLoading: function pdfViewInitPassiveLoading() {
var pdfDataRangeTransportReadyResolve;
var pdfDataRangeTransportReady = new Promise(function (resolve) {
pdfDataRangeTransportReadyResolve = resolve;
});
var pdfDataRangeTransport = {
rangeListeners: [],
progressListeners: [],
progressiveReadListeners: [],
ready: pdfDataRangeTransportReady,
addRangeListener: function PdfDataRangeTransport_addRangeListener(
listener) {
@ -530,6 +536,11 @@ var PDFView = {
this.progressListeners.push(listener);
},
addProgressiveReadListener:
function PdfDataRangeTransport_addProgressiveReadListener(listener) {
this.progressiveReadListeners.push(listener);
},
onDataRange: function PdfDataRangeTransport_onDataRange(begin, chunk) {
var listeners = this.rangeListeners;
for (var i = 0, n = listeners.length; i < n; ++i) {
@ -538,10 +549,26 @@ var PDFView = {
},
onDataProgress: function PdfDataRangeTransport_onDataProgress(loaded) {
var listeners = this.progressListeners;
for (var i = 0, n = listeners.length; i < n; ++i) {
listeners[i](loaded);
}
this.ready.then(function () {
var listeners = this.progressListeners;
for (var i = 0, n = listeners.length; i < n; ++i) {
listeners[i](loaded);
}
}.bind(this));
},
onDataProgressiveRead:
function PdfDataRangeTransport_onDataProgress(chunk) {
this.ready.then(function () {
var listeners = this.progressiveReadListeners;
for (var i = 0, n = listeners.length; i < n; ++i) {
listeners[i](chunk);
}
}.bind(this));
},
transportReady: function PdfDataRangeTransport_transportReady() {
pdfDataRangeTransportReadyResolve();
},
requestDataRange: function PdfDataRangeTransport_requestDataRange(
@ -574,6 +601,9 @@ var PDFView = {
case 'rangeProgress':
pdfDataRangeTransport.onDataProgress(args.loaded);
break;
case 'progressiveRead':
pdfDataRangeTransport.onDataProgressiveRead(args.chunk);
break;
case 'progress':
PDFView.progress(args.loaded / args.total);
break;
@ -1787,6 +1817,9 @@ function webViewerInitialized() {
if ('disablerange' in hashParams) {
PDFJS.disableRange = (hashParams['disablerange'] === 'true');
}
if ('disablestream' in hashParams) {
PDFJS.disableStream = (hashParams['disablestream'] === 'true');
}
if ('disableautofetch' in hashParams) {
PDFJS.disableAutoFetch = (hashParams['disableautofetch'] === 'true');
}