diff --git a/test/driver.js b/test/driver.js index e97e7c339..3479608fc 100644 --- a/test/driver.js +++ b/test/driver.js @@ -147,12 +147,16 @@ function nextTask() { }); } -function isLastPage(task) { - var limit = task.pageLimit || 0; - if (!limit || limit > task.pdfDoc.numPages) - limit = task.pdfDoc.numPages; +function getLastPageNum(task) { + var lastPageNum = task.lastPage || 0; + if (!lastPageNum || lastPageNum > task.pdfDoc.numPages) { + lastPageNum = task.pdfDoc.numPages; + } + return lastPageNum; +} - return task.pageNum > limit; +function isLastPage(task) { + return task.pageNum > getLastPageNum(task); } function canvasToDataURL() { @@ -347,7 +351,8 @@ function sendTaskResult(snapshot, task, failure, result) { browser: browser, id: task.id, numPages: task.pdfDoc ? - (task.pageLimit || task.pdfDoc.numPages) : 0, + (task.lastPage || task.pdfDoc.numPages) : 0, + lastPageNum: getLastPageNum(task), failure: failure, file: task.file, round: task.round, diff --git a/test/test.py b/test/test.py index 87ad2c822..9dba9172f 100644 --- a/test/test.py +++ b/test/test.py @@ -333,7 +333,12 @@ class PDFTestHandler(TestHandlerBase): print result['message'] return - id, failure, round, page, snapshot = result['id'], result['failure'], result['round'], result['page'], result['snapshot'] + id = result['id'] + failure = result['failure'] + round = result['round'] + page = result['page'] + snapshot = result['snapshot'] + taskResults = State.taskResults[browser][id] taskResults[round].append(Result(snapshot, failure, page)) if State.saveStats: @@ -346,25 +351,28 @@ class PDFTestHandler(TestHandlerBase): } State.stats.append(stat) - def isTaskDone(): - numPages = result["numPages"] - rounds = State.manifest[id]["rounds"] - for round in range(0,rounds): - if len(taskResults[round]) < numPages: - return False - return True + def isTaskDone(): + last_page_num = result['lastPageNum'] + rounds = State.manifest[id]['rounds'] + for round in range(0,rounds): + if not taskResults[round]: + return False + latest_page = taskResults[round][-1] + if not latest_page.page == last_page_num: + return False + return True - if isTaskDone(): - # sort the results since they sometimes come in out of order - for results in taskResults: - results.sort(key=lambda result: result.page) - check(State.manifest[id], taskResults, browser, - self.server.masterMode) - # Please oh please GC this ... - del State.taskResults[browser][id] - State.remaining[browser] -= 1 + if isTaskDone(): + # sort the results since they sometimes come in out of order + for results in taskResults: + results.sort(key=lambda result: result.page) + check(State.manifest[id], taskResults, browser, + self.server.masterMode) + # Please oh please GC this ... + del State.taskResults[browser][id] + State.remaining[browser] -= 1 - checkIfDone() + checkIfDone() def checkIfDone(): State.done = True @@ -643,12 +651,13 @@ def checkEq(task, results, browser, masterMode): taskType = task['type'] passed = True - for page in xrange(len(results)): - snapshot = results[page].snapshot + for result in results: + page = result.page + snapshot = result.snapshot ref = None eq = True - path = os.path.join(pfx, str(page + 1)) + path = os.path.join(pfx, str(page)) if not os.access(path, os.R_OK): State.numEqNoSnapshot += 1 if not masterMode: @@ -660,7 +669,7 @@ def checkEq(task, results, browser, masterMode): eq = (ref == snapshot) if not eq: - print 'TEST-UNEXPECTED-FAIL |', taskType, taskId, '| in', browser, '| rendering of page', page + 1, '!= reference rendering' + print 'TEST-UNEXPECTED-FAIL |', taskType, taskId, '| in', browser, '| rendering of page', page, '!= reference rendering' if not State.eqLog: State.eqLog = open(EQLOG_FILE, 'w') @@ -669,7 +678,7 @@ def checkEq(task, results, browser, masterMode): # NB: this follows the format of Mozilla reftest # output so that we can reuse its reftest-analyzer # script - eqLog.write('REFTEST TEST-UNEXPECTED-FAIL | ' + browser +'-'+ taskId +'-page'+ str(page + 1) + ' | image comparison (==)\n') + eqLog.write('REFTEST TEST-UNEXPECTED-FAIL | ' + browser +'-'+ taskId +'-page'+ str(page) + ' | image comparison (==)\n') eqLog.write('REFTEST IMAGE 1 (TEST): ' + snapshot + '\n') eqLog.write('REFTEST IMAGE 2 (REFERENCE): ' + ref + '\n') @@ -683,8 +692,8 @@ def checkEq(task, results, browser, masterMode): except OSError, e: if e.errno != 17: # file exists print >>sys.stderr, 'Creating', tmpTaskDir, 'failed!' - - of = open(os.path.join(tmpTaskDir, str(page + 1)), 'w') + + of = open(os.path.join(tmpTaskDir, str(page)), 'w') of.write(snapshot) of.close() diff --git a/test/test_manifest.json b/test/test_manifest.json index 9c065afe7..9467dcb6b 100644 --- a/test/test_manifest.json +++ b/test/test_manifest.json @@ -21,7 +21,7 @@ "file": "pdfs/issue2391-1.pdf", "md5": "25ae9cb959612e7b343b55da63af2716", "rounds": 1, - "pageLimit": 1, + "lastPage": 1, "type": "load" }, { "id": "issue2391-2", @@ -40,7 +40,7 @@ "file": "pdfs/intelisa.pdf", "md5": "cdbcf14d0d260c0b313c566a61b07d9f", "link": true, - "pageLimit": 100, + "lastPage": 100, "rounds": 1, "type": "eq" }, @@ -48,7 +48,7 @@ "file": "pdfs/issue2128.pdf", "md5": "b3f4d9ed3c41e1f1768264a3f8b93aad", "link": true, - "pageLimit": 2, + "lastPage": 2, "rounds": 1, "type": "eq" }, @@ -56,7 +56,7 @@ "file": "pdfs/german-umlaut.pdf", "md5": "331de67c1397702315970a871d8a369b", "link": true, - "pageLimit": 1, + "lastPage": 1, "rounds": 1, "type": "eq" }, @@ -64,7 +64,7 @@ "file": "pdfs/issue1512.pdf", "md5": "41a19fe03d522346ee3baa732403fca4", "link": true, - "pageLimit": 1, + "lastPage": 1, "rounds": 1, "type": "eq" }, @@ -72,7 +72,7 @@ "file": "pdfs/pdf.pdf", "md5": "dbdb23c939d2be09b43126c3c56060c7", "link": true, - "pageLimit": 500, + "lastPage": 500, "rounds": 1, "type": "load" }, @@ -80,7 +80,7 @@ "file": "pdfs/issue2129.pdf", "md5": "b082dd2cb3648f979fd668f498af14d6", "link": true, - "pageLimit": 1, + "lastPage": 1, "rounds": 1, "type": "load" }, @@ -156,7 +156,7 @@ "file": "pdfs/freeculture.pdf", "md5": "dcdf3a8268e6a18938a42d5149efcfca", "rounds": 1, - "pageLimit": 5, + "lastPage": 5, "type": "eq" }, { "id": "wnv_chinese-pdf", @@ -384,7 +384,7 @@ "file": "pdfs/bpl13210.pdf", "md5": "8a08512baa9fa95378d9ad4b995947c7", "link": true, - "pageLimit": 5, + "lastPage": 5, "rounds": 1, "type": "eq" }, @@ -400,7 +400,7 @@ "md5": "ecffc0ce38ffdf1e90dc952f186e9a91", "rounds": 1, "link": true, - "pageLimit": 5, + "lastPage": 5, "skipPages": [1], "type": "eq" }, @@ -409,7 +409,7 @@ "md5": "3a1716a512aca4d7a8d6106bd4885d14", "rounds": 1, "link": true, - "pageLimit": 3, + "lastPage": 3, "type": "eq" }, { "id": "issue1905", @@ -467,7 +467,7 @@ "file": "pdfs/issue1709.pdf", "md5": "84497bd23b7c82d03d2681a1cb1d9ed0", "rounds": 1, - "pageLimit": 10, + "lastPage": 10, "link": true, "type": "eq" }, @@ -482,7 +482,7 @@ "file": "pdfs/issue1096.pdf", "md5": "7f75d2b4b93c78d401ff39e8c1b00612", "rounds": 1, - "pageLimit": 9, + "lastPage": 9, "link": true, "type": "eq" }, @@ -503,7 +503,7 @@ "file": "pdfs/liveprogramming.pdf", "md5": "7bd4dad1188232ef597d36fd72c33e52", "rounds": 1, - "pageLimit": 3, + "lastPage": 3, "link": true, "type": "load" }, @@ -531,7 +531,7 @@ "file": "pdfs/issue1685.pdf", "md5": "b22c3741e6bd0e613d3eb3325ad31f7d", "rounds": 1, - "pageLimit": 2, + "lastPage": 2, "link": true, "type": "eq" }, @@ -553,7 +553,7 @@ "file": "pdfs/html5checker.pdf", "md5": "74bbd80d1e7eb5f2951582233ef9ebab", "rounds": 1, - "pageLimit": 7, + "lastPage": 7, "link": true, "type": "eq" }, @@ -567,7 +567,7 @@ "file": "pdfs/TAMReview.pdf", "md5": "8039aba56790d3597d2bc8c794a51301", "rounds": 1, - "pageLimit": 5, + "lastPage": 5, "link": true, "type": "eq" }, @@ -575,7 +575,7 @@ "file": "pdfs/preistabelle.pdf", "md5": "d2f0b2086160d4f3d325c79a5dc1fb4d", "rounds": 1, - "pageLimit": 2, + "lastPage": 2, "link": true, "type": "eq" }, @@ -604,7 +604,7 @@ "md5": "696ef6de6f4f71643771419ef04fc968", "rounds": 1, "skipPages": [1, 2, 3, 4, 5, 6, 7, 8], - "pageLimit": 9, + "lastPage": 9, "link": true, "type": "eq" }, @@ -627,7 +627,7 @@ "md5": "b71a0f641e83ad427b8bcfc180899a05", "rounds": 1, "skipPages": [1, 2, 3, 4, 5, 6, 7, 8, 9], - "pageLimit": 10, + "lastPage": 10, "link": true, "type": "eq" }, @@ -656,7 +656,7 @@ "file": "pdfs/issue1721.pdf", "md5": "b47177f9e5197a76ec498733ecab60e6", "rounds": 1, - "pageLimit": 2, + "lastPage": 2, "link": true, "type": "eq" }, @@ -664,7 +664,7 @@ "file": "pdfs/issue1233.pdf", "md5": "2d3565b0a286e29955796c37c66326c1", "rounds": 1, - "pageLimit": 1, + "lastPage": 1, "link": true, "type": "eq" }, @@ -672,7 +672,7 @@ "file": "pdfs/issue1243.pdf", "md5": "130c849b83513d5ac5e03c6421fc7489", "rounds": 1, - "pageLimit": 2, + "lastPage": 2, "link": true, "type": "eq" }, @@ -680,7 +680,7 @@ "file": "pdfs/issue1257.pdf", "md5": "9111533826bc21ed774e8e01603a2f54", "rounds": 1, - "pageLimit": 2, + "lastPage": 2, "link": true, "type": "eq" }, @@ -695,7 +695,7 @@ "file": "pdfs/issue1810.pdf", "md5": "b173a9dfb7bf00e1a298c6e8cb95c03e", "rounds": 1, - "pageLimit": 3, + "lastPage": 3, "link": true, "type": "eq" }, @@ -711,7 +711,7 @@ "md5": "b5b6c6405d7b48418bccf97277957664", "rounds": 1, "link": true, - "pageLimit": 1, + "lastPage": 1, "skipPages": [1], "type": "eq" }, @@ -739,7 +739,7 @@ "file": "pdfs/issue1796.pdf", "md5": "9b9b60dc2a4cc3ea05932785d71304fe", "rounds": 1, - "pageLimit": 2, + "lastPage": 2, "link": true, "type": "eq" }, @@ -754,7 +754,7 @@ "file": "pdfs/issue818.pdf", "md5": "dd2f8a5bd65164ad74da2b45a6ca90cc", "rounds": 1, - "pageLimit": 1, + "lastPage": 1, "link": true, "type": "eq" }, @@ -762,7 +762,7 @@ "file": "pdfs/issue2006.pdf", "md5": "71ec73831ece9b508ad20efa6ff28642", "rounds": 1, - "pageLimit": 1, + "lastPage": 1, "link": true, "type": "eq" }, @@ -770,7 +770,7 @@ "file": "pdfs/issue1729.pdf", "md5": "29b0eddc3e1dcb23a44384037032d470", "rounds": 1, - "pageLimit": 1, + "lastPage": 1, "link": true, "type": "load" }, @@ -802,7 +802,7 @@ "file": "pdfs/issue1998.pdf", "md5": "586e0213be2f461360ec26770b5a4e48", "rounds": 1, - "pageLimit": 2, + "lastPage": 2, "link": true, "type": "load" }, @@ -817,7 +817,7 @@ "file": "pdfs/issue2386.pdf", "md5": "7dc787639aa6765214e9ff5494d231ed", "rounds": 1, - "pageLimit": 2, + "lastPage": 2, "link": true, "type": "eq" }, @@ -825,7 +825,7 @@ "file": "pdfs/issue1936.pdf", "md5": "7302eb9b6a626308e2a933aaed9e1756", "rounds": 1, - "pageLimit": 1, + "lastPage": 1, "link": true, "type": "eq" }, @@ -833,7 +833,7 @@ "file": "pdfs/issue2337.pdf", "md5": "ea10f4131202b9b8f2a6cb7770d3f185", "rounds": 1, - "pageLimit": 1, + "lastPage": 1, "link": true, "type": "load" },