test-harness improvements

test.py

@@ -1,7 +1,10 @@
 import json, os, sys, subprocess
 from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
+from urlparse import urlparse
 
 ANAL = True
+DEFAULT_MANIFEST_FILE = 'test_manifest.json'
+REFDIR = 'ref'
 VERBOSE = False
 
 MIMEs = {
@@ -34,8 +37,11 @@ class PDFTestHandler(BaseHTTPRequestHandler):
             BaseHTTPRequestHandler.log_request(code, size)
 
     def do_GET(self):
+        url = urlparse(self.path)
+        # Ignore query string
+        path, _ = url.path, url.query
         cwd = os.getcwd()
-        path = os.path.abspath(os.path.realpath(cwd + os.sep + self.path))
+        path = os.path.abspath(os.path.realpath(cwd + os.sep + path))
         cwd = os.path.abspath(cwd)
         prefix = os.path.commonprefix(( path, cwd ))
         _, ext = os.path.splitext(path)
@@ -69,10 +75,10 @@ class PDFTestHandler(BaseHTTPRequestHandler):
         self.end_headers()
 
         result = json.loads(self.rfile.read(numBytes))
-        browser = 'firefox4'
+        browser, id, failure, round, page, snapshot = result['browser'], result['id'], result['failure'], result['round'], result['page'], result['snapshot']
-        id, failure, round, page, snapshot = result['id'], result['failure'], result['round'], result['page'], result['snapshot']
         taskResults = State.taskResults[browser][id]
-        taskResults[round][page - 1] = Result(snapshot, failure)
+        taskResults[round].append(Result(snapshot, failure))
+        assert len(taskResults[round]) == page
 
         if result['taskDone']:
             check(State.manifest[id], taskResults, browser)
@@ -81,7 +87,7 @@ class PDFTestHandler(BaseHTTPRequestHandler):
         State.done = (0 == State.remaining)
             
 
-def set_up():
+def set_up(manifestFile):
     # Only serve files from a pdf.js clone
     assert not ANAL or os.path.isfile('pdf.js') and os.path.isdir('.git')
 
@@ -90,7 +96,7 @@ def set_up():
 #'chrome12', 'chrome13', 'firefox5', 'firefox6','opera11' ):
                      if os.access(b, os.R_OK | os.X_OK) ]
 
-    mf = open('test_manifest.json')
+    mf = open(manifestFile)
     manifestList = json.load(mf)
     mf.close()
 
@@ -101,15 +107,16 @@ def set_up():
             State.manifest[id] = item
             taskResults = [ ]
             for r in xrange(rounds):
-                taskResults.append([ None ] * 100)
+                taskResults.append([ ])
             State.taskResults[b][id] = taskResults
 
     State.remaining = len(manifestList)
 
     for b in testBrowsers:
         print 'Launching', b
+        qs = 'browser='+ b +'&manifestFile='+ manifestFile
         subprocess.Popen(( os.path.abspath(os.path.realpath(b)),
-                           'http://localhost:8080/test_slave.html' ))
+                           'http://localhost:8080/test_slave.html?'+ qs))
 
 
 def check(task, results, browser):
@@ -129,7 +136,7 @@ def check(task, results, browser):
         return
 
     kind = task['type']
-    if '==' == kind:
+    if 'eq' == kind:
         checkEq(task, results, browser)
     elif 'fbf' == kind:
         checkFBF(task, results, browser)
@@ -140,8 +147,26 @@ def check(task, results, browser):
 
 
 def checkEq(task, results, browser):
-    print '  !!! [TODO: == tests] !!!'
+    pfx = os.path.join(REFDIR, sys.platform, browser, task['id'])
-    print 'TEST-PASS | == test', task['id'], '| in', browser
+    results = results[0]
+
+    passed = True
+    for page in xrange(len(results)):
+        ref = None
+        try:
+            path = os.path.join(pfx, str(page + 1))
+            f = open(path)
+            ref = f.read()
+            f.close()
+        except IOError, ioe:
+            continue
+
+        snapshot = results[page]
+        if ref != snapshot:
+            print 'TEST-UNEXPECTED-FAIL | eq', task['id'], '| in', browser, '| rendering of page', page + 1, '!= reference rendering'
+            passed = False
+    if passed:
+        print 'TEST-PASS | eq test', task['id'], '| in', browser
 
 
 printed = [False]
@@ -150,13 +175,16 @@ def checkFBF(task, results, browser):
     round0, round1 = results[0], results[1]
     assert len(round0) == len(round1)
 
+    passed = True
     for page in xrange(len(round1)):
         r0Page, r1Page = round0[page], round1[page]
         if r0Page is None:
             break
         if r0Page.snapshot != r1Page.snapshot:
             print 'TEST-UNEXPECTED-FAIL | forward-back-forward test', task['id'], '| in', browser, '| first rendering of page', page + 1, '!= second'
-    print 'TEST-PASS | forward-back-forward test', task['id'], '| in', browser
+            passed = False
+    if passed:
+        print 'TEST-PASS | forward-back-forward test', task['id'], '| in', browser
 
 
 def checkLoad(task, results, browser):
@@ -165,11 +193,12 @@ def checkLoad(task, results, browser):
     print 'TEST-PASS | load test', task['id'], '| in', browser
 
 
-def main():
+def main(args):
-    set_up()
+    manifestFile = args[0] if len(args) == 1 else DEFAULT_MANIFEST_FILE
+    set_up(manifestFile)
     server = HTTPServer(('127.0.0.1', 8080), PDFTestHandler)
     while not State.done:
         server.handle_request()
 
 if __name__ == '__main__':
-    main()
+    main(sys.argv[1:])

test_manifest.json

@@ -1,8 +1,8 @@
 [
-    {  "id": "tracemonkey-==",
+    {  "id": "tracemonkey-eq",
        "file": "tests/tracemonkey.pdf",
        "rounds": 1,
-       "type": "=="
+       "type": "eq"
     },
     {  "id": "tracemonkey-fbf",
        "file": "tests/tracemonkey.pdf",

test_slave.html

@@ -5,9 +5,24 @@
   <script type="text/javascript" src="fonts.js"></script>
   <script type="text/javascript" src="glyphlist.js"></script>
   <script type="application/javascript">
-var canvas, currentTask, currentTaskIdx, failure, manifest, pdfDoc, stdout;
+var browser, canvas, currentTask, currentTaskIdx, failure, manifest, pdfDoc, stdout;
+
+function queryParams() {
+    var qs = window.location.search.substring(1);
+    var kvs = qs.split("&");
+    var params = { };
+    for (var i = 0; i < kvs.length; ++i) {
+        var kv = kvs[i].split("=");
+        params[unescape(kv[0])] = unescape(kv[1]);
+    }
+    return params;
+}
 
 function load() {
+  var params = queryParams();
+  browser = params.browser;
+  manifestFile = params.manifestFile;
+
   canvas = document.createElement("canvas");
   // 8.5x11in @ 100% ... XXX need something better here
   canvas.width = 816;
@@ -15,10 +30,11 @@ function load() {
   canvas.mozOpaque = true;
   stdout = document.getElementById("stdout");
 
+  log("Harness thinks this browser is '"+ browser +"'\n");
   log("Fetching manifest ...");
 
   var r = new XMLHttpRequest();
-  r.open("GET", "test_manifest.json", false);
+  r.open("GET", manifestFile, false);
   r.onreadystatechange = function(e) {
     if (r.readyState == 4) {
       log("done\n");
@@ -111,7 +127,8 @@ function done() {
 }
 
 function sendTaskResult(snapshot) {
-  var result = { id: currentTask.id,
+  var result = { browser: browser,
+                 id: currentTask.id,
                  taskDone: currentTask.taskDone,
                  failure: failure,
                  file: currentTask.file,