Merge pull request #6196 from Rob--W/run-specific-test

Add --testfilter and -t flags

test/driver.js

@@ -132,6 +132,8 @@ var Driver = (function DriverClosure() {
     this.appPath = parameters.path;
     this.delay = (parameters.delay | 0) || 0;
     this.inFlightRequests = 0;
+    this.testFilter = parameters.testFilter ?
+      JSON.parse(parameters.testFilter) : [];
 
     // Create a working canvas
     this.canvas = document.createElement('canvas');
@@ -163,6 +165,11 @@ var Driver = (function DriverClosure() {
         if (r.readyState === 4) {
           self._log('done\n');
           self.manifest = JSON.parse(r.responseText);
+          if (self.testFilter && self.testFilter.length) {
+            self.manifest = self.manifest.filter(function(item) {
+              return self.testFilter.indexOf(item.id) !== -1;
+            });
+          }
           self.currentTask = 0;
           self._nextTask();
         }
@@ -422,9 +429,9 @@ var Driver = (function DriverClosure() {
     _done: function Driver_done() {
       if (this.inFlightRequests > 0) {
         this.inflight.textContent = this.inFlightRequests;
-        setTimeout(this._done(), WAITING_TIME);
+        setTimeout(this._done.bind(this), WAITING_TIME);
       } else {
-        setTimeout(this._quit(), WAITING_TIME);
+        setTimeout(this._quit.bind(this), WAITING_TIME);
       }
     },
 

test/test.js

@@ -40,8 +40,9 @@ function parseOptions() {
     .boolean(['help', 'masterMode', 'reftest', 'unitTest', 'fontTest',
               'noPrompts', 'noDownload', 'downloadOnly'])
     .string(['manifestFile', 'browser', 'browserManifestFile',
-             'port', 'statsFile', 'statsDelay'])
+             'port', 'statsFile', 'statsDelay', 'testfilter'])
     .alias('browser', 'b').alias('help', 'h').alias('masterMode', 'm')
+    .alias('testfilter', 't')
     .describe('help', 'Show this help message')
     .describe('masterMode', 'Run the script in master mode.')
     .describe('noPrompts',
@@ -54,6 +55,10 @@ function parseOptions() {
       'those found in resources/browser_manifests/')
     .describe('reftest', 'Automatically start reftest showing comparison ' +
       'test failures, if there are any.')
+    .describe('testfilter', 'Run specific reftest(s).')
+    .default('testfilter', [])
+    .example('$0 --b=firefox -t=issue5567 -t=issue5909',
+      'Run the reftest identified by issue5567 and issue5909 in Firefox.')
     .describe('port', 'The port the HTTP server should listen on.')
     .default('port', 8000)
     .describe('unitTest', 'Run the unit tests.')
@@ -85,6 +90,8 @@ function parseOptions() {
     yargs.showHelp();
     process.exit(0);
   }
+  result.testfilter = Array.isArray(result.testfilter) ?
+    result.testfilter : [result.testfilter];
   return result;
 }
 
@@ -254,7 +261,10 @@ function startRefTest(masterMode, showRefImages) {
   }
 
   var startTime;
-  var manifest = JSON.parse(fs.readFileSync(options.manifestFile));
+  var manifest = getTestManifest();
+  if (!manifest) {
+    return;
+  }
   if (options.noDownload) {
     checkRefsTmp();
   } else {
@@ -274,6 +284,26 @@ function handleSessionTimeout(session) {
   closeSession(browser);
 }
 
+function getTestManifest() {
+  var manifest = JSON.parse(fs.readFileSync(options.manifestFile));
+
+  var testFilter = options.testfilter.slice(0);
+  if (testFilter.length) {
+    manifest = manifest.filter(function(item) {
+      var i = testFilter.indexOf(item.id);
+      if (i !== -1) {
+        testFilter.splice(i, 1);
+        return true;
+      }
+    });
+    if (testFilter.length) {
+      console.error('Unrecognized test IDs: ' + testFilter.join(' '));
+      return;
+    }
+  }
+  return manifest;
+}
+
 function checkEq(task, results, browser, masterMode) {
   var taskId = task.id;
   var refSnapshotDir = path.join(refsDir, os.platform(), browser, taskId);
@@ -616,6 +646,7 @@ function startBrowsers(url, initSessionCallback) {
     var startUrl = getServerBaseAddress() + url +
       '?browser=' + encodeURIComponent(b.name) +
       '&manifestFile=' + encodeURIComponent('/test/' + options.manifestFile) +
+      '&testFilter=' + JSON.stringify(options.testfilter) +
       '&path=' + encodeURIComponent(b.path) +
       '&delay=' + options.statsDelay +
       '&masterMode=' + options.masterMode;
@@ -677,7 +708,7 @@ function closeSession(browser) {
 
 function ensurePDFsDownloaded(callback) {
   var downloadUtils = require('./downloadutils.js');
-  var manifest = JSON.parse(fs.readFileSync(options.manifestFile));
+  var manifest = getTestManifest();
   downloadUtils.downloadManifestFiles(manifest, function () {
     downloadUtils.verifyManifestFiles(manifest, function (hasErrors) {
       if (hasErrors) {

test/webbrowser.js

@@ -157,17 +157,18 @@ ChromiumBrowser.prototype.buildArguments = function (url) {
 
 WebBrowser.create = function (desc) {
   var name = desc.name;
-
+  var path = shelljs.which(desc.path);
-  // Throws an exception if the path doesn't exist.
+  if (!path) {
-  fs.statSync(desc.path);
+    throw new Error('Browser executable not found: ' + desc.path);
+  }
 
   if (/firefox/i.test(name)) {
-    return new FirefoxBrowser(desc.name, desc.path);
+    return new FirefoxBrowser(name, path);
   }
   if (/(chrome|chromium|opera)/i.test(name)) {
-    return new ChromiumBrowser(desc.name, desc.path);
+    return new ChromiumBrowser(name, path);
   }
-  return new WebBrowser(desc.name, desc.path);
+  return new WebBrowser(name, path);
 };