Fail when MD5 of test files fails on bots.

gulpfile.js

@@ -356,7 +356,7 @@ function getTempFile(prefix, suffix) {
   return path;
 }
 
-function createTestSource(testsName) {
+function createTestSource(testsName, bot) {
   var source = stream.Readable({ objectMode: true, });
   source._read = function () {
     console.log();
@@ -394,6 +394,9 @@ function createTestSource(testsName) {
         return null;
     }
     args.push('--browserManifestFile=' + PDF_BROWSERS);
+    if (bot) {
+      args.push('--strictVerify');
+    }
 
     var testProcess = spawn('node', args, { cwd: TEST_DIR, stdio: 'inherit', });
     testProcess.on('close', function (code) {
@@ -403,7 +406,7 @@ function createTestSource(testsName) {
   return source;
 }
 
-function makeRef(done, noPrompts) {
+function makeRef(done, bot) {
   console.log();
   console.log('### Creating reference images');
 
@@ -420,8 +423,8 @@ function makeRef(done, noPrompts) {
   }
 
   var args = ['test.js', '--masterMode'];
-  if (noPrompts) {
-    args.push('--noPrompts');
+  if (bot) {
+    args.push('--noPrompts', '--strictVerify');
   }
   args.push('--browserManifestFile=' + PDF_BROWSERS);
   var testProcess = spawn('node', args, { cwd: TEST_DIR, stdio: 'inherit', });
@@ -980,8 +983,8 @@ gulp.task('test', ['testing-pre', 'generic', 'components'], function() {
 
 gulp.task('bottest', ['testing-pre', 'generic', 'components'], function() {
   return streamqueue({ objectMode: true, },
-    createTestSource('unit'), createTestSource('font'),
-    createTestSource('browser (no reftest)'));
+    createTestSource('unit', true), createTestSource('font', true),
+    createTestSource('browser (no reftest)', true));
 });
 
 gulp.task('browsertest', ['testing-pre', 'generic', 'components'], function() {

test/test.js

@@ -36,7 +36,7 @@ function parseOptions() {
   var yargs = require('yargs')
     .usage('Usage: $0')
     .boolean(['help', 'masterMode', 'reftest', 'unitTest', 'fontTest',
-              'noPrompts', 'noDownload', 'downloadOnly'])
+              'noPrompts', 'noDownload', 'downloadOnly', 'strictVerify'])
     .string(['manifestFile', 'browser', 'browserManifestFile',
              'port', 'statsFile', 'statsDelay', 'testfilter'])
     .alias('browser', 'b').alias('help', 'h').alias('masterMode', 'm')
@@ -63,6 +63,7 @@ function parseOptions() {
     .describe('fontTest', 'Run the font tests.')
     .describe('noDownload', 'Skips test PDFs downloading.')
     .describe('downloadOnly', 'Download test PDFs without running the tests.')
+    .describe('strictVerify', 'Error if verifying the manifest files fails.')
     .describe('statsFile', 'The file where to store stats.')
     .describe('statsDelay', 'The amount of time in milliseconds the browser ' +
       'should wait before starting stats.')
@@ -716,6 +717,9 @@ function ensurePDFsDownloaded(callback) {
                     'used for testing.');
         console.log('Please re-download the files, or adjust the MD5 ' +
                     'checksum in the manifest for the files listed above.\n');
+        if (options.strictVerify) {
+          process.exit(1);
+        }
       }
       callback();
     });