Merge pull request #6189 from Rob--W/improved-build-tools

Improved build tools (preprocessor & postprocessor)

external/builder/builder.js

@@ -10,16 +10,32 @@ var fs = require('fs'),
     vm = require('vm');
 
 /**
- * A simple preprocessor that is based on the firefox preprocessor
+ * A simple preprocessor that is based on the Firefox preprocessor
- * see (https://developer.mozilla.org/en/Build/Text_Preprocessor).  The main
+ * (https://dxr.mozilla.org/mozilla-central/source/build/docs/preprocessor.rst).
- * difference is that this supports a subset of the commands and it supports
+ * The main difference is that this supports a subset of the commands and it
- * preproccesor commands in html style comments.
+ * supports preprocessor commands in HTML-style comments.
- * Currently Supported commands:
+ *
+ * Currently supported commands:
  * - if
+ * - elif
  * - else
  * - endif
  * - include
  * - expand
+ * - error
+ *
+ * Every #if must be closed with an #endif. Nested conditions are supported.
+ *
+ * Within an #if or #else block, one level of comment tokens is stripped. This
+ * allows us to write code that can run even without preprocessing. For example:
+ *
+ * //#if SOME_RARE_CONDITION
+ * // // Decrement by one
+ * // --i;
+ * //#else
+ * // // Increment by one.
+ * ++i;
+ * //#endif
  */
 function preprocess(inFilename, outFilename, defines) {
   // TODO make this really read line by line.
@@ -37,10 +53,28 @@ function preprocess(inFilename, outFilename, defines) {
     function(line) {
       out += line + '\n';
     });
+  function evaluateCondition(code) {
+    if (!code || !code.trim()) {
+      throw new Error('No JavaScript expression given at ' + loc());
+    }
+    try {
+      return vm.runInNewContext(code, defines, {displayErrors: false});
+    } catch (e) {
+      throw new Error('Could not evaluate "' + code + '" at ' + loc() + '\n' +
+                      e.name + ': ' + e.message);
+    }
+  }
   function include(file) {
     var realPath = fs.realpathSync(inFilename);
     var dir = path.dirname(realPath);
-    preprocess(path.join(dir, file), writeLine, defines);
+    try {
+      preprocess(path.join(dir, file), writeLine, defines);
+    } catch (e) {
+      if (e.code === 'ENOENT') {
+        throw new Error('Failed to include "' + file + '" at ' + loc());
+      }
+      throw e; // Some other error
+    }
   }
   function expand(line) {
     line = line.replace(/__[\w]+__/g, function(variable) {
@@ -53,52 +87,92 @@ function preprocess(inFilename, outFilename, defines) {
     writeLine(line);
   }
 
-  var s, state = 0, stack = [];
+  // not inside if or else (process lines)
+  var STATE_NONE = 0;
+  // inside if, condition false (ignore until #else or #endif)
+  var STATE_IF_FALSE = 1;
+  // inside else, #if was false, so #else is true (process lines until #endif)
+  var STATE_ELSE_TRUE = 2;
+  // inside if, condition true (process lines until #else or #endif)
+  var STATE_IF_TRUE = 3;
+  // inside else, #if was true, so #else is false (ignore lines until #endif)
+  var STATE_ELSE_FALSE = 4;
+
+  var line;
+  var state = STATE_NONE;
+  var stack = [];
   var control =
-    /^(?:\/\/|<!--)\s*#(if|else|endif|expand|include)(?:\s+(.*?)(?:-->)?$)?/;
+  /* jshint -W101 */
+    /^(?:\/\/|<!--)\s*#(if|elif|else|endif|expand|include|error)\b(?:\s+(.*?)(?:-->)?$)?/;
+  /* jshint +W101 */
   var lineNumber = 0;
-  while ((s = readLine()) !== null) {
+  var loc = function() {
+    return fs.realpathSync(inFilename) + ':' + lineNumber;
+  };
+  while ((line = readLine()) !== null) {
     ++lineNumber;
-    var m = control.exec(s);
+    var m = control.exec(line);
     if (m) {
       switch (m[1]) {
         case 'if':
           stack.push(state);
-          try {
+          state = evaluateCondition(m[2]) ? STATE_IF_TRUE : STATE_IF_FALSE;
-            state = vm.runInNewContext(m[2], defines) ? 3 : 1;
+          break;
-          } catch (e) {
+        case 'elif':
-            console.error('Could not evalute line \'' + m[2] + '\' at ' +
+          if (state === STATE_IF_TRUE) {
-                          fs.realpathSync(inFilename) + ':' + lineNumber);
+            state = STATE_ELSE_FALSE;
-            throw e;
+          } else if (state === STATE_IF_FALSE) {
+            state = evaluateCondition(m[2]) ? STATE_IF_TRUE : STATE_IF_FALSE;
+          } else if (state === STATE_ELSE_TRUE || state === STATE_ELSE_FALSE) {
+            throw new Error('Found #elif after #else at ' + loc());
+          } else {
+            throw new Error('Found #elif without matching #if at ' + loc());
           }
           break;
         case 'else':
-          state = state === 1 ? 3 : 2;
+          if (state === STATE_IF_TRUE) {
+            state = STATE_ELSE_FALSE;
+          } else if (state === STATE_IF_FALSE) {
+            state = STATE_ELSE_TRUE;
+          } else {
+            throw new Error('Found #else without matching #if at ' + loc());
+          }
           break;
         case 'endif':
+          if (state === STATE_NONE) {
+            throw new Error('Found #endif without #if at ' + loc());
+          }
           state = stack.pop();
           break;
         case 'expand':
-          if (state === 0 || state === 3) {
+          if (state !== STATE_IF_FALSE && state !== STATE_ELSE_FALSE) {
             expand(m[2]);
           }
           break;
         case 'include':
-          if (state === 0 || state === 3) {
+          if (state !== STATE_IF_FALSE && state !== STATE_ELSE_FALSE) {
             include(m[2]);
           }
           break;
+        case 'error':
+          if (state !== STATE_IF_FALSE && state !== STATE_ELSE_FALSE) {
+            throw new Error('Found #error ' + m[2] + ' at ' + loc());
+          }
+          break;
       }
     } else {
-      if (state === 0) {
+      if (state === STATE_NONE) {
-        writeLine(s);
+        writeLine(line);
-      } else if (state === 3) {
+      } else if ((state === STATE_IF_TRUE || state === STATE_ELSE_TRUE) &&
-        writeLine(s.replace(/^\/\/|^<!--|-->/g, '  '));
+          stack.indexOf(STATE_IF_FALSE) === -1 &&
+          stack.indexOf(STATE_ELSE_FALSE) === -1) {
+        writeLine(line.replace(/^\/\/|^<!--|-->$/g, '  '));
       }
     }
   }
-  if (state !== 0 || stack.length !== 0) {
+  if (state !== STATE_NONE || stack.length !== 0) {
-    throw new Error('Missing endif in preprocessor.');
+    throw new Error('Missing #endif in preprocessor for ' +
+                    fs.realpathSync(inFilename));
   }
   if (typeof outFilename !== 'function') {
     fs.writeFileSync(outFilename, out);
@@ -235,21 +309,29 @@ function getWorkerSrcFiles(filePath) {
   var src = fs.readFileSync(filePath).toString();
   var reSrcFiles = /var\s+otherFiles\s*=\s*(\[[^\]]*\])/;
   var match = reSrcFiles.exec(src);
-  try {
+  if (!match) {
-    var files = JSON.parse(match[1].replace(/'/g, '"'));
+    throw new Error('Cannot find otherFiles array in ' + filePath);
-    var srcFiles = files.filter(function(name) {
-      return name.indexOf('external') === -1;
-    });
-    var externalSrcFiles = files.filter(function(name) {
-      return name.indexOf('external') > -1;
-    });
-    return {
-      srcFiles: srcFiles,
-      externalSrcFiles: externalSrcFiles
-    };
-  } catch(e) {
-    return {};
   }
+
+  var files = match[1].replace(/'/g, '"').replace(/^\s*\/\/.*/gm, '')
+    .replace(/,\s*]$/, ']');
+  try {
+    files = JSON.parse(files);
+  } catch (e) {
+    throw new Error('Failed to parse otherFiles in ' + filePath + ' as JSON, ' +
+                    e);
+  }
+
+  var srcFiles = files.filter(function(name) {
+    return name.indexOf('external') === -1;
+  });
+  var externalSrcFiles = files.filter(function(name) {
+    return name.indexOf('external') > -1;
+  });
+  return {
+    srcFiles: srcFiles,
+    externalSrcFiles: externalSrcFiles
+  };
 }
 exports.getWorkerSrcFiles = getWorkerSrcFiles;
 

external/builder/fixtures/confusing-comment-expected.js

@@ -0,0 +1,4 @@
+'use strict';
+var i = 0;
+while(i-->0) {
+}

external/builder/fixtures/confusing-comment.js

@@ -0,0 +1,6 @@
+'use strict';
+//#if TRUE
+var i = 0;
+while(i-->0) {
+}
+//#endif

external/builder/fixtures/elif-expected.js

@@ -0,0 +1 @@
+//Error: Found #elif without matching #if at __filename:2

external/builder/fixtures/elif.js

@@ -0,0 +1,4 @@
+'use strict';
+//#elif TRUE
+var a;
+//#endif

external/builder/fixtures/else-expected.js

@@ -0,0 +1 @@
+//Error: Found #else without matching #if at __filename:2

external/builder/fixtures/else.js

@@ -0,0 +1,3 @@
+'use strict';
+//#else
+//#endif

external/builder/fixtures/error-expected.js

@@ -0,0 +1 @@
+//Error: Found #error "Some Error" at __filename:3

external/builder/fixtures/error-false-expected.js

@@ -0,0 +1,2 @@
+'use strict';
+var a;

external/builder/fixtures/error-false.js

@@ -0,0 +1,5 @@
+'use strict';
+//#if FALSE
+//#error "Some Error"
+//#endif
+var a;

external/builder/fixtures/error.js

@@ -0,0 +1,5 @@
+'use strict';
+//#if TRUE
+//#error "Some Error"
+//#endif
+var b;

external/builder/fixtures/expand-expected.html

@@ -0,0 +1 @@
+prefixtruesuffix

external/builder/fixtures/expand.html

@@ -0,0 +1 @@
+<!--#expand prefix__TRUE__suffix-->

external/builder/fixtures/if-empty-expected.js

@@ -0,0 +1 @@
+//Error: No JavaScript expression given at __filename:2

external/builder/fixtures/if-empty.js

@@ -0,0 +1,6 @@
+'use strict';
+//#if
+var a;
+//#else
+var b;
+//#endif

external/builder/fixtures/if-false-elif-false-else-expected.js

@@ -0,0 +1,2 @@
+'use strict';
+var c;

external/builder/fixtures/if-false-elif-false-else.js

@@ -0,0 +1,8 @@
+'use strict';
+//#if FALSE
+var a;
+//#elif FALSE
+var b;
+//#else
+var c;
+//#endif

external/builder/fixtures/if-false-elif-true-else-expected.js

@@ -0,0 +1,2 @@
+'use strict';
+var b;

external/builder/fixtures/if-false-elif-true-else.js

@@ -0,0 +1,8 @@
+'use strict';
+//#if FALSE
+var a;
+//#elif TRUE
+var b;
+//#else
+var c;
+//#endif

external/builder/fixtures/if-false-else-expected.js

@@ -0,0 +1,2 @@
+'use strict';
+var b;

external/builder/fixtures/if-false-else.js

@@ -0,0 +1,6 @@
+'use strict';
+//#if FALSE
+var a;
+//#else
+var b;
+//#endif

external/builder/fixtures/if-nested-expected.js

@@ -0,0 +1,6 @@
+'use strict';
+var a;
+
+var b;
+
+var d;

external/builder/fixtures/if-nested.js

@@ -0,0 +1,19 @@
+'use strict';
+//#if TRUE
+var a;
+
+//#if TRUE
+var b;
+//#else
+var c;
+//#endif
+
+var d;
+//#else
+var e;
+//#if TRUE
+var f;
+//#endif
+
+var g;
+//#endif

external/builder/fixtures/if-true-else-expected.js

@@ -0,0 +1,2 @@
+'use strict';
+var a;

external/builder/fixtures/if-true-else.js

@@ -0,0 +1,6 @@
+'use strict';
+//#if TRUE
+var a;
+//#else
+var b;
+//#endif

external/builder/fixtures/if-unclosed-expected.js

@@ -0,0 +1 @@
+//Error: Missing #endif in preprocessor for __filename

external/builder/fixtures/if-unclosed.js

@@ -0,0 +1,3 @@
+'use strict';
+//#if TRUE
+var a;

external/builder/fixtures/include-expected.html

@@ -0,0 +1,5 @@
+<script>
+'use strict';
+var a;
+
+</script>

external/builder/fixtures/include-non-existent-expected.html

@@ -0,0 +1 @@
+//Error: Failed to include "some file that does not exist" at __filename:2

external/builder/fixtures/include-non-existent.html

@@ -0,0 +1,2 @@
+<!-- Non-existent file -->
+<!--#include some file that does not exist-->

external/builder/fixtures/include.html

@@ -0,0 +1,5 @@
+<script>
+<!--#if TRUE-->
+<!--#include if-true-else.js-->
+<!--#endif-->
+</script>

external/builder/fixtures/js-comment-expected.js

@@ -0,0 +1,4 @@
+'use strict';
+  //var a;
+  var b;
+var c;

external/builder/fixtures/js-comment.js

@@ -0,0 +1,6 @@
+'use strict';
+//#if TRUE
+////var a;
+//var b;
+var c;
+//#endif

external/builder/fixtures/undefined-define-expected.js

@@ -0,0 +1,2 @@
+//Error: Could not evaluate "notdefined" at __filename:2
+//ReferenceError: notdefined is not defined

external/builder/fixtures/undefined-define.js

@@ -0,0 +1,6 @@
+'use strict';
+//#if notdefined
+var a;
+//#else
+var b;
+//#endif

external/builder/fixtures/unsupported-ifdef-expected.js

@@ -0,0 +1 @@
+//Error: Found #endif without #if at __filename:4

external/builder/fixtures/unsupported-ifdef.js

@@ -0,0 +1,5 @@
+'use strict';
+//#ifdef TRUE
+//ifdef should not be recognized
+//#endif
+var a;

external/builder/test.js

@@ -0,0 +1,54 @@
+/* -*- Mode: Java; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set shiftwidth=2 tabstop=2 autoindent cindent expandtab: */
+/* jshint node:true */
+/* globals cat, cd, echo, ls */
+'use strict';
+
+require('shelljs/make');
+
+var builder = require('./builder');
+var fs = require('fs');
+var path = require('path');
+
+var errors = 0;
+
+cd(__dirname);
+cd('fixtures');
+ls('*-expected.*').forEach(function(expectationFilename) {
+  var inFilename = expectationFilename.replace('-expected', '');
+  var expectation = cat(expectationFilename).trim()
+    .replace(/__filename/g, fs.realpathSync(inFilename));
+  var outLines = [];
+
+  var outFilename = function(line) {
+    outLines.push(line);
+  };
+  var defines = {
+    TRUE: true,
+    FALSE: false,
+  };
+  var out;
+  try {
+    builder.preprocess(inFilename, outFilename, defines);
+    out = outLines.join('\n').trim();
+  } catch (e) {
+    out = ('Error: ' + e.message).replace(/^/gm, '//');
+  }
+  if (out !== expectation) {
+    echo('Assertion failed for ' + inFilename);
+    echo('--------------------------------------------------');
+    echo('EXPECTED:');
+    echo(expectation);
+    echo('--------------------------------------------------');
+    echo('ACTUAL');
+    echo(out);
+    echo('--------------------------------------------------');
+    echo();
+  }
+});
+
+if (errors) {
+  echo('Found ' + errors + ' expectation failures.');
+} else {
+  echo('All tests completed without errors.');
+}

make.js

@@ -482,9 +482,8 @@ target.bundle = function(args) {
   cd(ROOT_DIR);
   echo();
   echo('### Bundling files into ' + BUILD_TARGET);
-  var reg = /\n\/\* -\*- Mode(.|\n)*?Mozilla Foundation(.|\n)*?'use strict';/g;
 
-  function bundle(filename, dir, SRC_FILES, EXT_SRC_FILES) {
+  function bundle(filename, outfilename, SRC_FILES, EXT_SRC_FILES) {
     for (var i = 0, length = excludes.length; i < length; ++i) {
       var exclude = excludes[i];
       var index = SRC_FILES.indexOf(exclude);
@@ -500,15 +499,17 @@ target.bundle = function(args) {
 
     crlfchecker.checkIfCrlfIsPresent(SRC_FILES);
 
-    // Strip out all the vim/license headers.
+    // Prepend a newline because stripCommentHeaders only strips comments that
-    bundleContent = bundleContent.replace(reg, '');
+    // follow a line feed. The file where bundleContent is inserted already
+    // contains a license header, so the header of bundleContent can be removed.
+    bundleContent = stripCommentHeaders('\n' + bundleContent);
 
     // Append external files last since we don't want to modify them.
     bundleContent += cat(EXT_SRC_FILES);
 
     // This just preprocesses the empty pdf.js file, we don't actually want to
     // preprocess everything yet since other build targets use this file.
-    builder.preprocess(filename, dir, builder.merge(defines,
+    builder.preprocess(filename, outfilename, builder.merge(defines,
                            {BUNDLE: bundleContent,
                             BUNDLE_VERSION: bundleVersion,
                             BUNDLE_BUILD: bundleBuild}));
@@ -600,12 +601,21 @@ target.singlefile = function() {
 
 };
 
+function stripCommentHeaders(content, filename) {
+  // Strip out all the vim/license headers.
+  var notEndOfComment = '(?:[^*]|\\*(?!/))+';
+  var reg = new RegExp(
+    '\n/\\* -\\*- Mode' + notEndOfComment + '\\*/\\s*' +
+    '(?:/\\*' + notEndOfComment + '\\*/\\s*|//(?!#).*\n\\s*)+' +
+    '\'use strict\';', 'g');
+  content = content.replace(reg, '');
+  return content;
+}
+
 function cleanupJSSource(file) {
   var content = cat(file);
 
-  // Strip out all the vim/license headers.
+  content = stripCommentHeaders(content, file);
-  var reg = /\n\/\* -\*- Mode(.|\n)*?Mozilla Foundation(.|\n)*?'use strict';/g;
-  content = content.replace(reg, '');
 
   content.to(file);
 }