Merge pull request #1444 from arturadib/new-bot

New files for Windows bot

external/shelljs/ChangeLog

@@ -0,0 +1,22 @@
+2012.03.22, Version 0.0.4
+
+* ls() and find() return arrays instead of hashes (Artur Adib)
+* exec({silent:...}) overrides global silent() state (Artur Adib)
+
+
+2012.03.21, Version 0.0.3
+
+* Wildcard bug fix (Artur Adib)
+* execSync() now uses dummy file I/O op to reduce CPU usage (Artur Adib)
+* Minor fixes
+
+
+2012.03.15, Version 0.0.2
+
+* New methods: find(), test() (Artur Adib)
+* Deprecated non-Unix methods: exists(), verbose()
+
+
+2012.03.03, Version 0.0.2pre1
+
+* First public release

external/shelljs/README.md

@@ -1,6 +1,7 @@
 # ShellJS - Unix shell commands for Node.js [![Build Status](https://secure.travis-ci.org/arturadib/shelljs.png)](http://travis-ci.org/arturadib/shelljs)
 
-_This project is young and experimental. Use at your own risk._
++ _This project is young and experimental. Use at your own risk._
++ _Major API change as of v0.0.4: `ls()` and `find()` now return arrays._
 
 ShellJS is a **portable** (Windows included) implementation of Unix shell commands on top of the Node.js API. You can use it to eliminate your shell script's dependency on Unix while still keeping its familiar and powerful commands.
 
@@ -18,11 +19,11 @@ cp('-R', 'stuff/*', 'out/Release');
 
 // Replace macros in each .js file
 cd('lib');
-for (file in ls('*.js')) {
+ls('*.js').forEach(function(file) {
   sed('-i', 'BUILD_VERSION', 'v0.1.2', file);
   sed('-i', /.*REMOVE_THIS_LINE.*\n/, '', file);
   sed('-i', /.*REPLACE_LINE_WITH_MACRO.*\n/, cat('macro.js'), file);
-}
+});
 cd('..');
 
 // Run external tool synchronously
@@ -73,11 +74,11 @@ target.docs = function() {
   cd(__dirname);
   mkdir('docs');
   cd('lib');
-  for (file in ls('*.js')) {
+  ls('*.js').forEach(function(file){
     var text = grep('//@', file); // extract special comments
     text.replace('//@', ''); // remove comment tags
     text.to('docs/my_docs.md');
-  }
+  });
 }
 ```
 
@@ -128,9 +129,7 @@ ls('-R', '/users/me', '/tmp');
 ls('-R', ['/users/me', '/tmp']); // same as above
 ```
 
-Returns list of files in the given path, or in current directory if no path provided.
+Returns array of files in the given path, or in current directory if no path provided.
-For convenient iteration via `for (file in ls())`, the format returned is a hash object:
-`{ 'file1':null, 'dir1/file2':null, ...}`.
 
 #### find(path [,path ...])
 #### find(path_array)
@@ -139,18 +138,12 @@ Examples:
 ```javascript
 find('src', 'lib');
 find(['src', 'lib']); // same as above
-for (file in find('.')) {
+find('.').filter(function(file) { return file.match(/\.js$/); });
-if (!file.match(/\.js$/))
-continue;
-// all files at this point end in '.js'
-}
 ```
 
-Returns list of all files (however deep) in the given paths. For convenient iteration 
+Returns array of all files (however deep) in the given paths.
-via `for (file in find(...))`, the format returned is a hash object:
-`{ 'file1':null, 'dir1/file2':null, ...}`.
 
-The main difference with respect to `ls('-R', path)` is that the resulting file names 
+The main difference from `ls('-R', path)` is that the resulting file names 
 include the base directories, e.g. `lib/resources/file1` instead of just `file1`.
 
 #### cp('[options ,] source [,source ...], dest')
@@ -332,6 +325,10 @@ When in synchronous mode returns the object `{ code:..., output:... }`, containi
 `output` (stdout + stderr)  and its exit `code`. Otherwise the `callback` gets the 
 arguments `(code, output)`.
 
+**Note:** For long-lived processes, it's best to run `exec()` asynchronously as
+the current synchronous implementation uses a lot of CPU. This should be getting
+fixed soon.
+
 ## Non-Unix commands
 
 
@@ -339,16 +336,35 @@ arguments `(code, output)`.
 Searches and returns string containing a writeable, platform-dependent temporary directory.
 Follows Python's [tempfile algorithm](http://docs.python.org/library/tempfile.html#tempfile.tempdir).
 
-#### exists(path [, path ...])
-#### exists(path_array)
-Returns true if all the given paths exist.
-
 #### error()
 Tests if error occurred in the last command. Returns `null` if no error occurred,
 otherwise returns string explaining the error
 
-#### verbose()
+#### silent([state])
-Enables all output (default)
+Example:
 
-#### silent()
+```javascript
-Suppresses all output, except for explict `echo()` calls
+var silentState = silent();
+silent(true);
+/* ... */
+silent(silentState); // restore old silent state
+```
+
+Suppresses all command output if `state = true`, except for `echo()` calls. 
+Returns state if no arguments given.
+
+## Deprecated
+
+
+#### exists(path [, path ...])
+#### exists(path_array)
+
+_This function is being deprecated. Use `test()` instead._
+
+Returns true if all the given paths exist.
+
+#### verbose()
+
+_This function is being deprecated. Use `silent(false) instead.`_
+
+Enables all output (default)

external/shelljs/make.js

@@ -23,7 +23,7 @@ setTimeout(function() {
         if (oldTarget.done && !force)
           return;
         oldTarget.done = true;
-        return oldTarget(arguments);
+        return oldTarget.apply(oldTarget, arguments);
       }
 
     })(t, target[t]);

external/shelljs/package.json

@@ -1,12 +1,29 @@
-{ "name": "shelljs"
+{
-, "version": "0.0.2pre1"
+  "name": "shelljs",
-, "author": "Artur Adib <aadib@mozilla.com>"
+  "version": "0.0.5pre4",
-, "description": "Portable Unix shell commands for Node.js"
+  "author": "Artur Adib <aadib@mozilla.com>",
-, "keywords": ["unix", "shell", "makefile", "make", "jake", "synchronous"]
+  "description": "Portable Unix shell commands for Node.js",
-, "repository": "git://github.com/arturadib/shelljs"
+  "keywords": [
-, "homepage": "http://github.com/arturadib/shelljs"
+    "unix",
-, "main": "./shell.js"
+    "shell",
-, "scripts": {
+    "makefile",
+    "make",
+    "jake",
+    "synchronous"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "git://github.com/arturadib/shelljs.git"
+  },
+  "homepage": "http://github.com/arturadib/shelljs",
+  "main": "./shell.js",
+  "scripts": {
     "test": "node scripts/run-tests"
+  },
+  "dependencies": {},
+  "devDependencies": {},
+  "optionalDependencies": {},
+  "engines": {
+    "node": "*"
   }
 }

external/shelljs/shell.js

@@ -74,9 +74,7 @@ exports.pwd = wrap('pwd', _pwd);
 //@ ls('-R', ['/users/me', '/tmp']); // same as above
 //@ ```
 //@
-//@ Returns list of files in the given path, or in current directory if no path provided.
+//@ Returns array of files in the given path, or in current directory if no path provided.
-//@ For convenient iteration via `for (file in ls())`, the format returned is a hash object:
-//@ `{ 'file1':null, 'dir1/file2':null, ...}`.
 function _ls(options, paths) {
   options = parseOptions(options, {
     'R': 'recursive',
@@ -90,24 +88,30 @@ function _ls(options, paths) {
   else if (typeof paths === 'string')
     paths = [].slice.call(arguments, 1);
 
-  var hash = {};
+  var list = [];
 
-  function pushHash(file, query) {
+  // Conditionally pushes file to list - returns true if pushed, false otherwise
+  // (e.g. prevents hidden files to be included unless explicitly told so)
+  function pushFile(file, query) {
     // hidden file?
     if (path.basename(file)[0] === '.') {
       // not explicitly asking for hidden files?
       if (!options.all && !(path.basename(query)[0] === '.' && path.basename(query).length > 1))
-        return;
+        return false;
     }
 
-    hash[file] = null;
+    if (platform === 'win')
+      file = file.replace(/\\/g, '/');
+
+    list.push(file);
+    return true;
   }
 
   paths.forEach(function(p) {
     if (fs.existsSync(p)) {
       // Simple file?
       if (fs.statSync(p).isFile()) {
-        pushHash(p, p);
+        pushFile(p, p);
         return; // continue
       }
       
@@ -115,14 +119,17 @@ function _ls(options, paths) {
       if (fs.statSync(p).isDirectory()) {
         // Iterate over p contents
         fs.readdirSync(p).forEach(function(file) {
-          pushHash(file, p);
+          if (!pushFile(file, p))
+            return;
 
-          // Recursive
+          // Recursive?
-          var oldDir = _pwd();
+          if (options.recursive) {
-          _cd('', p);
+            var oldDir = _pwd();
-          if (fs.statSync(file).isDirectory() && options.recursive)
+            _cd('', p);
-            hash = extend(hash, _ls('-R', file+'/*'));
+            if (fs.statSync(file).isDirectory())
-          _cd('', oldDir);
+              list = list.concat(_ls('-R'+(options.all?'a':''), file+'/*'));
+            _cd('', oldDir);
+          }
         });
         return; // continue
       }
@@ -137,17 +144,20 @@ function _ls(options, paths) {
       // Escape special regular expression chars
       var regexp = basename.replace(/(\^|\$|\(|\)|\<|\>|\[|\]|\{|\}|\.|\+|\?)/g, '\\$1');
       // Translates wildcard into regex
-      regexp = '^' + regexp.replace(/\*/g, '.*');
+      regexp = '^' + regexp.replace(/\*/g, '.*') + '$';
       // Iterate over directory contents
       fs.readdirSync(dirname).forEach(function(file) {
         if (file.match(new RegExp(regexp))) {
-          pushHash(path.normalize(dirname+'/'+file), basename);
+          if (!pushFile(path.normalize(dirname+'/'+file), basename))
+            return;
 
-          // Recursive
+          // Recursive?
-          var pp = dirname + '/' + file;
+          if (options.recursive) {
-          if (fs.statSync(pp).isDirectory() && options.recursive)
+            var pp = dirname + '/' + file;
-            hash = extend(hash, _ls('-R', pp+'/*'));
+            if (fs.statSync(pp).isDirectory())
-        }
+              list = list.concat(_ls('-R'+(options.all?'a':''), pp+'/*'));
+          } // recursive
+        } // if file matches
       }); // forEach
       return;
     }
@@ -155,7 +165,7 @@ function _ls(options, paths) {
     error('no such file or directory: ' + p, true);
   });
 
-  return hash;
+  return list;
 };
 exports.ls = wrap('ls', _ls);
 
@@ -168,16 +178,10 @@ exports.ls = wrap('ls', _ls);
 //@ ```javascript
 //@ find('src', 'lib');
 //@ find(['src', 'lib']); // same as above
-//@ for (file in find('.')) {
+//@ find('.').filter(function(file) { return file.match(/\.js$/); });
-//@   if (!file.match(/\.js$/))
-//@     continue;
-//@   // all files at this point end in '.js'
-//@ }
 //@ ```
 //@
-//@ Returns list of all files (however deep) in the given paths. For convenient iteration 
+//@ Returns array of all files (however deep) in the given paths.
-//@ via `for (file in find(...))`, the format returned is a hash object:
-//@ `{ 'file1':null, 'dir1/file2':null, ...}`.
 //@
 //@ The main difference from `ls('-R', path)` is that the resulting file names 
 //@ include the base directories, e.g. `lib/resources/file1` instead of just `file1`.
@@ -189,21 +193,28 @@ function _find(options, paths) {
   else if (typeof paths === 'string')
     paths = [].slice.call(arguments, 1);
 
-  var hash = {};
+  var list = [];
+
+  function pushFile(file) {
+    if (platform === 'win')
+      file = file.replace(/\\/g, '/');
+    list.push(file);
+  }
 
   // why not simply do ls('-R', paths)? because the output wouldn't give the base dirs
   // to get the base dir in the output, we need instead ls('-R', 'dir/*') for every directory
 
-  paths.forEach(function(file){
+  paths.forEach(function(file) {
-    hash[file] = null;
+    pushFile(file);
 
     if (fs.statSync(file).isDirectory()) {
-      for (subfile in _ls('-Ra', file+'/*'))
+      _ls('-Ra', file+'/*').forEach(function(subfile) {
-        hash[subfile] = null;
+        pushFile(subfile);
+      });
     }
   });
 
-  return hash;
+  return list;
 }
 exports.find = wrap('find', _find);
 
@@ -347,9 +358,20 @@ function _rm(options, files) {
 
     // Remove simple file
     if (fs.statSync(file).isFile()) {
-      fs.unlinkSync(file);
+
+      // Do not check for file writing permissions
+      if (options.force) {
+        _unlinkSync(file);
+        return;
+      }
+            
+      if (isWriteable(file))
+        _unlinkSync(file);
+      else
+        error('permission denied: '+file, true);
+
       return;
-    }
+    } // simple file
 
     // Path is an existing directory, but no -r flag given
     if (fs.statSync(file).isDirectory() && !options.recursive) {
@@ -359,7 +381,7 @@ function _rm(options, files) {
 
     // Recursively remove existing directory
     if (fs.statSync(file).isDirectory() && options.recursive) {
-      rmdirSyncRecursive(file);
+      rmdirSyncRecursive(file, options.force);
     }
   }); // forEach(file)
 }; // rm
@@ -582,7 +604,11 @@ function _to(options, file) {
   if (!fs.existsSync( path.dirname(file) ))
       error('no such file or directory: ' + path.dirname(file));
 
-  fs.writeFileSync(file, this.toString(), 'utf8');
+  try {
+    fs.writeFileSync(file, this.toString(), 'utf8');
+  } catch(e) {
+    error('could not write to file (code '+e.code+'): '+file, true);
+  }
 };
 // In the future, when Proxies are default, we can add methods like `.to()` to primitive strings. 
 // For now, this is a dummy function to bookmark places we need such strings
@@ -751,7 +777,7 @@ exports.which = wrap('which', _which);
 //@ like `.to()`.
 function _echo(options) {
   var messages = [].slice.call(arguments, 1);
-  log.apply(this, messages);
+  console.log.apply(this, messages);
   return ShellString(messages.join(' '));
 };
 exports.echo = wrap('echo', _echo);
@@ -783,6 +809,10 @@ exports.env = process.env;
 //@ When in synchronous mode returns the object `{ code:..., output:... }`, containing the program's 
 //@ `output` (stdout + stderr)  and its exit `code`. Otherwise the `callback` gets the 
 //@ arguments `(code, output)`.
+//@
+//@ **Note:** For long-lived processes, it's best to run `exec()` asynchronously as
+//@ the current synchronous implementation uses a lot of CPU. This should be getting
+//@ fixed soon.
 function _exec(command, options, callback) {
   if (!command)
     error('must specify command');
@@ -793,7 +823,7 @@ function _exec(command, options, callback) {
   }
 
   options = extend({
-    silent: false,
+    silent: state.silent,
     async: false
   }, options);
 
@@ -822,11 +852,53 @@ exports.exec = wrap('exec', _exec, {notUnix:true});
 //@ Follows Python's [tempfile algorithm](http://docs.python.org/library/tempfile.html#tempfile.tempdir).
 exports.tempdir = wrap('tempdir', tempDir);
 
+
+//@
+//@ #### error()
+//@ Tests if error occurred in the last command. Returns `null` if no error occurred,
+//@ otherwise returns string explaining the error
+exports.error = function() {
+  return state.error;
+}
+
+//@
+//@ #### silent([state])
+//@ Example:
+//@
+//@ ```javascript
+//@ var silentState = silent();
+//@ silent(true);
+//@ /* ... */
+//@ silent(silentState); // restore old silent state
+//@ ```
+//@
+//@ Suppresses all command output if `state = true`, except for `echo()` calls. 
+//@ Returns state if no arguments given.
+exports.silent = function(_state) {
+  if (typeof _state !== 'boolean')
+    return state.silent;
+  
+  state.silent = _state;
+}
+
+
+//@
+//@ ## Deprecated
+//@
+
+
+
+
 //@
 //@ #### exists(path [, path ...])
 //@ #### exists(path_array)
+//@
+//@ _This function is being deprecated. Use `test()` instead._
+//@
 //@ Returns true if all the given paths exist.
 function _exists(options, paths) {
+  deprecate('exists', 'Use test() instead.');
+
   if (!paths)
     error('no paths given');
 
@@ -844,32 +916,19 @@ function _exists(options, paths) {
 };
 exports.exists = wrap('exists', _exists);
 
-//@
-//@ #### error()
-//@ Tests if error occurred in the last command. Returns `null` if no error occurred,
-//@ otherwise returns string explaining the error
-exports.error = function() {
-  return state.error;
-}
 
 //@
 //@ #### verbose()
+//@
+//@ _This function is being deprecated. Use `silent(false) instead.`_
+//@
 //@ Enables all output (default)
 exports.verbose = function() {
+  deprecate('verbose', 'Use silent(false) instead.');
+
   state.silent = false;
 }
 
-//@
-//@ #### silent()
-//@ Suppresses all output, except for explict `echo()` calls
-exports.silent = function() {
-  state.silent = true;
-}
-
-
-
-
-
 
 
 
@@ -889,6 +948,10 @@ function log() {
     console.log.apply(this, arguments);
 }
 
+function deprecate(what, msg) {
+  console.log('*** ShellJS.'+what+': This function is deprecated.', msg);
+}
+
 function write(msg) {
   if (!state.silent)
     process.stdout.write(msg);
@@ -962,7 +1025,7 @@ function wrap(cmd, fn, options) {
     } catch (e) {
       if (!state.error) {
         // If state.error hasn't been set it's an error thrown by Node, not us - probably a bug...
-        console.log('maker.js: internal error');
+        console.log('shell.js: internal error');
         console.log(e.stack || e);
         process.exit(1);
       }
@@ -970,7 +1033,7 @@ function wrap(cmd, fn, options) {
         throw e;
     }
 
-    state.currentCmd = 'maker.js';
+    state.currentCmd = 'shell.js';
     return retValue;
   }
 } // wrap
@@ -984,10 +1047,22 @@ function copyFileSync(srcFile, destFile) {
 
   var BUF_LENGTH = 64*1024,
       buf = new Buffer(BUF_LENGTH),
-      fdr = fs.openSync(srcFile, 'r'),
-      fdw = fs.openSync(destFile, 'w'),
       bytesRead = BUF_LENGTH,
-      pos = 0;
+      pos = 0,
+      fdr = null,
+      fdw = null;
+
+  try {
+    fdr = fs.openSync(srcFile, 'r');
+  } catch(e) {
+    error('copyFileSync: could not read src file ('+srcFile+')');
+  }
+
+  try {
+    fdw = fs.openSync(destFile, 'w');
+  } catch(e) {
+    error('copyFileSync: could not write to dest file (code='+e.code+'):'+destFile);
+  }
 
   while (bytesRead === BUF_LENGTH) {
     bytesRead = fs.readSync(fdr, buf, 0, BUF_LENGTH, pos);
@@ -1050,28 +1125,41 @@ function cpdirSyncRecursive(sourceDir, destDir, opts) {
 //
 // Licensed under the MIT License
 // http://www.opensource.org/licenses/mit-license.php
-function rmdirSyncRecursive(dir) {
+function rmdirSyncRecursive(dir, force) {
   var files;
 
   files = fs.readdirSync(dir);
 
   // Loop through and delete everything in the sub-tree after checking it
   for(var i = 0; i < files.length; i++) {
-    var currFile = fs.lstatSync(dir + "/" + files[i]);
+    var file = dir + "/" + files[i],
+        currFile = fs.lstatSync(file);
 
-    if(currFile.isDirectory()) // Recursive function back to the beginning
+    if(currFile.isDirectory()) { // Recursive function back to the beginning
-      rmdirSyncRecursive(dir + "/" + files[i]);
+      rmdirSyncRecursive(file, force);
+    }
 
-    else if(currFile.isSymbolicLink()) // Unlink symlinks
+    else if(currFile.isSymbolicLink()) { // Unlink symlinks
-      fs.unlinkSync(dir + "/" + files[i]);
+      if (force || isWriteable(file))
+        _unlinkSync(file);
+    }
 
     else // Assume it's a file - perhaps a try/catch belongs here?
-      fs.unlinkSync(dir + "/" + files[i]);
+      if (force || isWriteable(file))
+        _unlinkSync(file);
   }
 
   // Now that we know everything in the sub-tree has been deleted, we can delete the main directory. 
   // Huzzah for the shopkeep.
-  return fs.rmdirSync(dir);
+
+  var result;
+  try {
+    result = fs.rmdirSync(dir);
+  } catch(e) {
+    error('could not remove directory (code '+e.code+'): ' + dir, true);
+  }
+
+  return result;
 }; // rmdirSyncRecursive
 
 // Recursively creates 'dir'
@@ -1118,7 +1206,7 @@ function writeableDir(dir) {
   var testFile = dir+'/'+randomFileName();
   try {
     fs.writeFileSync(testFile, ' ');
-    fs.unlinkSync(testFile);
+    _unlinkSync(testFile);
     return dir;
   } catch (e) {
     return false;
@@ -1151,6 +1239,10 @@ function tempDir() {
 // Wrapper around exec() to enable echoing output to console in real time
 function execAsync(cmd, opts, callback) {
   var output = '';
+
+  var options = extend({
+    silent: state.silent
+  }, opts);
   
   var c = child.exec(cmd, {env: process.env}, function(err) {
     if (callback) 
@@ -1159,14 +1251,14 @@ function execAsync(cmd, opts, callback) {
 
   c.stdout.on('data', function(data) {
     output += data;
-    if (!opts.silent)
+    if (!options.silent)
-      write(data);
+      process.stdout.write(data);
   });
 
   c.stderr.on('data', function(data) {
     output += data;
-    if (!opts.silent)
+    if (!options.silent)
-      write(data);
+      process.stdout.write(data);
   });
 }
 
@@ -1178,16 +1270,17 @@ function execAsync(cmd, opts, callback) {
 function execSync(cmd, opts) {
   var stdoutFile = path.resolve(tempDir()+'/'+randomFileName()),
       codeFile = path.resolve(tempDir()+'/'+randomFileName()),
-      scriptFile = path.resolve(tempDir()+'/'+randomFileName());
+      scriptFile = path.resolve(tempDir()+'/'+randomFileName()),
+      sleepFile = path.resolve(tempDir()+'/'+randomFileName());
 
   var options = extend({
-    silent: false
+    silent: state.silent
   }, opts);
 
   var previousStdoutContent = '';
   // Echoes stdout changes from running process, if not silent
   function updateStdout() {
-    if (state.silent || options.silent || !fs.existsSync(stdoutFile))
+    if (options.silent || !fs.existsSync(stdoutFile))
       return;
 
     var stdoutContent = fs.readFileSync(stdoutFile, 'utf8');
@@ -1214,9 +1307,9 @@ function execSync(cmd, opts) {
       fs.writeFileSync('"+escape(codeFile)+"', err ? err.code.toString() : '0'); \
     });";
 
-  if (fs.existsSync(scriptFile)) fs.unlinkSync(scriptFile);
+  if (fs.existsSync(scriptFile)) _unlinkSync(scriptFile);
-  if (fs.existsSync(stdoutFile)) fs.unlinkSync(stdoutFile);
+  if (fs.existsSync(stdoutFile)) _unlinkSync(stdoutFile);
-  if (fs.existsSync(codeFile)) fs.unlinkSync(codeFile);
+  if (fs.existsSync(codeFile)) _unlinkSync(codeFile);
 
   fs.writeFileSync(scriptFile, script);
   child.exec('node '+scriptFile, { 
@@ -1225,8 +1318,11 @@ function execSync(cmd, opts) {
   });
 
   // The wait loop
-  while (!fs.existsSync(codeFile)) { updateStdout(); };
+  // sleepFile is used as a dummy I/O op to mitigate unnecessary CPU usage
-  while (!fs.existsSync(stdoutFile)) { updateStdout(); };
+  // (tried many I/O sync ops, writeFileSync() seems to be only one that is effective in reducing
+  // CPU usage, though apparently not so much on Windows)
+  while (!fs.existsSync(codeFile)) { updateStdout(); fs.writeFileSync(sleepFile, 'a'); };
+  while (!fs.existsSync(stdoutFile)) { updateStdout(); fs.writeFileSync(sleepFile, 'a'); };
 
   // At this point codeFile exists, but it's not necessarily flushed yet.
   // Keep reading it until it is.
@@ -1236,10 +1332,12 @@ function execSync(cmd, opts) {
 
   var stdout = fs.readFileSync(stdoutFile, 'utf8');
 
-  fs.unlinkSync(scriptFile);
+  // No biggie if we can't erase the files now -- they're in a temp dir anyway
-  fs.unlinkSync(stdoutFile);
+  try { _unlinkSync(scriptFile); } catch(e) {};
-  fs.unlinkSync(codeFile);
+  try { _unlinkSync(stdoutFile); } catch(e) {};
-
+  try { _unlinkSync(codeFile); } catch(e) {};
+  try { _unlinkSync(sleepFile); } catch(e) {};
+  
   // True if successful, false if not
   var obj = {
     code: code,
@@ -1257,8 +1355,9 @@ function expand(list) {
   list.forEach(function(listEl) {
     // Wildcard present? 
     if (listEl.search(/\*/) > -1) {
-      for (file in _ls('', listEl))
+      _ls('', listEl).forEach(function(file) {
         expanded.push(file);
+      });
     } else {
       expanded.push(listEl);
     }
@@ -1290,3 +1389,33 @@ function extend(target) {
   
   return target;
 }
+
+// Normalizes _unlinkSync() across platforms to match Unix behavior, i.e.
+// file can be unlinked even if it's read-only, see joyent/node#3006
+function _unlinkSync(file) {
+  try {
+    fs.unlinkSync(file);
+  } catch(e) {
+    // Try to override file permission
+    if (e.code === 'EPERM') {
+      fs.chmodSync(file, '0666');
+      fs.unlinkSync(file);
+    } else {
+      throw e;
+    }
+  }
+}
+
+// Hack to determine if file has write permissions for current user
+// Avoids having to check user, group, etc, but it's probably slow
+function isWriteable(file) {
+  var writePermission = true;
+  try {
+    var __fd = fs.openSync(file, 'a');
+    fs.closeSync(__fd);
+  } catch(e) {
+    writePermission = false;
+  }
+
+  return writePermission;
+}

make.js

@@ -100,7 +100,7 @@ target.bundle = function() {
         'bidi.js',
         'metadata.js'];
 
-  if (!exists(BUILD_DIR))
+  if (!test('-d', BUILD_DIR))
     mkdir(BUILD_DIR);
 
   cd('src');
@@ -143,10 +143,10 @@ target.pagesrepo = function() {
   echo();
   echo('### Creating fresh clone of gh-pages');
 
-  if (!exists(BUILD_DIR))
+  if (!test('-d', BUILD_DIR))
     mkdir(BUILD_DIR);
 
-  if (!exists(GH_PAGES_DIR)) {
+  if (!test('-d', GH_PAGES_DIR)) {
     echo();
     echo('Cloning project repo...');
     echo('(This operation can take a while, depending on network conditions)');
@@ -278,10 +278,10 @@ target.firefox = function() {
   // We don't need pdf.js anymore since its inlined
   rm('-Rf', FIREFOX_BUILD_CONTENT_DIR + BUILD_DIR);
   // Remove '.DS_Store' and other hidden files
-  for (file in find(FIREFOX_BUILD_DIR)) {
+  find(FIREFOX_BUILD_DIR).forEach(function(file) {
     if (file.match(/^\./))
       rm('-f', file);
-  }
+  });
 
   // Update the build version number
   sed('-i', /PDFJSSCRIPT_VERSION/, EXTENSION_VERSION, FIREFOX_BUILD_DIR + '/install.rdf');
@@ -305,10 +305,10 @@ target.firefox = function() {
   // List all files for mozilla-central
   cd(FIREFOX_BUILD_DIR);
   var extensionFiles = '';
-  for (file in find(FIREFOX_MC_EXTENSION_FILES)) {
+  find(FIREFOX_MC_EXTENSION_FILES).forEach(function(file){
     if (test('-f', file))
       extensionFiles += file+'\n';
-  }
+  });
   extensionFiles.to('extension-files');
 };
 
@@ -361,10 +361,19 @@ target.test = function() {
   target.unittest();
 };
 
+//
+// make bottest
+// (Special tests for the Github bot)
+//
+target.bottest = function() {
+  target.browsertest({noreftest: true});
+  // target.unittest();
+};
+
 //
 // make browsertest
 //
-target.browsertest = function() {
+target.browsertest = function(options) {
   cd(ROOT_DIR);
   echo();
   echo('### Running browser tests');
@@ -372,14 +381,16 @@ target.browsertest = function() {
   var PDF_TEST = env['PDF_TEST'] || 'test_manifest.json',
       PDF_BROWSERS = env['PDF_BROWSERS'] || 'resources/browser_manifests/browser_manifest.json';
 
-  if (!exists('test/' + PDF_BROWSERS)) {
+  if (!test('-f', 'test/' + PDF_BROWSERS)) {
     echo('Browser manifest file test/' + PDF_BROWSERS + ' does not exist.');
     echo('Try copying one of the examples in test/resources/browser_manifests/');
     exit(1);
   }
 
+  var reftest = (options && options.noreftest) ? '' : '--reftest';
+
   cd('test');
-  exec(PYTHON_BIN + ' test.py --reftest --browserManifestFile=' + PDF_BROWSERS +
+  exec(PYTHON_BIN + ' -u test.py '+reftest+' --browserManifestFile=' + PDF_BROWSERS +
     ' --manifestFile=' + PDF_TEST, {async: true});
 };
 
@@ -391,10 +402,37 @@ target.unittest = function() {
   echo();
   echo('### Running unit tests');
 
+  if (!which('make')) {
+    echo('make not found. Skipping unit tests...');
+    return;
+  }
+
   cd('test/unit');
   exec('make', {async: true});
 };
 
+//
+// make botmakeref
+//
+target.botmakeref = function() {
+  cd(ROOT_DIR);
+  echo();
+  echo('### Creating reference images');
+
+  var PDF_TEST = env['PDF_TEST'] || 'test_manifest.json',
+      PDF_BROWSERS = env['PDF_BROWSERS'] || 'resources/browser_manifests/browser_manifest.json';
+
+  if (!test('-f', 'test/' + PDF_BROWSERS)) {
+    echo('Browser manifest file test/' + PDF_BROWSERS + ' does not exist.');
+    echo('Try copying one of the examples in test/resources/browser_manifests/');
+    exit(1);
+  }
+
+  cd('test');
+  exec(PYTHON_BIN + ' -u test.py --masterMode --noPrompts --browserManifestFile=' + PDF_BROWSERS,
+    {async: true});
+};
+
 
 ///////////////////////////////////////////////////////////////////////////////////////////
 //

test/pdfs/wnv_chinese.pdf.link

@@ -1 +1 @@
-http://www.cdc.gov/ncidod/dvbid/westnile/languages/chinese.pdf
+http://web.archive.org/web/20110623114753/http://www.cdc.gov/ncidod/dvbid/westnile/languages/chinese.pdf