Upgrading ShellJS, introducing 'makeref'

This commit is contained in:
Artur Adib 2012-03-26 16:33:00 -04:00
parent 32eb64389e
commit a0a5c58894
5 changed files with 304 additions and 123 deletions

22
external/shelljs/ChangeLog vendored Normal file
View File

@ -0,0 +1,22 @@
2012.03.22, Version 0.0.4
* ls() and find() return arrays instead of hashes (Artur Adib)
* exec({silent:...}) overrides global silent() state (Artur Adib)
2012.03.21, Version 0.0.3
* Wildcard bug fix (Artur Adib)
* execSync() now uses dummy file I/O op to reduce CPU usage (Artur Adib)
* Minor fixes
2012.03.15, Version 0.0.2
* New methods: find(), test() (Artur Adib)
* Deprecated non-Unix methods: exists(), verbose()
2012.03.03, Version 0.0.2pre1
* First public release

View File

@ -1,6 +1,7 @@
# ShellJS - Unix shell commands for Node.js [![Build Status](https://secure.travis-ci.org/arturadib/shelljs.png)](http://travis-ci.org/arturadib/shelljs)
_This project is young and experimental. Use at your own risk._
+ _This project is young and experimental. Use at your own risk._
+ _Major API change as of v0.0.4: `ls()` and `find()` now return arrays._
ShellJS is a **portable** (Windows included) implementation of Unix shell commands on top of the Node.js API. You can use it to eliminate your shell script's dependency on Unix while still keeping its familiar and powerful commands.
@ -18,11 +19,11 @@ cp('-R', 'stuff/*', 'out/Release');
// Replace macros in each .js file
cd('lib');
for (file in ls('*.js')) {
ls('*.js').forEach(function(file) {
sed('-i', 'BUILD_VERSION', 'v0.1.2', file);
sed('-i', /.*REMOVE_THIS_LINE.*\n/, '', file);
sed('-i', /.*REPLACE_LINE_WITH_MACRO.*\n/, cat('macro.js'), file);
}
});
cd('..');
// Run external tool synchronously
@ -73,11 +74,11 @@ target.docs = function() {
cd(__dirname);
mkdir('docs');
cd('lib');
for (file in ls('*.js')) {
ls('*.js').forEach(function(file){
var text = grep('//@', file); // extract special comments
text.replace('//@', ''); // remove comment tags
text.to('docs/my_docs.md');
}
});
}
```
@ -128,9 +129,7 @@ ls('-R', '/users/me', '/tmp');
ls('-R', ['/users/me', '/tmp']); // same as above
```
Returns list of files in the given path, or in current directory if no path provided.
For convenient iteration via `for (file in ls())`, the format returned is a hash object:
`{ 'file1':null, 'dir1/file2':null, ...}`.
Returns array of files in the given path, or in current directory if no path provided.
#### find(path [,path ...])
#### find(path_array)
@ -139,18 +138,12 @@ Examples:
```javascript
find('src', 'lib');
find(['src', 'lib']); // same as above
for (file in find('.')) {
if (!file.match(/\.js$/))
continue;
// all files at this point end in '.js'
}
find('.').filter(function(file) { return file.match(/\.js$/); })
```
Returns list of all files (however deep) in the given paths. For convenient iteration
via `for (file in find(...))`, the format returned is a hash object:
`{ 'file1':null, 'dir1/file2':null, ...}`.
Returns array of all files (however deep) in the given paths.
The main difference with respect to `ls('-R', path)` is that the resulting file names
The main difference from `ls('-R', path)` is that the resulting file names
include the base directories, e.g. `lib/resources/file1` instead of just `file1`.
#### cp('[options ,] source [,source ...], dest')
@ -339,16 +332,34 @@ arguments `(code, output)`.
Searches and returns string containing a writeable, platform-dependent temporary directory.
Follows Python's [tempfile algorithm](http://docs.python.org/library/tempfile.html#tempfile.tempdir).
#### exists(path [, path ...])
#### exists(path_array)
Returns true if all the given paths exist.
#### error()
Tests if error occurred in the last command. Returns `null` if no error occurred,
otherwise returns string explaining the error
#### verbose()
Enables all output (default)
#### silent([state])
Example:
#### silent()
Suppresses all output, except for explict `echo()` calls
```javascript
var silentState = silent();
silent(true);
/* ... */
silent(silentState); // restore old silent state
```
Suppresses all output if `state = true`. Returns state if no arguments given.
## Deprecated
#### exists(path [, path ...])
#### exists(path_array)
_This function is being deprecated. Use `test()` instead._
Returns true if all the given paths exist.
#### verbose()
_This function is being deprecated. Use `silent(false) instead.`_
Enables all output (default)

View File

@ -1,5 +1,5 @@
{ "name": "shelljs"
, "version": "0.0.2pre1"
, "version": "0.0.5pre2"
, "author": "Artur Adib <aadib@mozilla.com>"
, "description": "Portable Unix shell commands for Node.js"
, "keywords": ["unix", "shell", "makefile", "make", "jake", "synchronous"]

View File

@ -74,9 +74,7 @@ exports.pwd = wrap('pwd', _pwd);
//@ ls('-R', ['/users/me', '/tmp']); // same as above
//@ ```
//@
//@ Returns list of files in the given path, or in current directory if no path provided.
//@ For convenient iteration via `for (file in ls())`, the format returned is a hash object:
//@ `{ 'file1':null, 'dir1/file2':null, ...}`.
//@ Returns array of files in the given path, or in current directory if no path provided.
function _ls(options, paths) {
options = parseOptions(options, {
'R': 'recursive',
@ -90,24 +88,30 @@ function _ls(options, paths) {
else if (typeof paths === 'string')
paths = [].slice.call(arguments, 1);
var hash = {};
var list = [];
function pushHash(file, query) {
// Conditionally pushes file to list - returns true if pushed, false otherwise
// (e.g. prevents hidden files to be included unless explicitly told so)
function pushFile(file, query) {
// hidden file?
if (path.basename(file)[0] === '.') {
// not explicitly asking for hidden files?
if (!options.all && !(path.basename(query)[0] === '.' && path.basename(query).length > 1))
return;
return false;
}
hash[file] = null;
if (platform === 'win')
file = file.replace(/\\/g, '/');
list.push(file);
return true;
}
paths.forEach(function(p) {
if (fs.existsSync(p)) {
// Simple file?
if (fs.statSync(p).isFile()) {
pushHash(p, p);
pushFile(p, p);
return; // continue
}
@ -115,14 +119,17 @@ function _ls(options, paths) {
if (fs.statSync(p).isDirectory()) {
// Iterate over p contents
fs.readdirSync(p).forEach(function(file) {
pushHash(file, p);
if (!pushFile(file, p))
return;
// Recursive
var oldDir = _pwd();
_cd('', p);
if (fs.statSync(file).isDirectory() && options.recursive)
hash = extend(hash, _ls('-R', file+'/*'));
_cd('', oldDir);
// Recursive?
if (options.recursive) {
var oldDir = _pwd();
_cd('', p);
if (fs.statSync(file).isDirectory())
list = list.concat(_ls('-R'+(options.all?'a':''), file+'/*'));
_cd('', oldDir);
}
});
return; // continue
}
@ -137,17 +144,20 @@ function _ls(options, paths) {
// Escape special regular expression chars
var regexp = basename.replace(/(\^|\$|\(|\)|\<|\>|\[|\]|\{|\}|\.|\+|\?)/g, '\\$1');
// Translates wildcard into regex
regexp = '^' + regexp.replace(/\*/g, '.*');
regexp = '^' + regexp.replace(/\*/g, '.*') + '$';
// Iterate over directory contents
fs.readdirSync(dirname).forEach(function(file) {
if (file.match(new RegExp(regexp))) {
pushHash(path.normalize(dirname+'/'+file), basename);
if (!pushFile(path.normalize(dirname+'/'+file), basename))
return;
// Recursive
var pp = dirname + '/' + file;
if (fs.statSync(pp).isDirectory() && options.recursive)
hash = extend(hash, _ls('-R', pp+'/*'));
}
// Recursive?
if (options.recursive) {
var pp = dirname + '/' + file;
if (fs.statSync(pp).isDirectory())
list = list.concat(_ls('-R'+(options.all?'a':''), pp+'/*'));
} // recursive
} // if file matches
}); // forEach
return;
}
@ -155,7 +165,7 @@ function _ls(options, paths) {
error('no such file or directory: ' + p, true);
});
return hash;
return list;
};
exports.ls = wrap('ls', _ls);
@ -168,16 +178,10 @@ exports.ls = wrap('ls', _ls);
//@ ```javascript
//@ find('src', 'lib');
//@ find(['src', 'lib']); // same as above
//@ for (file in find('.')) {
//@ if (!file.match(/\.js$/))
//@ continue;
//@ // all files at this point end in '.js'
//@ }
//@ find('.').filter(function(file) { return file.match(/\.js$/); });
//@ ```
//@
//@ Returns list of all files (however deep) in the given paths. For convenient iteration
//@ via `for (file in find(...))`, the format returned is a hash object:
//@ `{ 'file1':null, 'dir1/file2':null, ...}`.
//@ Returns array of all files (however deep) in the given paths.
//@
//@ The main difference from `ls('-R', path)` is that the resulting file names
//@ include the base directories, e.g. `lib/resources/file1` instead of just `file1`.
@ -189,21 +193,28 @@ function _find(options, paths) {
else if (typeof paths === 'string')
paths = [].slice.call(arguments, 1);
var hash = {};
var list = [];
function pushFile(file) {
if (platform === 'win')
file = file.replace(/\\/g, '/');
list.push(file);
}
// why not simply do ls('-R', paths)? because the output wouldn't give the base dirs
// to get the base dir in the output, we need instead ls('-R', 'dir/*') for every directory
paths.forEach(function(file){
hash[file] = null;
paths.forEach(function(file) {
pushFile(file);
if (fs.statSync(file).isDirectory()) {
for (subfile in _ls('-Ra', file+'/*'))
hash[subfile] = null;
_ls('-Ra', file+'/*').forEach(function(subfile) {
pushFile(subfile);
});
}
});
return hash;
return list;
}
exports.find = wrap('find', _find);
@ -347,9 +358,20 @@ function _rm(options, files) {
// Remove simple file
if (fs.statSync(file).isFile()) {
fs.unlinkSync(file);
// Do not check for file writing permissions
if (options.force) {
_unlinkSync(file);
return;
}
if (isWriteable(file))
_unlinkSync(file);
else
error('permission denied: '+file, true);
return;
}
} // simple file
// Path is an existing directory, but no -r flag given
if (fs.statSync(file).isDirectory() && !options.recursive) {
@ -359,7 +381,7 @@ function _rm(options, files) {
// Recursively remove existing directory
if (fs.statSync(file).isDirectory() && options.recursive) {
rmdirSyncRecursive(file);
rmdirSyncRecursive(file, options.force);
}
}); // forEach(file)
}; // rm
@ -582,7 +604,11 @@ function _to(options, file) {
if (!fs.existsSync( path.dirname(file) ))
error('no such file or directory: ' + path.dirname(file));
fs.writeFileSync(file, this.toString(), 'utf8');
try {
fs.writeFileSync(file, this.toString(), 'utf8');
} catch(e) {
error('could not write to file (code '+e.code+'): '+file, true);
}
};
// In the future, when Proxies are default, we can add methods like `.to()` to primitive strings.
// For now, this is a dummy function to bookmark places we need such strings
@ -793,7 +819,7 @@ function _exec(command, options, callback) {
}
options = extend({
silent: false,
silent: state.silent,
async: false
}, options);
@ -822,11 +848,52 @@ exports.exec = wrap('exec', _exec, {notUnix:true});
//@ Follows Python's [tempfile algorithm](http://docs.python.org/library/tempfile.html#tempfile.tempdir).
exports.tempdir = wrap('tempdir', tempDir);
//@
//@ #### error()
//@ Tests if error occurred in the last command. Returns `null` if no error occurred,
//@ otherwise returns string explaining the error
exports.error = function() {
return state.error;
}
//@
//@ #### silent([state])
//@ Example:
//@
//@ ```javascript
//@ var silentState = silent();
//@ silent(true);
//@ /* ... */
//@ silent(silentState); // restore old silent state
//@ ```
//@
//@ Suppresses all output if `state = true`. Returns state if no arguments given.
exports.silent = function(_state) {
if (typeof _state !== 'boolean')
return state.silent;
state.silent = _state;
}
//@
//@ ## Deprecated
//@
//@
//@ #### exists(path [, path ...])
//@ #### exists(path_array)
//@
//@ _This function is being deprecated. Use `test()` instead._
//@
//@ Returns true if all the given paths exist.
function _exists(options, paths) {
deprecate('exists', 'Use test() instead.');
if (!paths)
error('no paths given');
@ -844,32 +911,19 @@ function _exists(options, paths) {
};
exports.exists = wrap('exists', _exists);
//@
//@ #### error()
//@ Tests if error occurred in the last command. Returns `null` if no error occurred,
//@ otherwise returns string explaining the error
exports.error = function() {
return state.error;
}
//@
//@ #### verbose()
//@
//@ _This function is being deprecated. Use `silent(false) instead.`_
//@
//@ Enables all output (default)
exports.verbose = function() {
deprecate('verbose', 'Use silent(false) instead.');
state.silent = false;
}
//@
//@ #### silent()
//@ Suppresses all output, except for explict `echo()` calls
exports.silent = function() {
state.silent = true;
}
@ -889,6 +943,10 @@ function log() {
console.log.apply(this, arguments);
}
function deprecate(what, msg) {
console.log('*** ShellJS.'+what+': This function is deprecated.', msg);
}
function write(msg) {
if (!state.silent)
process.stdout.write(msg);
@ -984,10 +1042,22 @@ function copyFileSync(srcFile, destFile) {
var BUF_LENGTH = 64*1024,
buf = new Buffer(BUF_LENGTH),
fdr = fs.openSync(srcFile, 'r'),
fdw = fs.openSync(destFile, 'w'),
bytesRead = BUF_LENGTH,
pos = 0;
pos = 0,
fdr = null,
fdw = null;
try {
fdr = fs.openSync(srcFile, 'r');
} catch(e) {
error('copyFileSync: could not read src file ('+srcFile+')');
}
try {
fdw = fs.openSync(destFile, 'w');
} catch(e) {
error('copyFileSync: could not write to dest file (code='+e.code+'):'+destFile);
}
while (bytesRead === BUF_LENGTH) {
bytesRead = fs.readSync(fdr, buf, 0, BUF_LENGTH, pos);
@ -1050,28 +1120,42 @@ function cpdirSyncRecursive(sourceDir, destDir, opts) {
//
// Licensed under the MIT License
// http://www.opensource.org/licenses/mit-license.php
function rmdirSyncRecursive(dir) {
function rmdirSyncRecursive(dir, force) {
var files;
files = fs.readdirSync(dir);
// Loop through and delete everything in the sub-tree after checking it
for(var i = 0; i < files.length; i++) {
var currFile = fs.lstatSync(dir + "/" + files[i]);
var file = dir + "/" + files[i],
currFile = fs.lstatSync(file);
if(currFile.isDirectory()) // Recursive function back to the beginning
rmdirSyncRecursive(dir + "/" + files[i]);
if(currFile.isDirectory()) { // Recursive function back to the beginning
rmdirSyncRecursive(file, force);
}
else if(currFile.isSymbolicLink()) // Unlink symlinks
fs.unlinkSync(dir + "/" + files[i]);
else if(currFile.isSymbolicLink()) { // Unlink symlinks
if (force || isWriteable(file))
_unlinkSync(file);
}
else // Assume it's a file - perhaps a try/catch belongs here?
fs.unlinkSync(dir + "/" + files[i]);
if (force || isWriteable(file))
_unlinkSync(file);
}
// Now that we know everything in the sub-tree has been deleted, we can delete the main directory.
// Huzzah for the shopkeep.
return fs.rmdirSync(dir);
var result;
try {
result = fs.rmdirSync(dir);
} catch(e) {
if (e.code === 'ENOTEMPTY')
error('directory not empty: ' + dir, true);
}
return result;
}; // rmdirSyncRecursive
// Recursively creates 'dir'
@ -1118,7 +1202,7 @@ function writeableDir(dir) {
var testFile = dir+'/'+randomFileName();
try {
fs.writeFileSync(testFile, ' ');
fs.unlinkSync(testFile);
_unlinkSync(testFile);
return dir;
} catch (e) {
return false;
@ -1150,7 +1234,8 @@ function tempDir() {
// Wrapper around exec() to enable echoing output to console in real time
function execAsync(cmd, opts, callback) {
var output = '';
var output = '',
silent = 'silent' in opts ? opts.silent : state.silent;
var c = child.exec(cmd, {env: process.env}, function(err) {
if (callback)
@ -1159,13 +1244,13 @@ function execAsync(cmd, opts, callback) {
c.stdout.on('data', function(data) {
output += data;
if (!opts.silent)
if (!silent)
write(data);
});
c.stderr.on('data', function(data) {
output += data;
if (!opts.silent)
if (!silent)
write(data);
});
}
@ -1178,16 +1263,17 @@ function execAsync(cmd, opts, callback) {
function execSync(cmd, opts) {
var stdoutFile = path.resolve(tempDir()+'/'+randomFileName()),
codeFile = path.resolve(tempDir()+'/'+randomFileName()),
scriptFile = path.resolve(tempDir()+'/'+randomFileName());
scriptFile = path.resolve(tempDir()+'/'+randomFileName()),
sleepFile = path.resolve(tempDir()+'/'+randomFileName());
var options = extend({
silent: false
silent: state.silent
}, opts);
var previousStdoutContent = '';
// Echoes stdout changes from running process, if not silent
function updateStdout() {
if (state.silent || options.silent || !fs.existsSync(stdoutFile))
if (options.silent || !fs.existsSync(stdoutFile))
return;
var stdoutContent = fs.readFileSync(stdoutFile, 'utf8');
@ -1214,9 +1300,9 @@ function execSync(cmd, opts) {
fs.writeFileSync('"+escape(codeFile)+"', err ? err.code.toString() : '0'); \
});";
if (fs.existsSync(scriptFile)) fs.unlinkSync(scriptFile);
if (fs.existsSync(stdoutFile)) fs.unlinkSync(stdoutFile);
if (fs.existsSync(codeFile)) fs.unlinkSync(codeFile);
if (fs.existsSync(scriptFile)) _unlinkSync(scriptFile);
if (fs.existsSync(stdoutFile)) _unlinkSync(stdoutFile);
if (fs.existsSync(codeFile)) _unlinkSync(codeFile);
fs.writeFileSync(scriptFile, script);
child.exec('node '+scriptFile, {
@ -1225,8 +1311,11 @@ function execSync(cmd, opts) {
});
// The wait loop
while (!fs.existsSync(codeFile)) { updateStdout(); };
while (!fs.existsSync(stdoutFile)) { updateStdout(); };
// sleepFile is used as a dummy I/O op to mitigate unnecessary CPU usage
// (tried many I/O sync ops, writeFileSync() seems to be only one that is effective in reducing
// CPU usage, though apparently not so much on Windows)
while (!fs.existsSync(codeFile)) { updateStdout(); fs.writeFileSync(sleepFile, 'a'); };
while (!fs.existsSync(stdoutFile)) { updateStdout(); fs.writeFileSync(sleepFile, 'a'); };
// At this point codeFile exists, but it's not necessarily flushed yet.
// Keep reading it until it is.
@ -1236,9 +1325,10 @@ function execSync(cmd, opts) {
var stdout = fs.readFileSync(stdoutFile, 'utf8');
fs.unlinkSync(scriptFile);
fs.unlinkSync(stdoutFile);
fs.unlinkSync(codeFile);
_unlinkSync(scriptFile);
_unlinkSync(stdoutFile);
_unlinkSync(codeFile);
_unlinkSync(sleepFile);
// True if successful, false if not
var obj = {
@ -1257,8 +1347,9 @@ function expand(list) {
list.forEach(function(listEl) {
// Wildcard present?
if (listEl.search(/\*/) > -1) {
for (file in _ls('', listEl))
_ls('', listEl).forEach(function(file) {
expanded.push(file);
});
} else {
expanded.push(listEl);
}
@ -1290,3 +1381,33 @@ function extend(target) {
return target;
}
// Normalizes _unlinkSync() across platforms to match Unix behavior, i.e.
// file can be unlinked even if it's read-only, see joyent/node#3006
function _unlinkSync(file) {
try {
fs.unlinkSync(file);
} catch(e) {
// Try to override file permission
if (e.code === 'EPERM') {
fs.chmodSync(file, '0666');
fs.unlinkSync(file);
} else {
throw e;
}
}
}
// Hack to determine if file has write permissions for current user
// Avoids having to check user, group, etc, but it's probably slow
function isWriteable(file) {
var writePermission = true;
try {
var __fd = fs.openSync(file, 'a');
fs.closeSync(__fd);
} catch(e) {
writePermission = false;
}
return writePermission;
}

47
make.js
View File

@ -99,7 +99,7 @@ target.bundle = function() {
'jpx.js',
'bidi.js'];
if (!exists(BUILD_DIR))
if (!test('-d', BUILD_DIR))
mkdir(BUILD_DIR);
cd('src');
@ -142,10 +142,10 @@ target.pagesrepo = function() {
echo();
echo('### Creating fresh clone of gh-pages');
if (!exists(BUILD_DIR))
if (!test('-d', BUILD_DIR))
mkdir(BUILD_DIR);
if (!exists(GH_PAGES_DIR)) {
if (!test('-d', GH_PAGES_DIR)) {
echo();
echo('Cloning project repo...');
echo('(This operation can take a while, depending on network conditions)');
@ -277,10 +277,10 @@ target.firefox = function() {
// We don't need pdf.js anymore since its inlined
rm('-Rf', FIREFOX_BUILD_CONTENT_DIR + BUILD_DIR);
// Remove '.DS_Store' and other hidden files
for (file in find(FIREFOX_BUILD_DIR)) {
find(FIREFOX_BUILD_DIR).forEach(function(file) {
if (file.match(/^\./))
rm('-f', file);
}
});
// Update the build version number
sed('-i', /PDFJSSCRIPT_VERSION/, EXTENSION_VERSION, FIREFOX_BUILD_DIR + '/install.rdf');
@ -304,10 +304,10 @@ target.firefox = function() {
// List all files for mozilla-central
cd(FIREFOX_BUILD_DIR);
var extensionFiles = '';
for (file in find(FIREFOX_MC_EXTENSION_FILES)) {
find(FIREFOX_MC_EXTENSION_FILES).forEach(function(file){
if (test('-f', file))
extensionFiles += file+'\n';
}
});
extensionFiles.to('extension-files');
};
@ -357,7 +357,7 @@ target.chrome = function() {
//
target.test = function() {
target.browsertest();
target.unittest();
// target.unittest();
};
//
@ -371,14 +371,14 @@ target.browsertest = function() {
var PDF_TEST = env['PDF_TEST'] || 'test_manifest.json',
PDF_BROWSERS = env['PDF_BROWSERS'] || 'resources/browser_manifests/browser_manifest.json';
if (!exists('test/' + PDF_BROWSERS)) {
if (!test('-f', 'test/' + PDF_BROWSERS)) {
echo('Browser manifest file test/' + PDF_BROWSERS + ' does not exist.');
echo('Try copying one of the examples in test/resources/browser_manifests/');
exit(1);
}
cd('test');
exec(PYTHON_BIN + ' test.py --reftest --browserManifestFile=' + PDF_BROWSERS +
exec(PYTHON_BIN + ' -u test.py --reftest --browserManifestFile=' + PDF_BROWSERS +
' --manifestFile=' + PDF_TEST, {async: true});
};
@ -390,10 +390,37 @@ target.unittest = function() {
echo();
echo('### Running unit tests');
if (!which('make')) {
echo('make not found. Skipping unit tests...');
return;
}
cd('test/unit');
exec('make', {async: true});
};
//
// make makeref
//
target.makeref = function() {
cd(ROOT_DIR);
echo();
echo('### Creating reference images');
var PDF_TEST = env['PDF_TEST'] || 'test_manifest.json',
PDF_BROWSERS = env['PDF_BROWSERS'] || 'resources/browser_manifests/browser_manifest.json';
if (!test('-f', 'test/' + PDF_BROWSERS)) {
echo('Browser manifest file test/' + PDF_BROWSERS + ' does not exist.');
echo('Try copying one of the examples in test/resources/browser_manifests/');
exit(1);
}
cd('test');
exec(PYTHON_BIN + ' -u test.py --masterMode --noPrompts --browserManifestFile=' + PDF_BROWSERS,
{async: true});
};
///////////////////////////////////////////////////////////////////////////////////////////
//