From bf52ff156d9d2f3d03160f4c3ffc856fcfb829f8 Mon Sep 17 00:00:00 2001 From: Yury Delendik Date: Tue, 10 May 2016 18:05:29 -0500 Subject: [PATCH] Adds esprima-based preprocessor. --- .jshintignore | 1 + .../fixtures_esprima/blocks-expected.js | 10 + external/builder/fixtures_esprima/blocks.js | 19 ++ .../fixtures_esprima/comments-expected.js | 28 ++ external/builder/fixtures_esprima/comments.js | 26 ++ .../fixtures_esprima/constants-expected.js | 15 + .../builder/fixtures_esprima/constants.js | 15 + .../fixtures_esprima/evals-expected.js | 13 + external/builder/fixtures_esprima/evals.js | 10 + external/builder/fixtures_esprima/evals.json | 1 + .../builder/fixtures_esprima/ifs-expected.js | 17 ++ external/builder/fixtures_esprima/ifs.js | 25 ++ external/builder/preprocessor2.js | 266 ++++++++++++++++++ external/builder/test2.js | 54 ++++ package.json | 2 + 15 files changed, 502 insertions(+) create mode 100644 external/builder/fixtures_esprima/blocks-expected.js create mode 100644 external/builder/fixtures_esprima/blocks.js create mode 100644 external/builder/fixtures_esprima/comments-expected.js create mode 100644 external/builder/fixtures_esprima/comments.js create mode 100644 external/builder/fixtures_esprima/constants-expected.js create mode 100644 external/builder/fixtures_esprima/constants.js create mode 100644 external/builder/fixtures_esprima/evals-expected.js create mode 100644 external/builder/fixtures_esprima/evals.js create mode 100644 external/builder/fixtures_esprima/evals.json create mode 100644 external/builder/fixtures_esprima/ifs-expected.js create mode 100644 external/builder/fixtures_esprima/ifs.js create mode 100644 external/builder/preprocessor2.js create mode 100644 external/builder/test2.js diff --git a/.jshintignore b/.jshintignore index 55fba5294..46b0d7b8c 100644 --- a/.jshintignore +++ b/.jshintignore @@ -9,6 +9,7 @@ external/jpgjs/ external/jasmine/ external/cmapscompress/ external/importL10n/ +external/builder/fixtures_esprima/ shared/ test/tmp/ test/features/ diff --git a/external/builder/fixtures_esprima/blocks-expected.js b/external/builder/fixtures_esprima/blocks-expected.js new file mode 100644 index 000000000..22e7b7434 --- /dev/null +++ b/external/builder/fixtures_esprima/blocks-expected.js @@ -0,0 +1,10 @@ +function test() { + "test"; + "1"; + "2"; + "3"; + if ("test") { + "5"; + } + "4"; +} diff --git a/external/builder/fixtures_esprima/blocks.js b/external/builder/fixtures_esprima/blocks.js new file mode 100644 index 000000000..a7f3c360d --- /dev/null +++ b/external/builder/fixtures_esprima/blocks.js @@ -0,0 +1,19 @@ +function test() { + {;} + ; + "test"; + { + "1"; + if (true) { + "2"; + } + ; + { + "3"; + if ("test") { + "5"; + } + } + "4"; + } +} diff --git a/external/builder/fixtures_esprima/comments-expected.js b/external/builder/fixtures_esprima/comments-expected.js new file mode 100644 index 000000000..270742445 --- /dev/null +++ b/external/builder/fixtures_esprima/comments-expected.js @@ -0,0 +1,28 @@ +function f1() { + /* head */ + "1"; + /* mid */ + "2"; +} +/* tail */ +function f2() { + // head + "1"; + // mid + "2"; +} +// tail +function f2() { + if ("1") { + // begin block + "1"; + } + "2"; + // trailing + if (/* s */ + "3") + /*e*/ + { + "4"; + } +} diff --git a/external/builder/fixtures_esprima/comments.js b/external/builder/fixtures_esprima/comments.js new file mode 100644 index 000000000..5ab6d4c0d --- /dev/null +++ b/external/builder/fixtures_esprima/comments.js @@ -0,0 +1,26 @@ +/* globals f0 */ +function f1() { + /* head */ + "1"; + /* mid */ + "2"; + /* tail */ +} + +function f2() { + // head + "1"; + // mid + "2"; + // tail +} + +function f2() { + if ("1") { // begin block + "1"; + } + "2"; // trailing + if (/* s */"3"/*e*/) { + "4"; + } +} diff --git a/external/builder/fixtures_esprima/constants-expected.js b/external/builder/fixtures_esprima/constants-expected.js new file mode 100644 index 000000000..4539481e5 --- /dev/null +++ b/external/builder/fixtures_esprima/constants-expected.js @@ -0,0 +1,15 @@ +var a = true; +var b = false; +var c = '1'; +var d = false; +var e = true; +var f = '0'; +var g = '1'; +var h = '0'; +var i = true; +var j = false; +var k = false; +var l = true; +var m = '1' === true; +var n = false; +var o = true; diff --git a/external/builder/fixtures_esprima/constants.js b/external/builder/fixtures_esprima/constants.js new file mode 100644 index 000000000..db2ff6e67 --- /dev/null +++ b/external/builder/fixtures_esprima/constants.js @@ -0,0 +1,15 @@ +var a = true; +var b = false; +var c = true && '1'; +var d = false && '0'; +var e = true || '1'; +var f = false || '0'; +var g = true ? '1' : '0'; +var h = false ? '1' : '0'; +var i = 'test' === 'test'; +var j = 'test' !== 'test'; +var k = 'test' === 'test2'; +var l = 'test' !== 'test2'; +var m = '1' === true; +var n = !true; +var o = !false; diff --git a/external/builder/fixtures_esprima/evals-expected.js b/external/builder/fixtures_esprima/evals-expected.js new file mode 100644 index 000000000..414de051c --- /dev/null +++ b/external/builder/fixtures_esprima/evals-expected.js @@ -0,0 +1,13 @@ +var a = false; +var b = true; +var c = true; +var d = false; +var e = true; +var f = 'text'; +var g = { + "obj": { "i": 1 }, + "j": 2 +}; +var h = { 'test': 'test' }; +var i = '0'; +var j = { "i": 1 }; diff --git a/external/builder/fixtures_esprima/evals.js b/external/builder/fixtures_esprima/evals.js new file mode 100644 index 000000000..c0649ae83 --- /dev/null +++ b/external/builder/fixtures_esprima/evals.js @@ -0,0 +1,10 @@ +var a = typeof PDFJSDev === 'undefined'; +var b = typeof PDFJSDev !== 'undefined'; +var c = PDFJSDev.test('TRUE'); +var d = PDFJSDev.test('FALSE'); +var e = PDFJSDev.eval('TRUE'); +var f = PDFJSDev.eval('TEXT'); +var g = PDFJSDev.eval('OBJ'); +var h = PDFJSDev.json('$ROOT/external/builder/fixtures_esprima/evals.json'); +var i = typeof PDFJSDev === 'undefined' ? PDFJSDev.eval('FALSE') : '0'; +var j = typeof PDFJSDev !== 'undefined' ? PDFJSDev.eval('OBJ.obj') : '0'; diff --git a/external/builder/fixtures_esprima/evals.json b/external/builder/fixtures_esprima/evals.json new file mode 100644 index 000000000..020d36b74 --- /dev/null +++ b/external/builder/fixtures_esprima/evals.json @@ -0,0 +1 @@ +{ 'test': 'test' } \ No newline at end of file diff --git a/external/builder/fixtures_esprima/ifs-expected.js b/external/builder/fixtures_esprima/ifs-expected.js new file mode 100644 index 000000000..12599c619 --- /dev/null +++ b/external/builder/fixtures_esprima/ifs-expected.js @@ -0,0 +1,17 @@ +if ('test') { + "1"; +} +{ + "1"; +} +{ + "1"; +} +; +{ + "2"; +} +; +if ('1') { + "1"; +} diff --git a/external/builder/fixtures_esprima/ifs.js b/external/builder/fixtures_esprima/ifs.js new file mode 100644 index 000000000..a2f3d217f --- /dev/null +++ b/external/builder/fixtures_esprima/ifs.js @@ -0,0 +1,25 @@ +if ('test') { + "1"; +} +if (true) { + "1"; +} +if (true) { + "1"; +} else { + "2"; +} +if (false) { + "1"; +} +if (false) { + "1"; +} else { + "2"; +} +if (true && false) { + "1"; +} +if (true && false || '1') { + "1"; +} diff --git a/external/builder/preprocessor2.js b/external/builder/preprocessor2.js new file mode 100644 index 000000000..02eb689a3 --- /dev/null +++ b/external/builder/preprocessor2.js @@ -0,0 +1,266 @@ +/* jshint node:true */ + +'use strict'; + +var esprima = require('esprima'); +var escodegen = require('escodegen'); +var vm = require('vm'); +var fs = require('fs'); +var path = require('path'); + +var PDFJS_PREPROCESSOR_NAME = 'PDFJSDev'; +var ROOT_PREFIX = '$ROOT/'; + +function isLiteral(obj, value) { + return obj.type === 'Literal' && obj.value === value; +} + +function isPDFJSPreprocessor(obj) { + return obj.type === 'Identifier' && + obj.name === PDFJS_PREPROCESSOR_NAME; +} + +function evalWithDefines(code, defines, loc) { + if (!code || !code.trim()) { + throw new Error('No JavaScript expression given'); + } + return vm.runInNewContext(code, defines, {displayErrors: false}); +} + +function handlePreprocessorAction(ctx, actionName, args, loc) { + try { + var arg; + switch (actionName) { + case 'test': + arg = args[0]; + if (!arg || arg.type !== 'Literal' || + typeof arg.value !== 'string') { + throw new Error('No code for testing is given'); + } + var isTrue = !!evalWithDefines(arg.value, ctx.defines); + return {type: 'Literal', value: isTrue, loc: loc}; + case 'eval': + arg = args[0]; + if (!arg || arg.type !== 'Literal' || + typeof arg.value !== 'string') { + throw new Error('No code for eval is given'); + } + var result = evalWithDefines(arg.value, ctx.defines); + if (typeof result === 'boolean' || typeof result === 'string' || + typeof result === 'number') { + return {type: 'Literal', value: result, loc: loc}; + } + if (typeof result === 'object') { + var parsedObj = esprima.parse('(' + JSON.stringify(result) + ')'); + parsedObj.body[0].expression.loc = loc; + return parsedObj.body[0].expression; + } + break; + case 'json': + arg = args[0]; + if (!arg || arg.type !== 'Literal' || + typeof arg.value !== 'string') { + throw new Error('Path to JSON is not provided'); + } + var jsonPath = arg.value; + if (jsonPath.indexOf(ROOT_PREFIX) === 0) { + jsonPath = path.join(ctx.rootPath, + jsonPath.substring(ROOT_PREFIX.length)); + } + var jsonContent = fs.readFileSync(jsonPath).toString(); + var parsedJSON = esprima.parse('(' +jsonContent + ')'); + parsedJSON.body[0].expression.loc = loc; + return parsedJSON.body[0].expression; + } + throw new Error('Unsupported action'); + } catch (e) { + throw new Error('Could not process ' + PDFJS_PREPROCESSOR_NAME + '.' + + actionName + ' at ' + JSON.stringify(loc) + '\n' + + e.name + ': ' + e.message); + } +} + +function postprocessNode(ctx, node) { + switch (node.type) { + case 'IfStatement': + if (isLiteral(node.test, true)) { + // if (true) stmt1; => stmt1 + return node.consequent; + } else if (isLiteral(node.test, false)) { + // if (false) stmt1; else stmt2; => stmt2 + return node.alternate || {type: 'EmptyStatement', loc: node.loc}; + } + break; + case 'ConditionalExpression': + if (isLiteral(node.test, true)) { + // true ? stmt1 : stmt2 => stmt1 + return node.consequent; + } else if (isLiteral(node.test, false)) { + // false ? stmt1 : stmt2 => stmt2 + return node.alternate; + } + break; + case 'UnaryExpression': + if (node.operator === 'typeof' && + isPDFJSPreprocessor(node.argument)) { + // typeof PDFJSDev => 'object' + return {type: 'Literal', value: 'object', loc: node.loc}; + } + if (node.operator === '!' && + node.argument.type === 'Literal' && + typeof node.argument.value === 'boolean') { + // !true => false, !false => true + return {type: 'Literal', value: !node.argument.value, loc: node.loc}; + } + break; + case 'LogicalExpression': + switch (node.operator) { + case '&&': + if (isLiteral(node.left, true)) { + return node.right; + } + if (isLiteral(node.left, false)) { + return node.left; + } + break; + case '||': + if (isLiteral(node.left, true)) { + return node.left; + } + if (isLiteral(node.left, false)) { + return node.right; + } + break; + } + break; + case 'BinaryExpression': + switch (node.operator) { + case '==': + case '===': + case '!=': + case '!==': + if (node.left.type === 'Literal' && + node.right.type === 'Literal' && + typeof node.left.value === typeof node.right.value) { + // folding two literals == and != check + switch (typeof node.left.value) { + case 'string': + case 'boolean': + case 'number': + var equal = node.left.value === node.right.value; + return { + type: 'Literal', + value: (node.operator[0] === '=') === equal, + loc: node.loc + }; + } + } + break; + } + break; + case 'CallExpression': + if (node.callee.type === 'MemberExpression' && + isPDFJSPreprocessor(node.callee.object) && + node.callee.property.type === 'Identifier') { + // PDFJSDev.xxxx(arg1, arg2, ...) => tranform + var action = node.callee.property.name; + return handlePreprocessorAction(ctx, action, + node.arguments, node.loc); + } + break; + case 'BlockStatement': + var subExpressionIndex = 0; + while (subExpressionIndex < node.body.length) { + if (node.body[subExpressionIndex].type === 'EmptyStatement') { + // Removing empty statements from the blocks. + node.body.splice(subExpressionIndex, 1); + continue; + } + if (node.body[subExpressionIndex].type === 'BlockStatement') { + // Block statements inside a block are moved to the parent one. + var subChildren = node.body[subExpressionIndex].body; + Array.prototype.splice.apply(node.body, + [subExpressionIndex, 1].concat(subChildren)); + subExpressionIndex += subChildren.length; + continue; + } + subExpressionIndex++; + } + break; + } + return node; +} + +function fixComments(ctx, node) { + if (!ctx.saveComments) { + return; + } + // Fixes double comments in the escodegen output. + delete node.trailingComments; + // Removes jshint and other service comments. + if (node.leadingComments) { + var i = 0; + while (i < node.leadingComments.length) { + var type = node.leadingComments[i].type; + var value = node.leadingComments[i].value; + if (type === 'Block' && + /^\s*(globals|jshint|falls through|umdutils)\b/.test(value)) { + node.leadingComments.splice(i, 1); + continue; + } + i++; + } + } +} + +function traverseTree(ctx, node) { + // generic node processing + for (var i in node) { + var child = node[i]; + if (typeof child === 'object' && child !== null && child.type) { + var result = traverseTree(ctx, child); + if (result !== child) { + node[i] = result; + } + } else if (Array.isArray(child)) { + child.forEach(function (childItem, index) { + if (typeof childItem === 'object' && childItem !== null && + childItem.type) { + var result = traverseTree(ctx, childItem); + if (result !== childItem) { + child[index] = result; + } + } + }); + } + } + + node = postprocessNode(ctx, node) || node; + + fixComments(ctx, node); + return node; +} + +function preprocessPDFJSCode(ctx, code) { + var saveComments = !!ctx.saveComments; + var format = ctx.format || { + indent: { + style: ' ', + adjustMultilineComment: saveComments, + } + }; + var parseComment = { + loc: true, + attachComment: saveComments + }; + var codegenOptions = { + format: format, + comment: saveComments, + parse: esprima.parse + }; + var syntax = esprima.parse(code, parseComment); + traverseTree(ctx, syntax); + return escodegen.generate(syntax, codegenOptions); +} + +exports.preprocessPDFJSCode = preprocessPDFJSCode; diff --git a/external/builder/test2.js b/external/builder/test2.js new file mode 100644 index 000000000..f29b434e5 --- /dev/null +++ b/external/builder/test2.js @@ -0,0 +1,54 @@ +/* jshint node:true */ +/* globals cat, cd, echo, ls */ +'use strict'; + +require('shelljs/make'); + +var p2 = require('./preprocessor2.js'); +var fs = require('fs'); + +var errors = 0; + +cd(__dirname); +cd('fixtures_esprima'); +ls('*-expected.*').forEach(function(expectationFilename) { + var inFilename = expectationFilename.replace('-expected', ''); + var expectation = cat(expectationFilename).trim() + .replace(/__filename/g, fs.realpathSync(inFilename)); + var input = fs.readFileSync(inFilename).toString(); + + var defines = { + TRUE: true, + FALSE: false, + OBJ: {obj: {i: 1}, j: 2}, + TEXT: 'text' + }; + var ctx = { + defines: defines, + rootPath: __dirname + '/../..', + saveComments: true + }; + var out; + try { + out = p2.preprocessPDFJSCode(ctx, input); + } catch (e) { + out = ('Error: ' + e.message).replace(/^/gm, '//'); + } + if (out !== expectation) { + echo('Assertion failed for ' + inFilename); + echo('--------------------------------------------------'); + echo('EXPECTED:'); + echo(expectation); + echo('--------------------------------------------------'); + echo('ACTUAL'); + echo(out); + echo('--------------------------------------------------'); + echo(); + } +}); + +if (errors) { + echo('Found ' + errors + ' expectation failures.'); +} else { + echo('All tests completed without errors.'); +} diff --git a/package.json b/package.json index bea67b9f1..458713d52 100644 --- a/package.json +++ b/package.json @@ -2,6 +2,8 @@ "name": "pdf.js", "version": "0.8.0", "devDependencies": { + "escodegen": "^1.8.0", + "esprima": "^2.7.2", "gulp": "^3.9.1", "gulp-util": "^3.0.7", "gulp-zip": "^3.2.0",