AST - the only true tool for building JavaScript

AST (Abstract Syntax Tree) The only true tool for building JavaScript

Source maps Epic win in debugging

Source maps – epic win in debugging

Builders Epic fail in debugging

Builders – epic fail in debugging umdify: // UMD definition output += '(function(root, factory) {n'; output += ' if (typeof define === "function" && define.amd) {n'; output += ' define([' + depNames.join(', ') + '], factory);n'; output += ' } else if (typeof exports === "object") {n'; output += ' module.exports = factory(require);n'; …

Builders – epic fail in debugging grunt-amd-wrap: var srcText = grunt.file.read(file.src[0]); var destText = amdWrap(srcText);

Builders – epic fail in debugging gulp-concat: buffer.push(file.contents); … var joinedContents = Buffer.concat(buffer);

Builders – epic fail in debugging universal-transformer: function transform(srcText) { return 'var answer = 42;'; }

Your code is not a string It has a soul

Your code has a soul // Life, Universe, and Everything var answer = 6 * 7;

Your code has a soul // Life, Universe, and Everything var answer = 6 * 7; '// Life, Universe and Everythingnvar answer = 6 * 7;'

Your code has a soul // Life, Universe, and Everything var answer = 6 * 7; [ { type: "Keyword", value: "var" }, { type: "Identifier", value: "answer" }, { type: "Punctuator", value: "=" }, { type: "Numeric", value: "6" }, { type: "Punctuator", value: "*" }, { type: "Numeric", value: "7" }, { type: "Punctuator", value: ";" } ]

Your code has a soul [ { type: "Keyword", value: "var" }, { type: "Identifier", value: "answer" }, { type: "Punctuator", value: "=" }, { type: "Numeric", value: "6" }, { type: "Punctuator", value: "*" }, { type: "Numeric", value: "7" }, { type: "Punctuator", value: ";" } ] { type: "Program", body: [{ type: "VariableDeclaration", declarations: [{ type: "VariableDeclarator", id: {type: "Identifier", name: "answer"}, init: { type: "BinaryExpression", operator: "*", left: {type: "Literal", value: 6}, right: {type: "Literal", value: 7} } }], kind: "var" }] }

Your code has a soul { type: "Program", body: [{ type: "VariableDeclaration", declarations: [{ type: "VariableDeclarator", id: {type: "Identifier", name: "answer"}, init: { type: "BinaryExpression", operator: "*", left: {type: "Literal", value: 6}, right: {type: "Literal", value: 7} } }], kind: "var" }] } Program VariableDeclaration VariableDeclarator Identifier(“answer”) BinaryExpression(*) Literal(6) Literal(7)

Your code has a soul // Life, Universe, and Everything var answer = 6 * 7; Program VariableDeclaration VariableDeclarator Identifier(“answer”) BinaryExpression(*) Literal(6) Literal(7)

Code tools How can we work with code AST?

Parsing • JavaScript • SpiderMonkey: Reflect.parse – Mozilla's Parser API • Esprima – most popular ECMAScript parser in JS • Acorn – faster alternative ECMAScript parser in JS • UglifyJS – has own parser with custom AST format • Traceur – has ES6 parser that can be used separately as well • … (as a lot of language tools do) … • CoffeeScript • CoffeeScriptRedux – rewrite of CS compiler that internally uses CS AST with conversion to JS AST • JSX • esprima-fb – Facebook's fork of Esprima Harmony branch • jsx-esprima – es* tools based JSX to JS AST transpiler

Parsing acorn.parse('var answer = 6 * 7;', {locations: true}); // In each node. loc: { start: { line: 2, column: 0 }, end: { line: 2, column: 19 } }

Linting

Querying var found; estraverse.traverse(tree, { enter: function (node) { if (node.type === 'Identifier' && node.name[0] === '_') { found = node; return estraverse.VisitorOption.Break; } } })

Querying require('grasp-equery') .query('if ($cond) return $yes; else return $no;', ast)

Querying require('grasp-squery') .query('if[then=return][else=return]', ast)

Constructing { type: "Program", body: [{ type: "VariableDeclaration", declarations: [{ type: "VariableDeclarator", id: {type: "Identifier", name: "answer"}, init: { type: "BinaryExpression", operator: "*", left: {type: "Literal", value: 6}, right: {type: "Literal", value: 7} } }], kind: "var" }] }

Constructing var b = require('ast-types').builders; b.variableDeclaration('var', [ b.variableDeclarator( b.identifier('answer'), b.binaryExpression( '*', b.literal(6), b.literal(7) ) ) ]);

Constructing estemplate('var <%= varName %> = 6 * 7;', { varName: {type: 'Identifier', name: 'answer'} });

Transforming var counter = 0, map = Object.create(null); result = estraverse.replace(tree, { enter: function (node) { if (node.type === 'Identifier' && node.name[0] === '_') node.name = map[node.name] || (map[node.name] = '$' + counter++); } });

Transforming var Renamer = recast.Visitor.extend({ init: function () { this.counter = 0; this.map = Object.create(null); }, getId: function (name) { return this.map[name] || (this.map[name] = '$' + this.counter++); }, visitIdentifier: function (node) { if (node.name[0] === '_') node.name = this.getId(node.name); } });

Generating var output = escodegen.generate(ast, { sourceMap: true, sourceMapWithCode: true }); fs.writeFileSync('out.js', output.code); fs.writeFileSync('out.js.map', output.map.toString());

Building with AST What can we improve here?

File-based builders (Grunt) Parsing code Transforming AST Generating code Reading file Writing file Plugin

Streaming builders (Gulp, Browserify) Transforming AST Generating code Parsing code Reading file Writing file Plugin

Transforming AST Next logical step Reading file Writing file Parsing code Generating code

aster - AST-based code builder https://github.com/asterjs/aster

Already available aster src watch dest runner aster-parse js esnext jsx coffee aster-changed aster-concat aster-equery aster-squery aster-rename-ids aster-traverse aster-uglify aster-umd npm keyword: aster-plugin

Sample build script aster.watch(['src/**/*.js', 'src/**/*.coffee', 'src/**/*.jsx']) .throttle(500) .map(changed( src => src.map(equery({ 'if ($cond) return $expr1; else return $expr2;': 'return <%= cond %> ? <%= expr1 %> : <%= expr2 %>' })) )) .map(concat('built.js')) .map(umd({exports: 'superLib'})) .map(aster.dest('dist', {sourceMap: true})) .subscribe(aster.runner);

Plugins – reactive AST transformers module.exports = source => { source = source || 'built.js'; return files => files .flatMap(file => Rx.Observable.fromArray(file.program.body)) .toArray() .map(body => ({ type: 'File', program: {type: 'Program', body}, loc: {source} })); };

Integration with generic build systems grunt.initConfig({ aster: { options: { equery: { 'if ($cond) return $expr1; else return $expr2;': 'return <%= cond %> ? <%= expr1 %> : <%= expr2 %>' }, concat: 'built.js', umd: {exports: 'superLib'}, dest: {sourceMap: true} }, files: { 'dist': ['src/**/*.js', 'src/**/*.coffee', 'src/**/*.jsx'] } } });

Integration with generic build systems gulp.src(['src/**/*.js', 'src/**/*.coffee', 'src/**/*.jsx']) .pipe(aster(src => src .map(equery({ 'if ($cond) return $expr1; else return $expr2;': 'return <%= cond %> ? <%= expr1 %> : <%= expr2 %>' }) .map(concat('built.js')) .map(umd({exports: 'superLib'})) )) .pipe(gulp.dest('dist'))

Questions? https://github.com/asterjs @RReverser me@rreverser.com

https://twitter.com	105
http://lanyrd.com	56
http://rreverser.com	40
http://www.slideee.com	1
http://feedly.com	1
http://tweetedtimes.com	1

AST - the only true tool for building JavaScript

by Ingvar Stepanyan , Web Developer at Microsoft Ukraine

on Jul 06, 2014

Statistics

Views

Actions

6 Embeds 205

Accessibility

Categories

Upload Details

Usage Rights

Report content

AST - the only true tool for building JavaScript Presentation Transcript