Rewrite the parser

Summary:
Make our own parser that doesn't use jison, so that we can handle
funny TeX syntax, and to make it smaller.

Test Plan: Make sure the tests pass with the new parser.

Reviewers: alpert

Reviewed By: alpert

Differential Revision: http://phabricator.khanacademy.org/D3029

.gitignore

@@ -1,3 +1,2 @@
 build
 node_modules
-parser.js

Lexer.js

@@ -0,0 +1,88 @@
+// The main lexer class
+function Lexer(input) {
+    this._input = input;
+};
+
+// The result of a single lex
+function LexResult(type, text, position) {
+    this.type = type;
+    this.text = text;
+    this.position = position;
+}
+
+// "normal" types of tokens
+var normals = [
+    [/^[/|@."`0-9]/, 'textord'],
+    [/^[a-zA-Z]/, 'mathord'],
+    [/^[*+-]/, 'bin'],
+    [/^[=<>]/, 'rel'],
+    [/^[,;]/, 'punct'],
+    [/^\^/, '^'],
+    [/^_/, '_'],
+    [/^{/, '{'],
+    [/^}/, '}'],
+    [/^[(\[]/, 'open'],
+    [/^[)\]?!]/, 'close']
+];
+
+// Different functions
+var funcs = [
+    // Bin symbols
+    'cdot', 'pm', 'div',
+    // Rel symbols
+    'leq', 'geq', 'neq', 'nleq', 'ngeq',
+    // Open/close symbols
+    'lvert', 'rvert',
+    // Punct symbols
+    'colon',
+    // Spacing symbols
+    'qquad', 'quad', ' ', 'space', ',', ':', ';',
+    // Colors
+    'blue', 'orange', 'pink', 'red', 'green', 'gray', 'purple',
+    // Mathy functions
+    "arcsin", "arccos", "arctan", "arg", "cos", "cosh", "cot", "coth", "csc",
+    "deg", "dim", "exp", "hom", "ker", "lg", "ln", "log", "sec", "sin", "sinh",
+    "tan", "tanh",
+    // Other functions
+    'dfrac', 'llap', 'rlap'
+];
+// Build a regex to easily parse the functions
+var anyFunc = new RegExp("^\\\\(" + funcs.join("|") + ")(?![a-zA-Z])");
+
+// Lex a single token
+Lexer.prototype.lex = function(pos) {
+    var input = this._input.slice(pos);
+
+    // Get rid of whitespace
+    var whitespace = input.match(/^\s*/)[0];
+    pos += whitespace.length;
+    input = input.slice(whitespace.length);
+
+    // If there's no more input to parse, return an EOF token
+    if (input.length === 0) {
+        return new LexResult('EOF', null, pos);
+    }
+
+    var match;
+    if ((match = input.match(anyFunc))) {
+        // If we match one of the tokens, extract the type
+        return new LexResult(match[1], match[0], pos + match[0].length);
+    } else {
+        // Otherwise, we look through the normal token regexes and see if it's
+        // one of them.
+        for (var i = 0; i < normals.length; i++) {
+            var normal = normals[i];
+
+            if ((match = input.match(normal[0]))) {
+                // If it is, return it
+                return new LexResult(
+                    normal[1], match[0], pos + match[0].length);
+            }
+        }
+    }
+
+    // We didn't match any of the tokens, so throw an error.
+    throw "Unexpected character: '" + input[0] + "' at position " + this._pos;
+};
+
+module.exports = Lexer;

Makefile

@@ -1,8 +1,16 @@
 .PHONY: build copy serve clean
 build: build/katex.js
 
-build/katex.js: katex.js parser.jison lexer.js
+compress: build/katex.min.js
-	./node_modules/.bin/browserify $< --standalone katex -t ./jisonify > $@
+	@echo -n "Minified, gzipped size: "
+	@gzip -c $^ | wc -c
+
+build/katex.js: katex.js Parser.js Lexer.js
+	./node_modules/.bin/browserify $< --standalone katex > $@
+
+build/katex.min.js: build/katex.js
+	uglifyjs --mangle < $< > $@
+
 
 copy: build
 	cp build/katex.js ../exercises/utils/katex.js

Parser.js

@@ -0,0 +1,263 @@
+var Lexer = require("./Lexer");
+
+// Main Parser class
+function Parser(options) {
+    this.options = options;
+};
+
+// Returned by the Parser.parse... functions. Stores the current results and
+// the new lexer position.
+function ParseResult(result, newPosition) {
+    this.result = result;
+    this.position = newPosition;
+}
+
+// The resulting parse tree nodes of the parse tree.
+function ParseNode(type, value) {
+    this.type = type;
+    this.value = value;
+}
+
+// Checks a result to make sure it has the right type, and throws an
+// appropriate error otherwise.
+var expect = function(result, type) {
+    if (result.type !== type) {
+        throw "Failed parsing: Expected '" + type + "', got '" + result.type + "'";
+    }
+};
+
+// Main parsing function, which parses an entire input. Returns either a list
+// of parseNodes or null if the parse fails.
+Parser.prototype.parse = function(input) {
+    // Make a new lexer
+    this.lexer = new Lexer(input);
+
+    // Try to parse the input
+    var parse = this.parseInput(0);
+    return parse.result;
+};
+
+// Parses an entire input tree
+Parser.prototype.parseInput = function(pos) {
+    // Parse an expression
+    var expression = this.parseExpression(pos);
+    // If we succeeded, make sure there's an EOF at the end
+    var EOF = this.lexer.lex(expression.position);
+    expect(EOF, 'EOF');
+    return expression;
+};
+
+// Parses an "expression", which is a list of atoms
+Parser.prototype.parseExpression = function(pos) {
+    // Start with a list of nodes
+    var expression = [];
+    while (true) {
+        // Try to parse atoms
+        var parse = this.parseAtom(pos);
+        if (parse) {
+            // Copy them into the list
+            expression.push(parse.result);
+            pos = parse.position;
+        } else {
+            break;
+        }
+    }
+    return new ParseResult(expression, pos);
+};
+
+// Parses a superscript expression, like "^3"
+Parser.prototype.parseSuperscript = function(pos) {
+    // Try to parse a "^" character
+    var sup = this.lexer.lex(pos);
+    if (sup.type === "^") {
+        // If we got one, parse the corresponding group
+        var group = this.parseGroup(sup.position);
+        if (group) {
+            return group;
+        } else {
+            // Throw an error if we didn't find a group
+            throw "Parse error: Couldn't find group after '^'";
+        }
+    } else {
+        return null;
+    }
+};
+
+// Parses a subscript expression, like "_3"
+Parser.prototype.parseSubscript = function(pos) {
+    // Try to parse a "_" character
+    var sub = this.lexer.lex(pos);
+    if (sub.type === "_") {
+        // If we got one, parse the corresponding group
+        var group = this.parseGroup(sub.position);
+        if (group) {
+            return group;
+        } else {
+            // Throw an error if we didn't find a group
+            throw "Parse error: Couldn't find group after '_'";
+        }
+    } else {
+        return null;
+    }
+};
+
+// Parses an atom, which consists of a nucleus, and an optional superscript and
+// subscript
+Parser.prototype.parseAtom = function(pos) {
+    // Parse the nucleus
+    var nucleus = this.parseGroup(pos);
+    if (nucleus) {
+        // Now, we try to parse a subscript or a superscript. If one of those
+        // succeeds, we then try to parse the opposite one, and depending on
+        // whether that succeeds, we return the correct type.
+        var sup, sub;
+        if (sup = this.parseSuperscript(nucleus.position)) {
+            if (sub = this.parseSubscript(sup.position)) {
+                return new ParseResult(
+                    new ParseNode("supsub",
+                        {base: nucleus.result, sup: sup.result,
+                            sub: sub.result}),
+                    sub.position);
+            } else {
+                return new ParseResult(
+                    new ParseNode("sup",
+                        {base: nucleus.result, sup: sup.result}),
+                    sup.position);
+            }
+        } else if (sub = this.parseSubscript(nucleus.position)) {
+            if (sup = this.parseSuperscript(sub.position)) {
+                return new ParseResult(
+                    new ParseNode("supsub",
+                        {base: nucleus.result, sup: sup.result,
+                            sub: sub.result}),
+                    sup.position);
+            } else {
+                return new ParseResult(
+                    new ParseNode("sub",
+                        {base: nucleus.result, sub: sub.result}),
+                    sub.position);
+            }
+        } else {
+            return nucleus;
+        }
+    } else {
+        return null;
+    }
+}
+
+// Parses a group, which is either a single nucleus (like "x") or an expression
+// in braces (like "{x+y}")
+Parser.prototype.parseGroup = function(pos) {
+    var start = this.lexer.lex(pos);
+    // Try to parse an open brace
+    if (start.type === "{") {
+        // If we get a brace, parse an expression
+        var expression = this.parseExpression(start.position);
+        // Make sure we get a close brace
+        var closeBrace = this.lexer.lex(expression.position);
+        expect(closeBrace, "}");
+        return new ParseResult(
+            new ParseNode("ordgroup", expression.result),
+            closeBrace.position);
+    } else {
+        // Otherwise, just return a nucleus
+        return this.parseNucleus(pos);
+    }
+};
+
+// Tests whether an element is in a list
+function contains(list, elem) {
+    return list.indexOf(elem) !== -1;
+}
+
+// A list of 1-argument color functions
+var colorFuncs = [
+    "blue", "orange", "pink", "red", "green", "gray", "purple"
+];
+
+// A map of elements that don't have arguments, and should simply be placed
+// into a group depending on their type. The keys are the groups that items can
+// be placed in, and the values are lists of element types that should be
+// placed in those groups.
+//
+// For example, if the lexer returns something of type "colon", we should
+// return a node of type "punct"
+var copyFuncs = {
+    "textord": ["textord"],
+    "mathord": ["mathord"],
+    "bin": ["bin", "pm", "div", "cdot"],
+    "open": ["open", "lvert"],
+    "close": ["close", "rvert"],
+    "rel": ["rel", "leq", "geq", "neq", "nleq", "ngeq"],
+    "spacing": ["qquad", "quad", "space", " ", ",", ":", ";"],
+    "punct": ["punct", "colon"],
+    "namedfn": ["arcsin", "arccos", "arctan", "arg", "cos", "cosh", "cot",
+        "coth", "csc", "deg", "dim", "exp", "hom", "ker", "lg", "ln", "log",
+        "sec", "sin", "sinh", "tan", "tanh"]
+};
+
+// Build a list of all of the different functions in the copyFuncs list, to
+// quickly check if the function should be interpreted by the map.
+var funcToType = {};
+for (var type in copyFuncs) {
+    for (var i = 0; i < copyFuncs[type].length; i++) {
+        var func = copyFuncs[type][i];
+        funcToType[func] = type;
+    }
+}
+
+// Parses a "nucleus", which is either a single token from the tokenizer or a
+// function and its arguments
+Parser.prototype.parseNucleus = function(pos) {
+    var nucleus = this.lexer.lex(pos);
+
+    if (contains(colorFuncs, nucleus.type)) {
+        // If this is a color function, parse its argument and return
+        var group = this.parseGroup(nucleus.position);
+        if (group) {
+            return new ParseResult(
+                new ParseNode("color",
+                    {color: nucleus.type, value: group.result}),
+                group.position);
+        } else {
+            throw "Parse error: Expected group after '" + nucleus.text + "'";
+        }
+    } else if (nucleus.type === "llap" || nucleus.type === "rlap") {
+        // If this is an llap or rlap, parse its argument and return
+        var group = this.parseGroup(nucleus.position);
+        if (group) {
+            return new ParseResult(
+                new ParseNode(nucleus.type, nucleus.text),
+                group.position);
+        } else {
+            throw "Parse error: Expected group after '" + nucleus.text + "'";
+        }
+    } else if (nucleus.type === "dfrac") {
+        // If this is a dfrac, parse its two arguments and return
+        var numer = this.parseGroup(nucleus.position);
+        if (numer) {
+            var denom = this.parseGroup(numer.position);
+            if (denom) {
+                return new ParseResult(
+                    new ParseNode("dfrac",
+                        {numer: numer.result, denom: denom.result}),
+                    denom.position);
+            } else {
+                throw "Parse error: Expected denominator after '\\dfrac'";
+            }
+        } else {
+            throw "Parse error: Expected numerator after '\\dfrac'"
+        }
+    } else if (funcToType[nucleus.type]) {
+        // Otherwise if this is a no-argument function, find the type it
+        // corresponds to in the map and return
+        return new ParseResult(
+            new ParseNode(funcToType[nucleus.type], nucleus.text),
+            nucleus.position);
+    } else {
+        // Otherwise, we couldn't parse it
+        return null;
+    }
+};
+
+module.exports = Parser;

jisonify.js

@@ -1,31 +0,0 @@
-var ebnfParser = require("ebnf-parser");
-var jison = require("jison");
-var through = require("through");
-
-module.exports = function(file) {
-    if (!(/\.jison$/).test(file)) {
-        return through();
-    }
-
-    var data = '';
-    return through(write, end);
-
-    function write(buf) {
-        data += buf;
-    }
-
-    function end() {
-        try {
-            var grammar = ebnfParser.parse(data);
-            var parser = new jison.Parser(grammar);
-            var js = parser.generate({moduleType: "js"});
-            js += "\nmodule.exports = parser;";
-
-            this.queue(js);
-            this.queue(null);
-        } catch (e) {
-            // TODO(alpert): Does this do anything? (Is it useful?)
-            this.emit("error", e);
-        }
-    }
-};

katex.js

@@ -12,12 +12,10 @@ var makeSpan = function(className, children) {
     var span = document.createElement("span");
     span.className = className || "";
 
-    if (_.isArray(children)) {
+    if (children) {
-        _.each(children, function(v) {
+        for (var i = 0; i < children.length; i++) {
-            span.appendChild(v);
+            span.appendChild(children[i]);
-        });
+        }
-    } else if (children) {
-        span.appendChild(children);
     }
 
     return span;
@@ -25,46 +23,46 @@ var makeSpan = function(className, children) {
 
 var buildGroup = function(group, prev) {
     if (group.type === "mathord") {
-        return makeSpan("mord", mathit(group.value));
+        return makeSpan("mord", [mathit(group.value)]);
     } else if (group.type === "textord") {
-        return makeSpan("mord", textit(group.value));
+        return makeSpan("mord", [textit(group.value)]);
     } else if (group.type === "bin") {
         var className = "mbin";
         if (prev == null || _.contains(["bin", "open", "rel"], prev.type)) {
             group.type = "ord";
             className = "mord";
         }
-        return makeSpan(className, textit(group.value));
+        return makeSpan(className, [textit(group.value)]);
     } else if (group.type === "rel") {
-        return makeSpan("mrel", textit(group.value));
+        return makeSpan("mrel", [textit(group.value)]);
     } else if (group.type === "sup") {
-        var sup = makeSpan("msup", buildExpression(group.value.sup));
+        var sup = makeSpan("msup", [buildGroup(group.value.sup)]);
-        return makeSpan("mord", buildExpression(group.value.base).concat(sup));
+        return makeSpan("mord", [buildGroup(group.value.base), sup]);
     } else if (group.type === "sub") {
-        var sub = makeSpan("msub", buildExpression(group.value.sub));
+        var sub = makeSpan("msub", [buildGroup(group.value.sub)]);
-        return makeSpan("mord", buildExpression(group.value.base).concat(sub));
+        return makeSpan("mord", [buildGroup(group.value.base), sub]);
     } else if (group.type === "supsub") {
-        var sup = makeSpan("msup", buildExpression(group.value.sup));
+        var sup = makeSpan("msup", [buildGroup(group.value.sup)]);
-        var sub = makeSpan("msub", buildExpression(group.value.sub));
+        var sub = makeSpan("msub", [buildGroup(group.value.sub)]);
 
         var supsub = makeSpan("msupsub", [sup, sub]);
 
-        return makeSpan("mord", buildExpression(group.value.base).concat(supsub));
+        return makeSpan("mord", [buildGroup(group.value.base), supsub]);
     } else if (group.type === "open") {
-        return makeSpan("mopen", textit(group.value));
+        return makeSpan("mopen", [textit(group.value)]);
     } else if (group.type === "close") {
-        return makeSpan("mclose", textit(group.value));
+        return makeSpan("mclose", [textit(group.value)]);
     } else if (group.type === "dfrac") {
-        var numer = makeSpan("mfracnum", makeSpan("", buildExpression(group.value.numer)));
+        var numer = makeSpan("mfracnum", [makeSpan("", [buildGroup(group.value.numer)])]);
-        var mid = makeSpan("mfracmid", makeSpan());
+        var mid = makeSpan("mfracmid", [makeSpan()]);
-        var denom = makeSpan("mfracden", buildExpression(group.value.denom));
+        var denom = makeSpan("mfracden", [buildGroup(group.value.denom)]);
 
         return makeSpan("minner mfrac", [numer, mid, denom]);
     } else if (group.type === "color") {
-        return makeSpan("mord " + group.value.color, buildExpression(group.value.value));
+        return makeSpan("mord " + group.value.color, [buildGroup(group.value.value)]);
     } else if (group.type === "spacing") {
         if (group.value === "\\ " || group.value === "\\space") {
-            return makeSpan("mord mspace", textit(group.value));
+            return makeSpan("mord mspace", [textit(group.value)]);
         } else {
             var spacingClassMap = {
                 "\\qquad": "qquad",
@@ -78,18 +76,18 @@ var buildGroup = function(group, prev) {
         }
     } else if (group.type === "llap") {
         var inner = makeSpan("", buildExpression(group.value));
-        return makeSpan("llap", inner);
+        return makeSpan("llap", [inner]);
     } else if (group.type === "rlap") {
         var inner = makeSpan("", buildExpression(group.value));
-        return makeSpan("rlap", inner);
+        return makeSpan("rlap", [inner]);
     } else if (group.type === "punct") {
-        return makeSpan("mpunct", textit(group.value));
+        return makeSpan("mpunct", [textit(group.value)]);
     } else if (group.type === "ordgroup") {
         return makeSpan("mord", buildExpression(group.value));
     } else if (group.type === "namedfn") {
-        return makeSpan("mop", textit(group.value.slice(1)));
+        return makeSpan("mop", [textit(group.value.slice(1))]);
     } else {
-        console.log("Unknown type:", group.type);
+        throw "Lex error: Got group of unknown type: '" + group.type + "'";
     }
 };
 
@@ -120,7 +118,7 @@ var textit = function(value) {
 };
 
 var mathit = function(value) {
-    return makeSpan("mathit", textit(value));
+    return makeSpan("mathit", [textit(value)]);
 };
 
 var clearNode = function(node) {
@@ -133,10 +131,16 @@ var clearNode = function(node) {
 
 var process = function(toParse, baseElem) {
     var tree = parseTree(toParse);
-    clearNode(baseElem);
+    if (tree) {
-    _.each(buildExpression(tree), function(elem) {
+        clearNode(baseElem);
-        baseElem.appendChild(elem);
+        var expression = buildExpression(tree);
-    });
+        for (var i = 0; i < expression.length; i++) {
+            baseElem.appendChild(expression[i]);
+        }
+        return true;
+    } else {
+        return false;
+    }
 };
 
 module.exports = {

lexer.js

@@ -1,99 +0,0 @@
-function Lexer() {
-};
-
-var normals = [
-    [/^[/|@."`0-9]/, 'TEXTORD'],
-    [/^[a-zA-Z]/, 'MATHORD'],
-    [/^[*+-]/, 'BIN'],
-    [/^[=<>]/, 'REL'],
-    [/^[,;]/, 'PUNCT'],
-    [/^\^/, '^'],
-    [/^_/, '_'],
-    [/^{/, '{'],
-    [/^}/, '}'],
-    [/^[(\[]/, 'OPEN'],
-    [/^[)\]?!]/, 'CLOSE']
-];
-
-var funcs = [
-    // Bin symbols
-    'cdot', 'pm', 'div',
-    // Rel symbols
-    'leq', 'geq', 'neq', 'nleq', 'ngeq',
-    // Open/close symbols
-    'lvert', 'rvert',
-    // Punct symbols
-    'colon',
-    // Spacing symbols
-    'qquad', 'quad', ' ', 'space', ',', ':', ';',
-    // Colors
-    'blue', 'orange', 'pink', 'red', 'green', 'gray', 'purple',
-    // Mathy functions
-    "arcsin", "arccos", "arctan", "arg", "cos", "cosh", "cot", "coth", "csc",
-    "deg", "dim", "exp", "hom", "ker", "lg", "ln", "log", "sec", "sin", "sinh",
-    "tan", "tanh",
-    // Other functions
-    'dfrac', 'llap', 'rlap'
-];
-var anyFunc = new RegExp("^\\\\(" + funcs.join("|") + ")(?![a-zA-Z])");
-
-Lexer.prototype.doMatch = function(match) {
-    this.yytext = match;
-    this.yyleng = match.length;
-
-    this.yylloc.first_column = this._pos;
-    this.yylloc.last_column = this._pos + match.length;
-
-    this._pos += match.length;
-    this._input = this._input.slice(match.length);
-};
-
-Lexer.prototype.lex = function() {
-    // Get rid of whitespace
-    var whitespace = this._input.match(/^\s*/)[0];
-    this._pos += whitespace.length;
-    this._input = this._input.slice(whitespace.length);
-
-    if (this._input.length === 0) {
-        return 'EOF';
-    }
-
-    var match;
-
-    if ((match = this._input.match(anyFunc))) {
-        this.doMatch(match[0]);
-
-        if (match[1] === " ") {
-            return "space";
-        }
-        return match[1];
-    } else {
-        for (var i = 0; i < normals.length; i++) {
-            var normal = normals[i];
-
-            if ((match = this._input.match(normal[0]))) {
-                this.doMatch(match[0]);
-                return normal[1];
-            }
-        }
-    }
-
-    throw "Unexpected character: '" + this._input[0] + "' at position " + this._pos;
-};
-
-Lexer.prototype.setInput = function(input) {
-    this._input = input;
-    this._pos = 0;
-
-    this.yyleng = 0;
-    this.yytext = "";
-    this.yylineno = 0;
-    this.yylloc = {
-        first_line: 1,
-        first_column: 0,
-        last_line: 1,
-        last_column: 0
-    };
-};
-
-module.exports = new Lexer();

parseTree.js

@@ -1,10 +1,5 @@
-var parser = require("./parser.jison");
+var Parser = require("./Parser");
-parser.lexer = require("./lexer");
+var parser = new Parser({verbose: true});
-parser.yy = {
-    parseError: function(str) {
-        throw new Error(str);
-    }
-};
 
 var parseTree = function(toParse) {
     return parser.parse(toParse);

parser.jison

@@ -1,161 +0,0 @@
-/* description: Parses end executes mathematical expressions. */
-
-/* operator associations and precedence */
-
-%left '^'
-%left '_'
-%left 'ORD'
-%left 'BIN'
-%left SUPSUB
-
-%start expression
-
-%% /* language grammar */
-
-expression
-    : ex 'EOF'
-        {return $1;}
-    ;
-
-ex
-    :
-        {$$ = [];}
-    | group ex
-        {$$ = $1.concat($2);}
-    | group '^' group ex
-        {$$ = [{type: 'sup', value: {base: $1, sup: $3}}].concat($4);}
-    | group '_' group ex
-        {$$ = [{type: 'sub', value: {base: $1, sub: $3}}].concat($4);}
-    | group '^' group '_' group ex %prec SUPSUB
-        {$$ = [{type: 'supsub', value: {base: $1, sup: $3, sub: $5}}].concat($6);}
-    | group '_' group '^' group ex %prec SUPSUB
-        {$$ = [{type: 'supsub', value: {base: $1, sup: $5, sub: $3}}].concat($6);}
-    ;
-
-group
-    : atom
-        {$$ = $1;}
-    | '{' ex '}'
-        {$$ = [{type: 'ordgroup', value: $2}];}
-    | func
-        {$$ = $1;}
-    ;
-
-func
-    : 'cdot'
-        {$$ = [{type: 'bin', value: yytext}];}
-    | 'pm'
-        {$$ = [{type: 'bin', value: yytext}];}
-    | 'div'
-        {$$ = [{type: 'bin', value: yytext}];}
-    | 'lvert'
-        {$$ = [{type: 'open', value: yytext}];}
-    | 'rvert'
-        {$$ = [{type: 'close', value: yytext}];}
-    | 'leq'
-        {$$ = [{type: 'rel', value: yytext}];}
-    | 'geq'
-        {$$ = [{type: 'rel', value: yytext}];}
-    | 'neq'
-        {$$ = [{type: 'rel', value: yytext}];}
-    | 'nleq'
-        {$$ = [{type: 'rel', value: yytext}];}
-    | 'ngeq'
-        {$$ = [{type: 'rel', value: yytext}];}
-    | 'qquad'
-        {$$ = [{type: 'spacing', value: yytext}];}
-    | 'quad'
-        {$$ = [{type: 'spacing', value: yytext}];}
-    | 'space'
-        {$$ = [{type: 'spacing', value: yytext}];}
-    | ','
-        {$$ = [{type: 'spacing', value: yytext}];}
-    | ':'
-        {$$ = [{type: 'spacing', value: yytext}];}
-    | ';'
-        {$$ = [{type: 'spacing', value: yytext}];}
-    | 'colon'
-        {$$ = [{type: 'punct', value: yytext}];}
-    | 'blue' group
-        {$$ = [{type: 'color', value: {color: 'blue', value: $2}}];}
-    | 'orange' group
-        {$$ = [{type: 'color', value: {color: 'orange', value: $2}}];}
-    | 'pink' group
-        {$$ = [{type: 'color', value: {color: 'pink', value: $2}}];}
-    | 'red' group
-        {$$ = [{type: 'color', value: {color: 'red', value: $2}}];}
-    | 'green' group
-        {$$ = [{type: 'color', value: {color: 'green', value: $2}}];}
-    | 'gray' group
-        {$$ = [{type: 'color', value: {color: 'gray', value: $2}}];}
-    | 'purple' group
-        {$$ = [{type: 'color', value: {color: 'purple', value: $2}}];}
-    | 'dfrac' group group
-        {$$ = [{type: 'dfrac', value: {numer: $2, denom: $3}}];}
-    | 'llap' group
-        {$$ = [{type: 'llap', value: $2}];}
-    | 'rlap' group
-        {$$ = [{type: 'rlap', value: $2}];}
-    | 'arcsin'
-        {$$ = [{type: 'namedfn', value: yytext}];}
-    | 'arccos'
-        {$$ = [{type: 'namedfn', value: yytext}];}
-    | 'arctan'
-        {$$ = [{type: 'namedfn', value: yytext}];}
-    | 'arg'
-        {$$ = [{type: 'namedfn', value: yytext}];}
-    | 'cos'
-        {$$ = [{type: 'namedfn', value: yytext}];}
-    | 'cosh'
-        {$$ = [{type: 'namedfn', value: yytext}];}
-    | 'cot'
-        {$$ = [{type: 'namedfn', value: yytext}];}
-    | 'coth'
-        {$$ = [{type: 'namedfn', value: yytext}];}
-    | 'csc'
-        {$$ = [{type: 'namedfn', value: yytext}];}
-    | 'deg'
-        {$$ = [{type: 'namedfn', value: yytext}];}
-    | 'dim'
-        {$$ = [{type: 'namedfn', value: yytext}];}
-    | 'exp'
-        {$$ = [{type: 'namedfn', value: yytext}];}
-    | 'hom'
-        {$$ = [{type: 'namedfn', value: yytext}];}
-    | 'ker'
-        {$$ = [{type: 'namedfn', value: yytext}];}
-    | 'lg'
-        {$$ = [{type: 'namedfn', value: yytext}];}
-    | 'ln'
-        {$$ = [{type: 'namedfn', value: yytext}];}
-    | 'log'
-        {$$ = [{type: 'namedfn', value: yytext}];}
-    | 'sec'
-        {$$ = [{type: 'namedfn', value: yytext}];}
-    | 'sin'
-        {$$ = [{type: 'namedfn', value: yytext}];}
-    | 'sinh'
-        {$$ = [{type: 'namedfn', value: yytext}];}
-    | 'tan'
-        {$$ = [{type: 'namedfn', value: yytext}];}
-    | 'tanh'
-        {$$ = [{type: 'namedfn', value: yytext}];}
-    ;
-
-atom
-    : 'TEXTORD'
-        {$$ = [{type: 'textord', value: yytext}];}
-    | 'MATHORD'
-        {$$ = [{type: 'mathord', value: yytext}];}
-    | 'BIN'
-        {$$ = [{type: 'bin', value: yytext}];}
-    | 'REL'
-        {$$ = [{type: 'rel', value: yytext}];}
-    | 'PUNCT'
-        {$$ = [{type: 'punct', value: yytext}];}
-    | 'OPEN'
-        {$$ = [{type: 'open', value: yytext}];}
-    | 'CLOSE'
-        {$$ = [{type: 'close', value: yytext}];}
-    ;
-

server.js

@@ -3,8 +3,6 @@ var path = require("path");
 var browserify = require("browserify");
 var express = require("express");
 
-var jisonify = require("./jisonify");
-
 var app = express();
 
 app.use(express.logger());
@@ -12,7 +10,6 @@ app.use(express.logger());
 app.get("/katex.js", function(req, res, next) {
     var b = browserify();
     b.add("./katex");
-    b.transform(jisonify);
 
     var stream = b.bundle({standalone: "katex"});
 
@@ -28,7 +25,6 @@ app.get("/katex.js", function(req, res, next) {
 app.get("/test/katex-tests.js", function(req, res, next) {
     var b = browserify();
     b.add("./test/katex-tests");
-    b.transform(jisonify);
 
     var stream = b.bundle({});
 

static/index.html

@@ -9,7 +9,7 @@
         <link href="main.css" rel="stylesheet" type="text/css">
     </head>
     <body>
-        <input type="text" value="2x^2 + 3" id="input" />
+    <input type="text" value="\blue\dfrac{2(y-z)}{3} \div \orange{\arctan x^{4/3}}" id="input" />
         <div id="math" class="mathmathmath"></div>
     </body>
 </html>

test/katex-tests.js

@@ -207,6 +207,18 @@ describe("A subscript and superscript parser", function() {
         expect(parseA).toEqual(parseB);
     });
 
+    it("should not parse x^x^x", function() {
+        expect(function() {
+            parseTree("x^x^x");
+        }).toThrow();
+    });
+
+    it("should not parse x_x_x", function() {
+        expect(function() {
+            parseTruee("x_x_x");
+        }).toThrow();
+    });
+
     it("should work correctly with {}s", function() {
         expect(function() {
             parseTree("x^{2+3}");