Allow macro definitions in settings (#493)

* Introduce MacroExpander

The job of the MacroExpander is turning a stream of possibly expandable
tokens, as obtained from the Lexer, into a stream of non-expandable tokens
(in KaTeX, even though they may well be expandable in TeX) which can be
processed by the Parser.  The challenge here is that we don't have
mode-specific lexer implementations any more, so we need to do everything on
the token level, including reassembly of sizes and colors.

* Make macros available in development server

Now one can specify macro definitions like \foo=bar as part of the query
string and use these macros in the formula being typeset.

* Add tests for macro expansions

* Handle end of input in special groups

This avoids an infinite loop if input ends prematurely.

* Simplify parseSpecialGroup

The parseSpecialGroup methos now returns a single token spanning the whole
special group, and leaves matching that string against a suitable regular
expression to whoever is calling the method.  Suggested by @cbreeden.

* Incorporate review suggestions

Add improvements suggested by Kevin Barabash during review.

* Input range sanity checks

Ensure that both tokens of a token range come from the same lexer,
and that the range has a non-negative length.

* Improved wording of two comments

test/katex-spec.js

@@ -57,8 +57,39 @@ var getParsed = function(expr, settings) {
     return parseTree(expr, usedSettings);
 };
 
+var stripPositions = function(expr) {
+    if (typeof expr !== "object" || expr === null) {
+        return expr;
+    }
+    if (expr.lexer && typeof expr.start === "number") {
+        delete expr.lexer;
+        delete expr.start;
+        delete expr.end;
+    }
+    Object.keys(expr).forEach(function(key) {
+        stripPositions(expr[key]);
+    });
+    return expr;
+};
+
+var parseAndSetResult = function(expr, result, settings) {
+    try {
+        return parseTree(expr, settings || defaultSettings);
+    } catch (e) {
+        result.pass = false;
+        if (e instanceof ParseError) {
+            result.message = "'" + expr + "' failed " +
+                "parsing with error: " + e.message;
+        } else {
+            result.message = "'" + expr + "' failed " +
+                "parsing with unknown error: " + e.message;
+        }
+    }
+};
+
 beforeEach(function() {
     jasmine.addMatchers({
+
         toParse: function() {
             return {
                 compare: function(actual, settings) {
@@ -68,20 +99,7 @@ beforeEach(function() {
                         pass: true,
                         message: "'" + actual + "' succeeded parsing",
                     };
-
-                    try {
-                        parseTree(actual, usedSettings);
-                    } catch (e) {
-                        result.pass = false;
-                        if (e instanceof ParseError) {
-                            result.message = "'" + actual + "' failed " +
-                                "parsing with error: " + e.message;
-                        } else {
-                            result.message = "'" + actual + "' failed " +
-                                "parsing with unknown error: " + e.message;
-                        }
-                    }
-
+                    parseAndSetResult(actual, result, usedSettings);
                     return result;
                 },
             };
@@ -145,6 +163,36 @@ beforeEach(function() {
                 },
             };
         },
+
+        toParseLike: function(util, baton) {
+            return {
+                compare: function(actual, expected) {
+                    var result = {
+                        pass: true,
+                        message: "Parse trees of '" + actual +
+                            "' and '" + expected + "' are equivalent",
+                    };
+
+                    var actualTree = parseAndSetResult(actual, result);
+                    if (!actualTree) {
+                        return result;
+                    }
+                    var expectedTree = parseAndSetResult(expected, result);
+                    if (!expectedTree) {
+                        return result;
+                    }
+                    stripPositions(actualTree);
+                    stripPositions(expectedTree);
+                    if (!util.equals(actualTree, expectedTree, baton)) {
+                        result.pass = false;
+                        result.message = "Parse trees of '" + actual +
+                            "' and '" + expected + "' are not equivalent";
+                    }
+                    return result;
+                },
+            };
+        },
+
     });
 });
 
@@ -154,8 +202,8 @@ describe("A parser", function() {
     });
 
     it("should ignore whitespace", function() {
-        var parseA = getParsed("    x    y    ");
-        var parseB = getParsed("xy");
+        var parseA = stripPositions(getParsed("    x    y    "));
+        var parseB = stripPositions(getParsed("xy"));
         expect(parseA).toEqual(parseB);
     });
 });
@@ -340,8 +388,8 @@ describe("A subscript and superscript parser", function() {
     });
 
     it("should produce the same thing regardless of order", function() {
-        var parseA = getParsed("x^2_3");
-        var parseB = getParsed("x_3^2");
+        var parseA = stripPositions(getParsed("x^2_3"));
+        var parseB = stripPositions(getParsed("x_3^2"));
 
         expect(parseA).toEqual(parseB);
     });
@@ -623,6 +671,13 @@ describe("An over parser", function() {
         expect(parse.value.denom).toBeDefined();
     });
 
+    it("should handle \\textstyle correctly", function() {
+        expect("\\textstyle 1 \\over 2")
+            .toParseLike("\\frac{\\textstyle 1}{2}");
+        expect("{\\textstyle 1} \\over 2")
+            .toParseLike("\\frac{\\textstyle 1}{2}");
+    });
+
     it("should handle nested factions", function() {
         var nestedOverExpression = "{1 \\over 2} \\over 3";
         var parse = getParsed(nestedOverExpression)[0];
@@ -1523,7 +1578,7 @@ describe("A markup generator", function() {
 
 describe("A parse tree generator", function() {
     it("generates a tree", function() {
-        var tree = katex.__parse("\\sigma^2");
+        var tree = stripPositions(katex.__parse("\\sigma^2"));
         expect(JSON.stringify(tree)).toEqual(JSON.stringify([
             {
                 "type": "supsub",
@@ -1802,3 +1857,24 @@ describe("The symbol table integraty", function() {
             .toEqual(getBuilt("\\left\\lt\\frac{1}{x}\\right\\gt"));
     });
 });
+
+describe("A macro expander", function() {
+
+    var compareParseTree = function(actual, expected, macros) {
+        var settings = new Settings({macros: macros});
+        actual = stripPositions(parseTree(actual, settings));
+        expected = stripPositions(parseTree(expected, defaultSettings));
+        expect(actual).toEqual(expected);
+    };
+
+    it("should produce individual tokens", function() {
+        compareParseTree("e^\\foo", "e^1 23", {"\\foo": "123"});
+    });
+
+    it("should allow for multiple expansion", function() {
+        compareParseTree("1\\foo2", "1aa2", {
+            "\\foo": "\\bar\\bar",
+            "\\bar": "a",
+        });
+    });
+});