feat(macro): improve argument parsing (#2085)

* Improve macro argument parsing

* Make \above a primitive command

* Fix screenshotter data

* Normalize argument where necessary

* Improve argument location info

* Update comments

* Minor refactor

* Modularize group parsers

* Allow braced and blank size argument

for non-strict mode and \genfrac, respectively.

* Minor refactor & update comments

* Remove raw option in parseStringGroup

* Update tests

* Fix { delimited parameter

* Update tests

* Update tests

* Normalize argument in \genfrac

* Update tests

* Consume space before scanning an optional argument

* Fix \\, \newline, and \cr behavior

* Fix flow error

* Update comments

* Remove unnecessary mode switching

Parser mode affects neither fetch nor consume.

* Allow single (active) character macro

* Add function property `primitive`

* Set \mathchoice and \*style primitive

* Separate size-related improvements out to #2139

* Fix flow error

* Update screenshots

* Update demo example

* Add a migration guide

* Fix capitalization

* Make a primitive function unexpandable

* Update screenshots

* Update screenshots

* Revert "Document \def doesn't support delimiters (#2288) (#2289)"

This reverts commit f96fba6f7f.

* Update comments, errors, and tests

* Update screenshots

docs/migration.md

@@ -0,0 +1,24 @@
+---
+id: migration
+title: Migration Guide
+---
+
+As of KaTeX 1.0, we've changed how MacroExpander and Parser work in order to close
+some gaps between KaTeX and LaTeX and therefore there may be breaking changes.
+
+## Macro arguments
+Tokens will not be expanded while parsing a macro argument. For example, `\frac\foo\foo`,
+where the `\foo` is defined as `12`, will be parsed as `\frac{12}{12}`, not
+`\frac{1}{2}12`. <!--To expand the argument before parsing, `\expandafter` can
+be used` like `\expandafter\frac\foo\foo`.-->
+
+## `\def`
+`\def` no longer accepts a control sequence enclosed in braces. For example,
+`\def{\foo}{}` no longer works and should be changed to `\def\foo{}`.
+
+It also no longer accepts replacement text not enclosed in braces. For example,
+`\def\foo1` no longer works and should be changed to `\def\foo{1}`.
+
+## `\newline` and `\cr`
+`\newline` and `\cr` no longer takes an optional size argument. To specify vertical
+spacing, `\\` should be used.

docs/supported.md

@@ -315,7 +315,6 @@ Direct Input: $∀ ∴ ∁ ∵ ∃ ∣ ∈ ∉ ∋ ⊂ ⊃ ∧ ∨ ↦ → ←
 Macros can also be defined in the KaTeX [rendering options](options.md).
 
 Macros accept up to nine arguments: #1, #2, etc.
-Delimiters (such as `\def\add#1+#2{#1\oplus#2}`) are not currently supported.
 
 `\gdef`, `\xdef`, `\global\def`, `\global\edef`, `\global\let`, and `\global\futurelet` will persist between math expressions.
 

src/MacroExpander.js

@@ -13,7 +13,7 @@ import ParseError from "./ParseError";
 import Namespace from "./Namespace";
 import builtinMacros from "./macros";
 
-import type {MacroContextInterface, MacroDefinition, MacroExpansion}
+import type {MacroContextInterface, MacroDefinition, MacroExpansion, MacroArg}
     from "./macros";
 import type Settings from "./Settings";
 
@@ -108,6 +108,32 @@ export default class MacroExpander implements MacroContextInterface {
         this.stack.push(...tokens);
     }
 
+    /**
+     * Find an macro argument without expanding tokens and append the array of
+     * tokens to the token stack. Uses Token as a container for the result.
+     */
+    scanArgument(isOptional: boolean): ?Token {
+        let start;
+        let end;
+        let tokens;
+        if (isOptional) {
+            this.consumeSpaces(); // \@ifnextchar gobbles any space following it
+            if (this.future().text !== "[") {
+                return null;
+            }
+            start = this.popToken(); // don't include [ in tokens
+            ({tokens, end} = this.consumeArg(["]"]));
+        } else {
+            ({tokens, start, end} = this.consumeArg());
+        }
+
+        // indicate the end of an argument
+        this.pushToken(new Token("EOF", end.loc));
+
+        this.pushTokens(tokens);
+        return start.range(end, "");
+    }
+
     /**
      * Consume all following space tokens, without expansion.
      */
@@ -123,40 +149,91 @@ export default class MacroExpander implements MacroContextInterface {
     }
 
     /**
-     * Consume the specified number of arguments from the token stream,
+     * Consume an argument from the token stream, and return the resulting array
-     * and return the resulting array of arguments.
+     * of tokens and start/end token.
      */
-    consumeArgs(numArgs: number): Token[][] {
+    consumeArg(delims?: ?string[]): MacroArg {
-        const args: Token[][] = [];
+        // The argument for a delimited parameter is the shortest (possibly
-        // obtain arguments, either single token or balanced {…} group
+        // empty) sequence of tokens with properly nested {...} groups that is
-        for (let i = 0; i < numArgs; ++i) {
+        // followed ... by this particular list of non-parameter tokens.
-            this.consumeSpaces();  // ignore spaces before each argument
+        // The argument for an undelimited parameter is the next nonblank
-            const startOfArg = this.popToken();
+        // token, unless that token is ‘{’, when the argument will be the
-            if (startOfArg.text === "{") {
+        // entire {...} group that follows.
-                const arg: Token[] = [];
+        const tokens: Token[] = [];
-                let depth = 1;
+        const isDelimited = delims && delims.length > 0;
-                while (depth !== 0) {
+        if (!isDelimited) {
-                    const tok = this.popToken();
+            // Ignore spaces between arguments.  As the TeXbook says:
-                    arg.push(tok);
+            // "After you have said ‘\def\row#1#2{...}’, you are allowed to
-                    if (tok.text === "{") {
+            //  put spaces between the arguments (e.g., ‘\row x n’), because
-                        ++depth;
+            //  TeX doesn’t use single spaces as undelimited arguments."
-                    } else if (tok.text === "}") {
+            this.consumeSpaces();
-                        --depth;
+        }
-                    } else if (tok.text === "EOF") {
+        const start = this.future();
-                        throw new ParseError(
+        let tok;
-                            "End of input in macro argument",
+        let depth = 0;
-                            startOfArg);
+        let match = 0;
-                    }
+        do {
+            tok = this.popToken();
+            tokens.push(tok);
+            if (tok.text === "{") {
+                ++depth;
+            } else if (tok.text === "}") {
+                --depth;
+                if (depth === -1) {
+                    throw new ParseError("Extra }", tok);
                 }
-                arg.pop(); // remove last }
+            } else if (tok.text === "EOF") {
-                arg.reverse(); // like above, to fit in with stack order
+                throw new ParseError("Unexpected end of input in a macro argument" +
-                args[i] = arg;
+                    ", expected '" + (delims && isDelimited ? delims[match] : "}") +
-            } else if (startOfArg.text === "EOF") {
+                    "'", tok);
-                throw new ParseError(
-                    "End of input expecting macro argument");
-            } else {
-                args[i] = [startOfArg];
             }
+            if (delims && isDelimited) {
+                if ((depth === 0 || (depth === 1 && delims[match] === "{")) &&
+                    tok.text === delims[match]) {
+                    ++match;
+                    if (match === delims.length) {
+                        // don't include delims in tokens
+                        tokens.splice(-match, match);
+                        break;
+                    }
+                } else {
+                    match = 0;
+                }
+            }
+        } while (depth !== 0 || isDelimited);
+        // If the argument found ... has the form ‘{<nested tokens>}’,
+        // ... the outermost braces enclosing the argument are removed
+        if (start.text === "{" && tokens[tokens.length - 1].text === "}") {
+            tokens.pop();
+            tokens.shift();
+        }
+        tokens.reverse(); // to fit in with stack order
+        return {tokens, start, end: tok};
+    }
+
+    /**
+     * Consume the specified number of (delimited) arguments from the token
+     * stream and return the resulting array of arguments.
+     */
+    consumeArgs(numArgs: number, delimiters?: string[][]): Token[][] {
+        if (delimiters) {
+            if (delimiters.length !== numArgs + 1) {
+                throw new ParseError(
+                    "The length of delimiters doesn't match the number of args!");
+            }
+            const delims = delimiters[0];
+            for (let i = 0; i < delims.length; i++) {
+                const tok = this.popToken();
+                if (delims[i] !== tok.text) {
+                    throw new ParseError(
+                        "Use of the macro doesn't match its definition", tok);
+                }
+            }
+        }
+
+        const args: Token[][] = [];
+        for (let i = 0; i < numArgs; i++) {
+            args.push(this.consumeArg(delimiters && delimiters[i + 1]).tokens);
         }
         return args;
     }
@@ -177,10 +254,6 @@ export default class MacroExpander implements MacroContextInterface {
      *
      * Used to implement `expandAfterFuture` and `expandNextToken`.
      *
-     * At the moment, macro expansion doesn't handle delimited macros,
-     * i.e. things like those defined by \def\foo#1\end{…}.
-     * See the TeX book page 202ff. for details on how those should behave.
-     *
      * If expandableOnly, only expandable tokens are expanded and
      * an undefined control sequence results in an error.
      */
@@ -202,8 +275,8 @@ export default class MacroExpander implements MacroContextInterface {
                 "need to increase maxExpand setting");
         }
         let tokens = expansion.tokens;
+        const args = this.consumeArgs(expansion.numArgs, expansion.delimiters);
         if (expansion.numArgs) {
-            const args = this.consumeArgs(expansion.numArgs);
             // paste arguments in place of the placeholders
             tokens = tokens.slice(); // make a shallow copy
             for (let i = tokens.length - 1; i >= 0; --i) {
@@ -368,7 +441,6 @@ export default class MacroExpander implements MacroContextInterface {
         const macro = this.macros.get(name);
         return macro != null ? typeof macro === "string"
                 || typeof macro === "function" || !macro.unexpandable
-            // TODO(ylem): #2085
+            : functions.hasOwnProperty(name) && !functions[name].primitive;
-            : functions.hasOwnProperty(name)/* && !functions[name].primitive*/;
     }
 }

src/Parser.js

@@ -145,12 +145,6 @@ export default class Parser {
 
     static endOfExpression = ["}", "\\endgroup", "\\end", "\\right", "&"];
 
-    static endOfGroup = {
-        "[": "]",
-        "{": "}",
-        "\\begingroup": "\\endgroup",
-    }
-
     /**
      * Parses an "expression", which is a list of atoms.
      *
@@ -265,9 +259,8 @@ export default class Parser {
         const symbolToken = this.fetch();
         const symbol = symbolToken.text;
         this.consume();
-        const group = this.parseGroup(name, false, Parser.SUPSUB_GREEDINESS,
+        this.consumeSpaces(); // ignore spaces before sup/subscript argument
-            undefined, undefined, true);
+        const group = this.parseGroup(name, Parser.SUPSUB_GREEDINESS);
-                                 // ignore spaces before sup/subscript argument
 
         if (!group) {
             throw new ParseError(
@@ -312,7 +305,7 @@ export default class Parser {
     parseAtom(breakOnTokenText?: BreakToken): ?AnyParseNode {
         // The body of an atom is an implicit group, so that things like
         // \left(x\right)^2 work correctly.
-        const base = this.parseGroup("atom", false, null, breakOnTokenText);
+        const base = this.parseGroup("atom", null, breakOnTokenText);
 
         // In text mode, we don't have superscripts or subscripts
         if (this.mode === "text") {
@@ -480,31 +473,24 @@ export default class Parser {
         const optArgs = [];
 
         for (let i = 0; i < totalArgs; i++) {
-            const argType = funcData.argTypes && funcData.argTypes[i];
+            let argType = funcData.argTypes && funcData.argTypes[i];
             const isOptional = i < funcData.numOptionalArgs;
-            // Ignore spaces between arguments.  As the TeXbook says:
+
-            // "After you have said ‘\def\row#1#2{...}’, you are allowed to
+            if ((funcData.primitive && argType == null) ||
-            //  put spaces between the arguments (e.g., ‘\row x n’), because
+                // \sqrt expands into primitive if optional argument doesn't exist
-            //  TeX doesn’t use single spaces as undelimited arguments."
+                (funcData.type === "sqrt" && i === 1 && optArgs[0] == null)) {
-            const consumeSpaces = (i > 0 && !isOptional) ||
+                argType = "primitive";
-            // Also consume leading spaces in math mode, as parseSymbol
+            }
-            // won't know what to do with them.  This can only happen with
+
-            // macros, e.g. \frac\foo\foo where \foo expands to a space symbol.
+            const arg = this.parseGroupOfType(`argument to '${func}'`,
-            // In LaTeX, the \foo's get treated as (blank) arguments.
+                argType, isOptional, baseGreediness);
-            // In KaTeX, for now, both spaces will get consumed.
+            if (isOptional) {
-            // TODO(edemaine)
+                optArgs.push(arg);
-                (i === 0 && !isOptional && this.mode === "math");
+            } else if (arg != null) {
-            const arg = this.parseGroupOfType(`argument to '${func}'`,
+                args.push(arg);
-                argType, isOptional, baseGreediness, consumeSpaces);
+            } else { // should be unreachable
-            if (!arg) {
+                throw new ParseError("Null argument, please report this as a bug");
-                if (isOptional) {
-                    optArgs.push(null);
-                    continue;
-                }
-                throw new ParseError(
-                    `Expected group after '${func}'`, this.fetch());
             }
-            (isOptional ? optArgs : args).push(arg);
         }
 
         return {args, optArgs};
@@ -518,64 +504,50 @@ export default class Parser {
         type: ?ArgType,
         optional: boolean,
         greediness: ?number,
-        consumeSpaces: boolean,
     ): ?AnyParseNode {
         switch (type) {
             case "color":
-                if (consumeSpaces) {
-                    this.consumeSpaces();
-                }
                 return this.parseColorGroup(optional);
             case "size":
-                if (consumeSpaces) {
-                    this.consumeSpaces();
-                }
                 return this.parseSizeGroup(optional);
             case "url":
-                return this.parseUrlGroup(optional, consumeSpaces);
+                return this.parseUrlGroup(optional);
             case "math":
             case "text":
-                return this.parseGroup(
+                return this.parseArgumentGroup(optional, type);
-                    name, optional, greediness, undefined, type, consumeSpaces);
             case "hbox": {
                 // hbox argument type wraps the argument in the equivalent of
                 // \hbox, which is like \text but switching to \textstyle size.
-                const group = this.parseGroup(name, optional, greediness,
+                const group = this.parseArgumentGroup(optional, "text");
-                    undefined, "text", consumeSpaces);
+                return group != null ? {
-                if (!group) {
-                    return group;
-                }
-                const styledGroup = {
                     type: "styling",
                     mode: group.mode,
                     body: [group],
                     style: "text", // simulate \textstyle
-                };
+                } : null;
-                return styledGroup;
             }
             case "raw": {
-                if (consumeSpaces) {
+                const token = this.parseStringGroup("raw", optional);
-                    this.consumeSpaces();
+                return token != null ? {
+                    type: "raw",
+                    mode: "text",
+                    string: token.text,
+                } : null;
+            }
+            case "primitive": {
+                if (optional) {
+                    throw new ParseError("A primitive argument cannot be optional");
                 }
-                if (optional && this.fetch().text === "{") {
+                const group = this.parseGroup(name, greediness);
-                    return null;
+                if (group == null) {
-                }
+                    throw new ParseError("Expected group as " + name, this.fetch());
-                const token = this.parseStringGroup("raw", optional, true);
-                if (token) {
-                    return {
-                        type: "raw",
-                        mode: "text",
-                        string: token.text,
-                    };
-                } else {
-                    throw new ParseError("Expected raw group", this.fetch());
                 }
+                return group;
             }
             case "original":
             case null:
             case undefined:
-                return this.parseGroup(name, optional, greediness,
+                return this.parseArgumentGroup(optional);
-                    undefined, undefined, consumeSpaces);
             default:
                 throw new ParseError(
                     "Unknown group type as " + name, this.fetch());
@@ -598,49 +570,20 @@ export default class Parser {
     parseStringGroup(
         modeName: ArgType,  // Used to describe the mode in error messages.
         optional: boolean,
-        raw?: boolean,
     ): ?Token {
-        const groupBegin = optional ? "[" : "{";
+        const argToken = this.gullet.scanArgument(optional);
-        const groupEnd = optional ? "]" : "}";
+        if (argToken == null) {
-        const beginToken = this.fetch();
+            return null;
-        if (beginToken.text !== groupBegin) {
-            if (optional) {
-                return null;
-            } else if (raw && beginToken.text !== "EOF" &&
-                    /[^{}[\]]/.test(beginToken.text)) {
-                this.consume();
-                return beginToken;
-            }
         }
-        const outerMode = this.mode;
-        this.mode = "text";
-        this.expect(groupBegin);
         let str = "";
-        const firstToken = this.fetch();
-        let nested = 0; // allow nested braces in raw string group
-        let lastToken = firstToken;
         let nextToken;
-        while ((nextToken = this.fetch()).text !== groupEnd ||
+        while ((nextToken = this.fetch()).text !== "EOF") {
-               (raw && nested > 0)) {
+            str += nextToken.text;
-            switch (nextToken.text) {
-                case "EOF":
-                    throw new ParseError(
-                        "Unexpected end of input in " + modeName,
-                        firstToken.range(lastToken, str));
-                case groupBegin:
-                    nested++;
-                    break;
-                case groupEnd:
-                    nested--;
-                    break;
-            }
-            lastToken = nextToken;
-            str += lastToken.text;
             this.consume();
         }
-        this.expect(groupEnd);
+        this.consume(); // consume the end of the argument
-        this.mode = outerMode;
+        argToken.text = str;
-        return firstToken.range(lastToken, str);
+        return argToken;
     }
 
     /**
@@ -652,8 +595,6 @@ export default class Parser {
         regex: RegExp,
         modeName: string,   // Used to describe the mode in error messages.
     ): Token {
-        const outerMode = this.mode;
-        this.mode = "text";
         const firstToken = this.fetch();
         let lastToken = firstToken;
         let str = "";
@@ -669,7 +610,6 @@ export default class Parser {
                 "Invalid " + modeName + ": '" + firstToken.text + "'",
                 firstToken);
         }
-        this.mode = outerMode;
         return firstToken.range(lastToken, str);
     }
 
@@ -678,7 +618,7 @@ export default class Parser {
      */
     parseColorGroup(optional: boolean): ?ParseNode<"color-token"> {
         const res = this.parseStringGroup("color", optional);
-        if (!res) {
+        if (res == null) {
             return null;
         }
         const match = (/^(#[a-f0-9]{3}|#?[a-f0-9]{6}|[a-z]+)$/i).exec(res.text);
@@ -705,7 +645,9 @@ export default class Parser {
     parseSizeGroup(optional: boolean): ?ParseNode<"size"> {
         let res;
         let isBlank = false;
-        if (!optional && this.fetch().text !== "{") {
+        // don't expand before parseStringGroup
+        this.gullet.consumeSpaces();
+        if (!optional && this.gullet.future().text !== "{") {
             res = this.parseRegexGroup(
                 /^[-+]? *(?:$|\d+|\d+\.\d*|\.\d*) *[a-z]{0,2} *$/, "size");
         } else {
@@ -744,11 +686,11 @@ export default class Parser {
      * Parses an URL, checking escaped letters and allowed protocols,
      * and setting the catcode of % as an active character (as in \hyperref).
      */
-    parseUrlGroup(optional: boolean, consumeSpaces: boolean): ?ParseNode<"url"> {
+    parseUrlGroup(optional: boolean): ?ParseNode<"url"> {
         this.gullet.lexer.setCatcode("%", 13); // active character
-        const res = this.parseStringGroup("url", optional, true); // get raw string
+        const res = this.parseStringGroup("url", optional);
         this.gullet.lexer.setCatcode("%", 14); // comment character
-        if (!res) {
+        if (res == null) {
             return null;
         }
         // hyperref package allows backslashes alone in href, but doesn't
@@ -764,52 +706,61 @@ export default class Parser {
     }
 
     /**
-     * If `optional` is false or absent, this parses an ordinary group,
+     * Parses an argument with the mode specified.
-     * which is either a single nucleus (like "x") or an expression
+     */
-     * in braces (like "{x+y}") or an implicit group, a group that starts
+    parseArgumentGroup(optional: boolean, mode?: Mode): ?ParseNode<"ordgroup"> {
-     * at the current position, and ends right before a higher explicit
+        const argToken = this.gullet.scanArgument(optional);
+        if (argToken == null) {
+            return null;
+        }
+        const outerMode = this.mode;
+        if (mode) { // Switch to specified mode
+            this.switchMode(mode);
+        }
+
+        this.gullet.beginGroup();
+        const expression = this.parseExpression(false, "EOF");
+        // TODO: find an alternative way to denote the end
+        this.expect("EOF"); // expect the end of the argument
+        this.gullet.endGroup();
+        const result = {
+            type: "ordgroup",
+            mode: this.mode,
+            loc: argToken.loc,
+            body: expression,
+        };
+
+        if (mode) { // Switch mode back
+            this.switchMode(outerMode);
+        }
+        return result;
+    }
+
+    /**
+     * Parses an ordinary group, which is either a single nucleus (like "x")
+     * or an expression in braces (like "{x+y}") or an implicit group, a group
+     * that starts at the current position, and ends right before a higher explicit
      * group ends, or at EOF.
-     * If `optional` is true, it parses either a bracket-delimited expression
-     * (like "[x+y]") or returns null to indicate the absence of a
-     * bracket-enclosed group.
-     * If `mode` is present, switches to that mode while parsing the group,
-     * and switches back after.
      */
     parseGroup(
         name: string, // For error reporting.
-        optional?: boolean,
         greediness?: ?number,
         breakOnTokenText?: BreakToken,
-        mode?: Mode,
-        consumeSpaces?: boolean,
     ): ?AnyParseNode {
-        // Switch to specified mode
-        const outerMode = this.mode;
-        if (mode) {
-            this.switchMode(mode);
-        }
-        // Consume spaces if requested, crucially *after* we switch modes,
-        // so that the next non-space token is parsed in the correct mode.
-        if (consumeSpaces) {
-            this.consumeSpaces();
-        }
-        // Get first token
         const firstToken = this.fetch();
         const text = firstToken.text;
 
         let result;
         // Try to parse an open brace or \begingroup
-        if (optional ? text === "["  : text === "{" || text === "\\begingroup") {
+        if (text === "{" || text === "\\begingroup") {
             this.consume();
-            const groupEnd = Parser.endOfGroup[text];
+            const groupEnd = text === "{" ? "}" : "\\endgroup";
-            // Start a new group namespace
+
             this.gullet.beginGroup();
             // If we get a brace, parse an expression
             const expression = this.parseExpression(false, groupEnd);
             const lastToken = this.fetch();
-            // Check that we got a matching closing brace
+            this.expect(groupEnd); // Check that we got a matching closing brace
-            this.expect(groupEnd);
-            // End group namespace
             this.gullet.endGroup();
             result = {
                 type: "ordgroup",
@@ -822,9 +773,6 @@ export default class Parser {
                 // use-begingroup-instead-of-bgroup
                 semisimple: text === "\\begingroup" || undefined,
             };
-        } else if (optional) {
-            // Return nothing for an optional group
-            result = null;
         } else {
             // If there exists a function with this name, parse the function.
             // Otherwise, just return a nucleus
@@ -840,11 +788,6 @@ export default class Parser {
                 this.consume();
             }
         }
-
-        // Switch mode back
-        if (mode) {
-            this.switchMode(outerMode);
-        }
         return result;
     }
 

src/defineFunction.js

@@ -83,6 +83,9 @@ export type FunctionPropSpec = {
 
     // Must be true if the function is an infix operator.
     infix?: boolean,
+
+    // Whether or not the function is a TeX primitive.
+    primitive?: boolean,
 };
 
 type FunctionDefSpec<NODETYPE: NodeType> = {|
@@ -128,6 +131,7 @@ export type FunctionSpec<NODETYPE: NodeType> = {|
     allowedInMath: boolean,
     numOptionalArgs: number,
     infix: boolean,
+    primitive: boolean,
 
     // FLOW TYPE NOTES: Doing either one of the following two
     //
@@ -186,6 +190,7 @@ export default function defineFunction<NODETYPE: NodeType>({
             : props.allowedInMath,
         numOptionalArgs: props.numOptionalArgs || 0,
         infix: !!props.infix,
+        primitive: !!props.primitive,
         handler: handler,
     };
     for (let i = 0; i < names.length; ++i) {
@@ -223,6 +228,10 @@ export function defineFunctionBuilders<NODETYPE: NodeType>({
     });
 }
 
+export const normalizeArgument = function(arg: AnyParseNode): AnyParseNode {
+    return arg.type === "ordgroup" && arg.body.length === 1 ? arg.body[0] : arg;
+};
+
 // Since the corresponding buildHTML/buildMathML function expects a
 // list of elements, we normalize for different kinds of arguments
 export const ordargument = function(arg: AnyParseNode): AnyParseNode[] {

src/environments/array.js

@@ -86,12 +86,11 @@ function parseArray(
     |},
     style: StyleStr,
 ): ParseNode<"array"> {
-    // Parse body of array with \\ temporarily mapped to \cr
     parser.gullet.beginGroup();
-    if (singleRow) {
+    if (!singleRow) {
-        parser.gullet.macros.set("\\\\", ""); // {equation} acts this way.
+        // \cr is equivalent to \\ without the optional size argument (see below)
-    } else {
+        // TODO: provide helpful error when \cr is used outside array environment
-        parser.gullet.macros.set("\\\\", "\\cr");
+        parser.gullet.macros.set("\\cr", "\\\\\\relax");
     }
 
     // Get current arraystretch if it's not set by the environment
@@ -121,7 +120,7 @@ function parseArray(
 
     while (true) {  // eslint-disable-line no-constant-condition
         // Parse each cell in its own group (namespace)
-        let cell = parser.parseExpression(false, "\\cr");
+        let cell = parser.parseExpression(false, singleRow ? "\\end" : "\\\\");
         parser.gullet.endGroup();
         parser.gullet.beginGroup();
 
@@ -165,12 +164,18 @@ function parseArray(
                 hLinesBeforeRow.push([]);
             }
             break;
-        } else if (next === "\\cr") {
+        } else if (next === "\\\\") {
-            if (singleRow) {
+            parser.consume();
-                throw new ParseError("Misplaced \\cr.", parser.nextToken);
+            let size;
+            // \def\Let@{\let\\\math@cr}
+            // \def\math@cr{...\math@cr@}
+            // \def\math@cr@{\new@ifnextchar[\math@cr@@{\math@cr@@[\z@]}}
+            // \def\math@cr@@[#1]{...\math@cr@@@...}
+            // \def\math@cr@@@{\cr}
+            if (parser.gullet.future().text !== " ") {
+                size = parser.parseSizeGroup(true);
             }
-            const cr = assertNodeType(parser.parseFunction(), "cr");
+            rowGaps.push(size ? size.value : null);
-            rowGaps.push(cr.size);
 
             // check for \hline(s) following the row separator
             hLinesBeforeRow.push(getHLines(parser));
@@ -185,7 +190,7 @@ function parseArray(
 
     // End cell group
     parser.gullet.endGroup();
-    // End array group defining \\
+    // End array group defining \cr
     parser.gullet.endGroup();
 
     return {

src/functions/accent.js

@@ -1,5 +1,5 @@
 // @flow
-import defineFunction from "../defineFunction";
+import defineFunction, {normalizeArgument} from "../defineFunction";
 import buildCommon from "../buildCommon";
 import mathMLTree from "../mathMLTree";
 import utils from "../utils";
@@ -218,7 +218,7 @@ defineFunction({
         numArgs: 1,
     },
     handler: (context, args) => {
-        const base = args[0];
+        const base = normalizeArgument(args[0]);
 
         const isStretchy = !NON_STRETCHY_ACCENT_REGEX.test(context.funcName);
         const isShifty = !isStretchy ||
@@ -250,6 +250,7 @@ defineFunction({
         numArgs: 1,
         allowedInText: true,
         allowedInMath: false,
+        argTypes: ["primitive"],
     },
     handler: (context, args) => {
         const base = args[0];

src/functions/cr.js

@@ -5,17 +5,12 @@ import defineFunction from "../defineFunction";
 import buildCommon from "../buildCommon";
 import mathMLTree from "../mathMLTree";
 import {calculateSize} from "../units";
-import ParseError from "../ParseError";
 import {assertNodeType} from "../parseNode";
 
-// \\ is a macro mapping to either \cr or \newline.  Because they have the
+// \DeclareRobustCommand\\{...\@xnewline}
-// same signature, we implement them as one megafunction, with newRow
-// indicating whether we're in the \cr case, and newLine indicating whether
-// to break the line in the \newline case.
-
 defineFunction({
     type: "cr",
-    names: ["\\cr", "\\newline"],
+    names: ["\\\\"],
     props: {
         numArgs: 0,
         numOptionalArgs: 1,
@@ -23,25 +18,16 @@ defineFunction({
         allowedInText: true,
     },
 
-    handler({parser, funcName}, args, optArgs) {
+    handler({parser}, args, optArgs) {
         const size = optArgs[0];
-        const newRow = (funcName === "\\cr");
+        const newLine = !parser.settings.displayMode ||
-        let newLine = false;
+            !parser.settings.useStrictBehavior(
-        if (!newRow) {
+                "newLineInDisplayMode", "In LaTeX, \\\\ or \\newline " +
-            if (parser.settings.displayMode &&
+                "does nothing in display mode");
-                parser.settings.useStrictBehavior(
-                    "newLineInDisplayMode", "In LaTeX, \\\\ or \\newline " +
-                    "does nothing in display mode")) {
-                newLine = false;
-            } else {
-                newLine = true;
-            }
-        }
         return {
             type: "cr",
             mode: parser.mode,
             newLine,
-            newRow,
             size: size && assertNodeType(size, "size").value,
         };
     },
@@ -50,10 +36,6 @@ defineFunction({
     // not within tabular/array environments.
 
     htmlBuilder(group, options) {
-        if (group.newRow) {
-            throw new ParseError(
-                "\\cr valid only within a tabular/array environment");
-        }
         const span = buildCommon.makeSpan(["mspace"], [], options);
         if (group.newLine) {
             span.classes.push("newline");

src/functions/def.js

@@ -88,41 +88,65 @@ defineFunction({
     props: {
         numArgs: 0,
         allowedInText: true,
+        primitive: true,
     },
     handler({parser, funcName}) {
-        let arg = parser.gullet.consumeArgs(1)[0];
+        let tok = parser.gullet.popToken();
-        if (arg.length !== 1) {
+        const name = tok.text;
-            throw new ParseError("\\gdef's first argument must be a macro name");
+        if (/^(?:[\\{}$&#^_]|EOF)$/.test(name)) {
+            throw new ParseError("Expected a control sequence", tok);
         }
-        const name = arg[0].text;
+
-        // Count argument specifiers, and check they are in the order #1 #2 ...
         let numArgs = 0;
-        arg = parser.gullet.consumeArgs(1)[0];
+        let insert;
-        while (arg.length === 1 && arg[0].text === "#") {
+        const delimiters = [[]];
-            arg = parser.gullet.consumeArgs(1)[0];
+        // <parameter text> contains no braces
-            if (arg.length !== 1) {
+        while (parser.gullet.future().text !== "{") {
-                throw new ParseError(
+            tok = parser.gullet.popToken();
-                    `Invalid argument number length "${arg.length}"`);
+            if (tok.text === "#") {
+                // If the very last character of the <parameter text> is #, so that
+                // this # is immediately followed by {, TeX will behave as if the {
+                // had been inserted at the right end of both the parameter text
+                // and the replacement text.
+                if (parser.gullet.future().text === "{") {
+                    insert = parser.gullet.future();
+                    delimiters[numArgs].push("{");
+                    break;
+                }
+
+                // A parameter, the first appearance of # must be followed by 1,
+                // the next by 2, and so on; up to nine #’s are allowed
+                tok = parser.gullet.popToken();
+                if (!(/^[1-9]$/.test(tok.text))) {
+                    throw new ParseError(`Invalid argument number "${tok.text}"`);
+                }
+                if (parseInt(tok.text) !== numArgs + 1) {
+                    throw new ParseError(
+                        `Argument number "${tok.text}" out of order`);
+                }
+                numArgs++;
+                delimiters.push([]);
+            } else if (tok.text === "EOF") {
+                throw new ParseError("Expected a macro definition");
+            } else {
+                delimiters[numArgs].push(tok.text);
             }
-            if (!(/^[1-9]$/.test(arg[0].text))) {
-                throw new ParseError(
-                    `Invalid argument number "${arg[0].text}"`);
-            }
-            numArgs++;
-            if (parseInt(arg[0].text) !== numArgs) {
-                throw new ParseError(
-                    `Argument number "${arg[0].text}" out of order`);
-            }
-            arg = parser.gullet.consumeArgs(1)[0];
         }
+        // replacement text, enclosed in '{' and '}' and properly nested
+        let {tokens} = parser.gullet.consumeArg();
+        if (insert) {
+            tokens.unshift(insert);
+        }
+
         if (funcName === "\\edef" || funcName === "\\xdef") {
-            arg = parser.gullet.expandTokens(arg);
+            tokens = parser.gullet.expandTokens(tokens);
-            arg.reverse(); // to fit in with stack order
+            tokens.reverse(); // to fit in with stack order
         }
         // Final arg is the expansion of the macro
         parser.gullet.macros.set(name, {
-            tokens: arg,
+            tokens,
             numArgs,
+            delimiters,
         }, funcName === globalMap[funcName]);
 
         return {
@@ -145,6 +169,7 @@ defineFunction({
     props: {
         numArgs: 0,
         allowedInText: true,
+        primitive: true,
     },
     handler({parser, funcName}) {
         const name = checkControlSequence(parser.gullet.popToken());
@@ -168,6 +193,7 @@ defineFunction({
     props: {
         numArgs: 0,
         allowedInText: true,
+        primitive: true,
     },
     handler({parser, funcName}) {
         const name = checkControlSequence(parser.gullet.popToken());

src/functions/delimsizing.js

@@ -81,6 +81,7 @@ defineFunction({
     ],
     props: {
         numArgs: 1,
+        argTypes: ["primitive"],
     },
     handler: (context, args) => {
         const delim = checkDelimiter(args[0], context);
@@ -145,6 +146,7 @@ defineFunction({
     names: ["\\right"],
     props: {
         numArgs: 1,
+        primitive: true,
     },
     handler: (context, args) => {
         // \left case below triggers parsing of \right in
@@ -170,6 +172,7 @@ defineFunction({
     names: ["\\left"],
     props: {
         numArgs: 1,
+        primitive: true,
     },
     handler: (context, args) => {
         const delim = checkDelimiter(args[0], context);
@@ -303,6 +306,7 @@ defineFunction({
     names: ["\\middle"],
     props: {
         numArgs: 1,
+        primitive: true,
     },
     handler: (context, args) => {
         const delim = checkDelimiter(args[0], context);

src/functions/font.js

@@ -2,7 +2,7 @@
 // TODO(kevinb): implement \\sl and \\sc
 
 import {binrelClass} from "./mclass";
-import defineFunction from "../defineFunction";
+import defineFunction, {normalizeArgument} from "../defineFunction";
 import utils from "../utils";
 
 import * as html from "../buildHTML";
@@ -47,7 +47,7 @@ defineFunction({
         greediness: 2,
     },
     handler: ({parser, funcName}, args) => {
-        const body = args[0];
+        const body = normalizeArgument(args[0]);
         let func = funcName;
         if (func in fontAliases) {
             func = fontAliases[func];

src/functions/genfrac.js

@@ -1,5 +1,5 @@
 // @flow
-import defineFunction from "../defineFunction";
+import defineFunction, {normalizeArgument} from "../defineFunction";
 import buildCommon from "../buildCommon";
 import delimiter from "../delimiter";
 import mathMLTree from "../mathMLTree";
@@ -382,10 +382,12 @@ defineFunction({
         const denom = args[5];
 
         // Look into the parse nodes to get the desired delimiters.
-        const leftDelim = args[0].type === "atom" && args[0].family === "open"
+        const leftNode = normalizeArgument(args[0]);
-            ? delimFromValue(args[0].text) : null;
+        const leftDelim = leftNode.type === "atom" && leftNode.family === "open"
-        const rightDelim = args[1].type === "atom" && args[1].family === "close"
+            ? delimFromValue(leftNode.text) : null;
-            ? delimFromValue(args[1].text) : null;
+        const rightNode = normalizeArgument(args[1]);
+        const rightDelim = rightNode.type === "atom" && rightNode.family === "close"
+            ? delimFromValue(rightNode.text) : null;
 
         const barNode = assertNodeType(args[2], "size");
         let hasBarLine;

src/functions/kern.js

@@ -15,6 +15,7 @@ defineFunction({
     props: {
         numArgs: 1,
         argTypes: ["size"],
+        primitive: true,
         allowedInText: true,
     },
     handler({parser, funcName}, args) {

src/functions/mathchoice.js

@@ -23,6 +23,7 @@ defineFunction({
     names: ["\\mathchoice"],
     props: {
         numArgs: 4,
+        primitive: true,
     },
     handler: ({parser}, args) => {
         return {

src/functions/mclass.js

@@ -65,6 +65,7 @@ defineFunction({
     ],
     props: {
         numArgs: 1,
+        primitive: true,
     },
     handler({parser, funcName}, args) {
         const body = args[0];
@@ -106,7 +107,7 @@ defineFunction({
             type: "mclass",
             mode: parser.mode,
             mclass: binrelClass(args[0]),
-            body: [args[1]],
+            body: ordargument(args[1]),
             isCharacterBox: utils.isCharacterBox(args[1]),
         };
     },

src/functions/op.js

@@ -225,6 +225,7 @@ defineFunction({
     names: ["\\mathop"],
     props: {
         numArgs: 1,
+        primitive: true,
     },
     handler: ({parser}, args) => {
         const body = args[0];

src/functions/styling.js

@@ -22,6 +22,7 @@ defineFunction({
     props: {
         numArgs: 0,
         allowedInText: true,
+        primitive: true,
     },
     handler({breakOnTokenText, funcName, parser}, args) {
         // parse out the implicit body

src/macros.js

@@ -71,6 +71,12 @@ export interface MacroContextInterface {
      */
     expandMacroAsText(name: string): string | void;
 
+    /**
+     * Consume an argument from the token stream, and return the resulting array
+     * of tokens and start/end token.
+     */
+    consumeArg(delims?: ?string[]): MacroArg;
+
     /**
      * Consume the specified number of arguments from the token stream,
      * and return the resulting array of arguments.
@@ -91,10 +97,17 @@ export interface MacroContextInterface {
     isExpandable(name: string): boolean;
 }
 
+export type MacroArg = {
+    tokens: Token[],
+    start: Token,
+    end: Token
+};
+
 /** Macro tokens (in reverse order). */
 export type MacroExpansion = {
     tokens: Token[],
     numArgs: number,
+    delimiters?: string[][],
     unexpandable?: boolean, // used in \let
 };
 
@@ -240,7 +253,7 @@ defineMacro("\\char", function(context) {
 // \renewcommand{\macro}[args]{definition}
 // TODO: Optional arguments: \newcommand{\macro}[args][default]{definition}
 const newcommand = (context, existsOK: boolean, nonexistsOK: boolean) => {
-    let arg = context.consumeArgs(1)[0];
+    let arg = context.consumeArg().tokens;
     if (arg.length !== 1) {
         throw new ParseError(
             "\\newcommand's first argument must be a macro name");
@@ -258,7 +271,7 @@ const newcommand = (context, existsOK: boolean, nonexistsOK: boolean) => {
     }
 
     let numArgs = 0;
-    arg = context.consumeArgs(1)[0];
+    arg = context.consumeArg().tokens;
     if (arg.length === 1 && arg[0].text === "[") {
         let argText = '';
         let token = context.expandNextToken();
@@ -271,7 +284,7 @@ const newcommand = (context, existsOK: boolean, nonexistsOK: boolean) => {
             throw new ParseError(`Invalid number of arguments: ${argText}`);
         }
         numArgs = parseInt(argText);
-        arg = context.consumeArgs(1)[0];
+        arg = context.consumeArg().tokens;
     }
 
     // Final arg is the expansion of the macro
@@ -696,8 +709,10 @@ defineMacro("\\pmb", "\\html@mathml{" +
 //////////////////////////////////////////////////////////////////////
 // LaTeX source2e
 
-// \\ defaults to \newline, but changes to \cr within array environment
+// \expandafter\let\expandafter\@normalcr
-defineMacro("\\\\", "\\newline");
+//     \csname\expandafter\@gobble\string\\ \endcsname
+// \DeclareRobustCommand\newline{\@normalcr\relax}
+defineMacro("\\newline", "\\\\\\relax");
 
 // \def\TeX{T\kern-.1667em\lower.5ex\hbox{E}\kern-.125emX\@}
 // TODO: Doesn't normally work in math mode because \@ fails.  KaTeX doesn't

src/parseNode.js

@@ -209,7 +209,6 @@ type ParseNodeTypes = {
         type: "cr",
         mode: Mode,
         loc?: ?SourceLocation,
-        newRow: boolean,
         newLine: boolean,
         size: ?Measurement,
     |},

src/types.js

@@ -21,13 +21,14 @@ export type Mode = "math" | "text";
 //                 argument is parsed normally)
 //   - Mode: Node group parsed in given mode.
 export type ArgType = "color" | "size" | "url" | "raw" | "original" | "hbox" |
-    Mode;
+    "primitive" | Mode;
 
 // LaTeX display style.
 export type StyleStr = "text" | "display" | "script" | "scriptscript";
 
 // Allowable token text for "break" arguments in parser.
-export type BreakToken = "]" | "}" | "\\endgroup" | "$" | "\\)" | "\\cr";
+export type BreakToken = "]" | "}" | "\\endgroup" | "$" | "\\)" | "\\\\" | "\\end" |
+    "EOF";
 
 // Math font variants.
 export type FontVariant = "bold" | "bold-italic" | "bold-sans-serif" |

test/errors-spec.js

@@ -94,11 +94,12 @@ describe("Parser:", function() {
     describe("#parseArguments", function() {
         it("complains about missing argument at end of input", function() {
             expect`2\sqrt`.toFailWithParseError(
-                   "Expected group after '\\sqrt' at end of input: 2\\sqrt");
+                   "Expected group as argument to '\\sqrt'" +
+                   " at end of input: 2\\sqrt");
         });
         it("complains about missing argument at end of group", function() {
             expect`1^{2\sqrt}`.toFailWithParseError(
-                   "Expected group after '\\sqrt'" +
+                   "Expected group as argument to '\\sqrt'" +
                    " at position 10: 1^{2\\sqrt}̲");
         });
         it("complains about functions as arguments to others", function() {
@@ -166,7 +167,7 @@ describe("Parser.expect calls:", function() {
     describe("#parseSpecialGroup expecting braces", function() {
         it("complains about missing { for color", function() {
             expect`\textcolor#ffffff{text}`.toFailWithParseError(
-                   "Expected '{', got '#' at position 11:" +
+                   "Invalid color: '#' at position 11:" +
                    " \\textcolor#̲ffffff{text}");
         });
         it("complains about missing { for size", function() {
@@ -176,23 +177,23 @@ describe("Parser.expect calls:", function() {
         // Can't test for the [ of an optional group since it's optional
         it("complains about missing } for color", function() {
             expect`\textcolor{#ffffff{text}`.toFailWithParseError(
-                   "Invalid color: '#ffffff{text' at position 12:" +
+                   "Unexpected end of input in a macro argument," +
-                   " \\textcolor{#̲f̲f̲f̲f̲f̲f̲{̲t̲e̲x̲t̲}");
+                   " expected '}' at end of input: …r{#ffffff{text}");
         });
         it("complains about missing ] for size", function() {
             expect`\rule[1em{2em}{3em}`.toFailWithParseError(
-                   "Unexpected end of input in size" +
+                   "Unexpected end of input in a macro argument," +
-                   " at position 7: \\rule[1̲e̲m̲{̲2̲e̲m̲}̲{̲3̲e̲m̲}̲");
+                   " expected ']' at end of input: …e[1em{2em}{3em}");
         });
         it("complains about missing ] for size at end of input", function() {
             expect`\rule[1em`.toFailWithParseError(
-                   "Unexpected end of input in size" +
+                   "Unexpected end of input in a macro argument," +
-                   " at position 7: \\rule[1̲e̲m̲");
+                   " expected ']' at end of input: \\rule[1em");
         });
         it("complains about missing } for color at end of input", function() {
             expect`\textcolor{#123456`.toFailWithParseError(
-                   "Unexpected end of input in color" +
+                   "Unexpected end of input in a macro argument," +
-                   " at position 12: \\textcolor{#̲1̲2̲3̲4̲5̲6̲");
+                   " expected '}' at end of input: …xtcolor{#123456");
         });
     });
 
@@ -206,11 +207,13 @@ describe("Parser.expect calls:", function() {
     describe("#parseOptionalGroup expecting ]", function() {
         it("at end of file", function() {
             expect`\sqrt[3`.toFailWithParseError(
-                   "Expected ']', got 'EOF' at end of input: \\sqrt[3");
+                   "Unexpected end of input in a macro argument," +
+                   " expected ']' at end of input: \\sqrt[3");
         });
         it("before group", function() {
             expect`\sqrt[3{2}`.toFailWithParseError(
-                   "Expected ']', got 'EOF' at end of input: \\sqrt[3{2}");
+                   "Unexpected end of input in a macro argument," +
+                   " expected ']' at end of input: \\sqrt[3{2}");
         });
     });
 
@@ -269,7 +272,7 @@ describe("functions.js:", function() {
     describe("\\begin and \\end", function() {
         it("reject invalid environment names", function() {
             expect`\begin x\end y`.toFailWithParseError(
-                   "Invalid environment name at position 8: \\begin x̲\\end y");
+                   "No such environment: x at position 8: \\begin x̲\\end y");
         });
     });
 
@@ -293,22 +296,22 @@ describe("Lexer:", function() {
         it("reject 3-digit hex notation without #", function() {
             expect`\textcolor{1a2}{foo}`.toFailWithParseError(
                    "Invalid color: '1a2'" +
-                   " at position 12: \\textcolor{1̲a̲2̲}{foo}");
+                   " at position 11: \\textcolor{̲1̲a̲2̲}̲{foo}");
         });
     });
 
     describe("#_innerLexSize", function() {
         it("reject size without unit", function() {
             expect`\rule{0}{2em}`.toFailWithParseError(
-                   "Invalid size: '0' at position 7: \\rule{0̲}{2em}");
+                   "Invalid size: '0' at position 6: \\rule{̲0̲}̲{2em}");
         });
         it("reject size with bogus unit", function() {
             expect`\rule{1au}{2em}`.toFailWithParseError(
-                   "Invalid unit: 'au' at position 7: \\rule{1̲a̲u̲}{2em}");
+                   "Invalid unit: 'au' at position 6: \\rule{̲1̲a̲u̲}̲{2em}");
         });
         it("reject size without number", function() {
             expect`\rule{em}{2em}`.toFailWithParseError(
-                   "Invalid size: 'em' at position 7: \\rule{e̲m̲}{2em}");
+                   "Invalid size: 'em' at position 6: \\rule{̲e̲m̲}̲{2em}");
         });
     });
 

test/katex-spec.js

@@ -1282,8 +1282,13 @@ describe("A begin/end parser", function() {
         expect(m2).toParse();
     });
 
-    it("should allow \\cr as a line terminator", function() {
+    it("should allow \\cr and \\\\ as a line terminator", function() {
         expect`\begin{matrix}a&b\cr c&d\end{matrix}`.toParse();
+        expect`\begin{matrix}a&b\\c&d\end{matrix}`.toParse();
+    });
+
+    it("should not allow \\cr to scan for an optional size argument", function() {
+        expect`\begin{matrix}a&b\cr[c]&d\end{matrix}`.toParse();
     });
 
     it("should eat a final newline", function() {
@@ -1318,6 +1323,16 @@ describe("A sqrt parser", function() {
     it("should build sized square roots", function() {
         expect("\\Large\\sqrt[3]{x}").toBuild();
     });
+
+    it("should expand argument if optional argument doesn't exist", function() {
+        expect("\\sqrt\\foo").toParseLike("\\sqrt123",
+            new Settings({macros: {"\\foo": "123"}}));
+    });
+
+    it("should not expand argument if optional argument exists", function() {
+        expect("\\sqrt[2]\\foo").toParseLike("\\sqrt[2]{123}",
+            new Settings({macros: {"\\foo": "123"}}));
+    });
 });
 
 describe("A TeX-compliant parser", function() {
@@ -2576,23 +2591,6 @@ describe("A smash builder", function() {
     });
 });
 
-describe("A document fragment", function() {
-    it("should have paddings applied inside an extensible arrow", function() {
-        const markup = katex.renderToString("\\tiny\\xrightarrow\\textcolor{red}{x}");
-        expect(markup).toContain("x-arrow-pad");
-    });
-
-    it("should have paddings applied inside an enclose", function() {
-        const markup = katex.renderToString(r`\fbox\textcolor{red}{x}`);
-        expect(markup).toContain("boxpad");
-    });
-
-    it("should have paddings applied inside a square root", function() {
-        const markup = katex.renderToString(r`\sqrt\textcolor{red}{x}`);
-        expect(markup).toContain("padding-left");
-    });
-});
-
 describe("A parser error", function() {
     it("should report the position of an error", function() {
         try {
@@ -2884,11 +2882,6 @@ describe("href and url commands", function() {
 });
 
 describe("A raw text parser", function() {
-    it("should not not parse a mal-formed string", function() {
-        // In the next line, the first character passed to \includegraphics is a
-        // Unicode combining character. So this is a test that the parser will catch a bad string.
-        expect("\\includegraphics[\u030aheight=0.8em, totalheight=0.9em, width=0.9em]{" + "https://cdn.kastatic.org/images/apple-touch-icon-57x57-precomposed.new.png}").not.toParse();
-    });
     it("should return null for a omitted optional string", function() {
         expect("\\includegraphics{https://cdn.kastatic.org/images/apple-touch-icon-57x57-precomposed.new.png}").toParse();
     });
@@ -3039,12 +3032,17 @@ describe("A macro expander", function() {
     });
 
     it("should allow for macro argument", function() {
-        expect`\foo\bar`.toParseLike("(x)", new Settings({macros: {
+        expect`\foo\bar`.toParseLike("(xyz)", new Settings({macros: {
             "\\foo": "(#1)",
-            "\\bar": "x",
+            "\\bar": "xyz",
         }}));
     });
 
+    it("should allow properly nested group for macro argument", function() {
+        expect`\foo{e^{x_{12}+3}}`.toParseLike("(e^{x_{12}+3})",
+            new Settings({macros: {"\\foo": "(#1)"}}));
+    });
+
     it("should delay expansion if preceded by \\expandafter", function() {
         expect`\expandafter\foo\bar`.toParseLike("x+y", new Settings({macros: {
             "\\foo": "#1+#2",
@@ -3064,9 +3062,8 @@ describe("A macro expander", function() {
             new Settings({macros: {"\\foo": "x"}}));
         // \frac is a macro and therefore expandable
         expect`\noexpand\frac xy`.toParseLike`xy`;
-        // TODO(ylem): #2085
         // \def is not expandable, so is not affected by \noexpand
-        // expect`\noexpand\def\foo{xy}\foo`.toParseLike`xy`;
+        expect`\noexpand\def\foo{xy}\foo`.toParseLike`xy`;
     });
 
     it("should allow for space macro argument (text version)", function() {
@@ -3104,15 +3101,11 @@ describe("A macro expander", function() {
         }}));
     });
 
-    // TODO: The following is not currently possible to get working, given that
-    // functions and macros are dealt with separately.
-/*
     it("should allow for space function arguments", function() {
         expect`\frac\bar\bar`.toParseLike(r`\frac{}{}`, new Settings({macros: {
             "\\bar": " ",
         }}));
     });
-*/
 
     it("should build \\overset and \\underset", function() {
         expect`\overset{f}{\rightarrow} Y`.toBuild();
@@ -3222,32 +3215,36 @@ describe("A macro expander", function() {
         expect`\varsubsetneqq\varsupsetneq\varsupsetneqq`.toBuild();
     });
 
-    // TODO(edemaine): This doesn't work yet.  Parses like `\text text`,
+    it("\\TextOrMath should work in a macro passed to \\text", function() {
-    // which doesn't treat all four letters as an argument.
+        expect`\text\mode`.toParseLike(r`\text{text}`, new Settings({macros:
-    //it("\\TextOrMath should work in a macro passed to \\text", function() {
+            {"\\mode": "\\TextOrMath{text}{math}"}}));
-    //    expect`\text\mode`.toParseLike(r`\text{text}`, new Settings({macros:
+    });
-    //        {"\\mode": "\\TextOrMath{text}{math}"}});
-    //});
 
     it("\\gdef defines macros", function() {
         expect`\gdef\foo{x^2}\foo+\foo`.toParseLike`x^2+x^2`;
-        expect`\gdef{\foo}{x^2}\foo+\foo`.toParseLike`x^2+x^2`;
+        expect`\gdef\foo{hi}\foo+\text\foo`.toParseLike`hi+\text{hi}`;
-        expect`\gdef\foo{hi}\foo+\text{\foo}`.toParseLike`hi+\text{hi}`;
         expect`\gdef\foo#1{hi #1}\text{\foo{Alice}, \foo{Bob}}`
             .toParseLike`\text{hi Alice, hi Bob}`;
         expect`\gdef\foo#1#2{(#1,#2)}\foo 1 2+\foo 3 4`.toParseLike`(1,2)+(3,4)`;
         expect`\gdef\foo#2{}`.not.toParse();
+        expect`\gdef\foo#a{}`.not.toParse();
         expect`\gdef\foo#1#3{}`.not.toParse();
         expect`\gdef\foo#1#2#3#4#5#6#7#8#9{}`.toParse();
         expect`\gdef\foo#1#2#3#4#5#6#7#8#9#10{}`.not.toParse();
-        expect`\gdef\foo#{}`.not.toParse();
+        expect`\gdef\foo1`.not.toParse();
-        expect`\gdef\foo\bar`.toParse();
+        expect`\gdef{\foo}{}`.not.toParse();
+        expect`\gdef\foo\bar`.not.toParse();
         expect`\gdef{\foo\bar}{}`.not.toParse();
         expect`\gdef{}{}`.not.toParse();
-        // TODO: These shouldn't work, but `1` and `{1}` are currently treated
+    });
-        // the same, as are `\foo` and `{\foo}`.
+
-        //expect`\gdef\foo1`.not.toParse();
+    it("\\gdef defines macros with delimited parameter", function() {
-        //expect`\gdef{\foo}{}`.not.toParse();
+        expect`\gdef\foo|#1||{#1}\text{\foo| x y ||}`.toParseLike`\text{ x y }`;
+        expect`\gdef\foo#1|#2{#1+#2}\foo 1 2 |34`.toParseLike`12+34`;
+        expect`\gdef\foo#1#{#1}\foo1^{23}`.toParseLike`1^{23}`;
+        expect`\gdef\foo|{}\foo`.not.toParse();
+        expect`\gdef\foo#1|{#1}\foo1`.not.toParse();
+        expect`\gdef\foo#1|{#1}\foo1}|`.not.toParse();
     });
 
     it("\\xdef should expand definition", function() {
@@ -3344,7 +3341,7 @@ describe("A macro expander", function() {
         expect`\def\foo{1}\let\bar\foo\def\foo{2}\bar`.toParseLike`1`;
         expect`\let\foo=\kern\edef\bar{\foo1em}\let\kern=\relax\bar`.toParseLike`\kern1em`;
         // \foo = { (left brace)
-        expect`\let\foo{\frac\foo1}{2}`.toParseLike`\frac{1}{2}`;
+        expect`\let\foo{\sqrt\foo1}`.toParseLike`\sqrt{1}`;
         // \equals = = (equal sign)
         expect`\let\equals==a\equals b`.toParseLike`a=b`;
         // \foo should not be expandable and not affected by \noexpand or \edef
@@ -3534,9 +3531,9 @@ describe("\\@binrel automatic bin/rel/ord", () => {
         expect("L\\@binrel+xR").toParseLike("L\\mathbin xR");
         expect("L\\@binrel=xR").toParseLike("L\\mathrel xR");
         expect("L\\@binrel xxR").toParseLike("L\\mathord xR");
-        expect("L\\@binrel{+}{x}R").toParseLike("L\\mathbin{{x}}R");
+        expect("L\\@binrel{+}{x}R").toParseLike("L\\mathbin{x}R");
-        expect("L\\@binrel{=}{x}R").toParseLike("L\\mathrel{{x}}R");
+        expect("L\\@binrel{=}{x}R").toParseLike("L\\mathrel{x}R");
-        expect("L\\@binrel{x}{x}R").toParseLike("L\\mathord{{x}}R");
+        expect("L\\@binrel{x}{x}R").toParseLike("L\\mathord{x}R");
     });
 
     it("should base on just first character in group", () => {
@@ -3772,21 +3769,18 @@ describe("The \\mathchoice function", function() {
 });
 
 describe("Newlines via \\\\ and \\newline", function() {
-    it("should build \\\\ and \\newline the same", () => {
+    it("should build \\\\ without the optional argument and \\newline the same", () => {
         expect`hello \\ world`.toBuildLike`hello \newline world`;
-        expect`hello \\[1ex] world`.toBuildLike(
+    });
-            "hello \\newline[1ex] world");
+
+    it("should not allow \\newline to scan for an optional size argument", () => {
+        expect`hello \newline[w]orld`.toBuild();
     });
 
     it("should not allow \\cr at top level", () => {
         expect`hello \cr world`.not.toBuild();
     });
 
-    it("array redefines and resets \\\\", () => {
-        expect`a\\b\begin{matrix}x&y\\z&w\end{matrix}\\c`
-            .toParseLike`a\newline b\begin{matrix}x&y\cr z&w\end{matrix}\newline c`;
-    });
-
     it("\\\\ causes newline, even after mrel and mop", () => {
         const markup = katex.renderToString(r`M = \\ a + \\ b \\ c`);
         // Ensure newlines appear outside base spans (because, in this regexp,

website/pages/index.html

@@ -62,9 +62,9 @@
             </a>
           </div>
           <textarea id="demo-input" spellcheck="false">
-% \f is defined as f(#1) using the macro
+% \f is defined as #1f(#2) using the macro
-\f{x} = \int_{-\infty}^\infty
+\f\relax{x} = \int_{-\infty}^\infty
-    \hat \f\xi\,e^{2 \pi i \xi x}
+    \f\hat\xi,e^{2 \pi i \xi x}
     \,d\xi</textarea>
         </div>
         <div class="demo-right">
@@ -181,7 +181,7 @@
       <h4><label for="macros">macros</label></h4>
       <textarea id="macros" placeholder="JSON">
 {
-    "\\f": "f(#1)"
+    "\\f": "#1f(#2)"
 }</textarea>
 
       <h3>Editor Options</h3>

website/sidebars.json

@@ -3,6 +3,6 @@
     "Installation": ["node", "browser"],
     "Usage": ["api", "cli", "autorender", "libs"],
     "Configuring KaTeX": ["options", "security", "error", "font"],
-    "Misc": ["supported", "support_table", "issues"]
+    "Misc": ["supported", "support_table", "issues", "migration"]
   }
 }