Add support for \expandafter, \noexpand, \edef, \let, and \long (#2122)

* Add support for \expandafter

* Add support for \noexpand

* Add support for \edef

* Update comments

* Allow \long before macro definition

* Update documentation

* Update comments

* Fix defPrefix

* Add support for \let

* Update documentation

* Print error token

* Update documentation

* Check whether command is expandable

* Add tests

* Fix token order

* Make noexpand a Token property

* Throw error if control sequence is undefined when expanding

* Rename expandableOnly to expandOnly

* Make unexpandable macro property

* Move \expandafter to macros.js

* Add TODO

* Fix merge conflict

* Update a test case

* Remove unused functions in MacroContextInterface

* Update comments

* Refactor code

* Move \noexpand to macros

* Update MacroExpander.js

* Add a test case

* Separate control sequence check to a function

* Add support for \futurelet

* Separate RHS getter to a function

* Update documentation

* Move expandOnly logic to expandOnce

* Refactor code and update comments

Co-authored-by: Kevin Barabash <kevinb@khanacademy.org>

src/functions/def.js

@@ -4,37 +4,87 @@ import ParseError from "../ParseError";
 import {assertNodeType} from "../parseNode";
 
 const globalMap = {
+    "\\global": "\\global",
+    "\\long": "\\\\globallong",
+    "\\\\globallong": "\\\\globallong",
     "\\def": "\\gdef",
     "\\gdef": "\\gdef",
+    "\\edef": "\\xdef",
+    "\\xdef": "\\xdef",
+    "\\let": "\\\\globallet",
+    "\\futurelet": "\\\\globalfuture",
 };
 
-// Basic support for macro definitions:
-//     \def\macro{expansion}
-//     \def\macro#1{expansion}
-//     \def\macro#1#2{expansion}
-//     \def\macro#1#2#3#4#5#6#7#8#9{expansion}
-// Also the \gdef and \global\def equivalents
+const checkControlSequence = (tok) => {
+    const name = tok.text;
+    if (/^(?:[\\{}$&#^_]|EOF)$/.test(name)) {
+        throw new ParseError("Expected a control sequence", tok);
+    }
+    return name;
+};
+
+const getRHS = (parser) => {
+    let tok = parser.gullet.popToken();
+    if (tok.text === "=") { // consume optional equals
+        tok = parser.gullet.popToken();
+        if (tok.text === " ") { // consume one optional space
+            tok = parser.gullet.popToken();
+        }
+    }
+    return tok;
+};
+
+const letCommand = (parser, name, tok, global) => {
+    let macro = parser.gullet.macros.get(tok.text);
+    if (macro == null) {
+        // don't expand it later even if a macro with the same name is defined
+        // e.g., \let\foo=\frac \def\frac{\relax} \frac12
+        tok.noexpand = true;
+        macro = {
+            tokens: [tok],
+            numArgs: 0,
+            // reproduce the same behavior in expansion
+            unexpandable: !parser.gullet.isExpandable(tok.text),
+        };
+    }
+    parser.gullet.macros.set(name, macro, global);
+};
+
+// <assignment> -> <non-macro assignment>|<macro assignment>
+// <non-macro assignment> -> <simple assignment>|\global<non-macro assignment>
+// <macro assignment> -> <definition>|<prefix><macro assignment>
+// <prefix> -> \global|\long|\outer
 defineFunction({
     type: "internal",
-    names: ["\\global"],
+    names: [
+        "\\global", "\\long",
+        "\\\\globallong", // can’t be entered directly
+    ],
     props: {
         numArgs: 0,
         allowedInText: true,
     },
-    handler({parser}) {
+    handler({parser, funcName}) {
         parser.consumeSpaces();
         const token = parser.fetch();
         if (globalMap[token.text]) {
-            token.text = globalMap[token.text];
+            // KaTeX doesn't have \par, so ignore \long
+            if (funcName === "\\global" || funcName === "\\\\globallong") {
+                token.text = globalMap[token.text];
+            }
             return assertNodeType(parser.parseFunction(), "internal");
         }
-        throw new ParseError(`Invalid token after \\global`, token);
+        throw new ParseError(`Invalid token after macro prefix`, token);
     },
 });
 
+// Basic support for macro definitions: \def, \gdef, \edef, \xdef
+// <definition> -> <def><control sequence><definition text>
+// <def> -> \def|\gdef|\edef|\xdef
+// <definition text> -> <parameter text><left brace><balanced text><right brace>
 defineFunction({
     type: "internal",
-    names: ["\\def", "\\gdef"],
+    names: ["\\def", "\\gdef", "\\edef", "\\xdef"],
     props: {
         numArgs: 0,
         allowedInText: true,
@@ -65,11 +115,15 @@ defineFunction({
             }
             arg = parser.gullet.consumeArgs(1)[0];
         }
+        if (funcName === "\\edef" || funcName === "\\xdef") {
+            arg = parser.gullet.expandTokens(arg);
+            arg.reverse(); // to fit in with stack order
+        }
         // Final arg is the expansion of the macro
         parser.gullet.macros.set(name, {
             tokens: arg,
             numArgs,
-        }, funcName === "\\gdef");
+        }, funcName === globalMap[funcName]);
 
         return {
             type: "internal",
@@ -77,3 +131,54 @@ defineFunction({
         };
     },
 });
+
+// <simple assignment> -> <let assignment>
+// <let assignment> -> \futurelet<control sequence><token><token>
+//     | \let<control sequence><equals><one optional space><token>
+// <equals> -> <optional spaces>|<optional spaces>=
+defineFunction({
+    type: "internal",
+    names: [
+        "\\let",
+        "\\\\globallet", // can’t be entered directly
+    ],
+    props: {
+        numArgs: 0,
+        allowedInText: true,
+    },
+    handler({parser, funcName}) {
+        const name = checkControlSequence(parser.gullet.popToken());
+        parser.gullet.consumeSpaces();
+        const tok = getRHS(parser);
+        letCommand(parser, name, tok, funcName === "\\\\globallet");
+        return {
+            type: "internal",
+            mode: parser.mode,
+        };
+    },
+});
+
+// ref: https://www.tug.org/TUGboat/tb09-3/tb22bechtolsheim.pdf
+defineFunction({
+    type: "internal",
+    names: [
+        "\\futurelet",
+        "\\\\globalfuture", // can’t be entered directly
+    ],
+    props: {
+        numArgs: 0,
+        allowedInText: true,
+    },
+    handler({parser, funcName}) {
+        const name = checkControlSequence(parser.gullet.popToken());
+        const middle = parser.gullet.popToken();
+        const tok = parser.gullet.popToken();
+        letCommand(parser, name, tok, funcName === "\\\\globalfuture");
+        parser.gullet.pushToken(tok);
+        parser.gullet.pushToken(middle);
+        return {
+            type: "internal",
+            mode: parser.mode,
+        };
+    },
+});