Add catcode to Lexer, move comment parsing back to Lexer (#1789)

* Remove redundant consumeSpaces()

- Spaces after command sequence are ignored in Lexer
- parseExpression consumes spaces in the math mode

* Add catcode to Lexer, move comment parsing back to Lexer

- Fix parsing a comment before a sup/subscript argument
- Fix parsing a comment before an expression
- Fix parsing a comment before or between \hline
- Fix parsing a comment in the macro definition
- Fix parsing a comment including a command sequence

* Update Lexer.js

* Update Parser.js

* catcode -> catcodes

test/katex-spec.js

@@ -1627,6 +1627,8 @@ describe("A comment parser", function() {
 
     it("should parse comments between subscript and superscript", () => {
         expect("x_3 %comment\n^2").toParseLike`x_3^2`;
+        expect("x^ %comment\n{2}").toParseLike`x^{2}`;
+        expect("x^ %comment\n\\frac{1}{2}").toParseLike`x^\frac{1}{2}`;
     });
 
     it("should parse comments in size and color groups", () => {
@@ -1635,6 +1637,24 @@ describe("A comment parser", function() {
         expect("\\color{#f00%red\n}").toParse();
     });
 
+    it("should parse comments before an expression", () => {
+        expect("%comment\n{2}").toParseLike`{2}`;
+    });
+
+    it("should parse comments before and between \\hline", () => {
+        expect("\\begin{matrix}a&b\\\\ %hline\n" +
+            "\\hline %hline\n" +
+            "\\hline c&d\\end{matrix}").toParse();
+    });
+
+    it("should parse comments in the macro definition", () => {
+        expect("\\def\\foo{1 %}\n2}\n\\foo").toParseLike`12`;
+    });
+
+    it("should not expand nor ignore spaces after a command sequence in a comment", () => {
+        expect("\\def\\foo{1\n2}\nx %\\foo\n").toParseLike`x`;
+    });
+
     it("should not parse a comment without newline in strict mode", () => {
         expect`x%y`.not.toParse(strictSettings);
         expect`x%y`.toParse(nonstrictSettings);
@@ -2586,9 +2606,8 @@ describe("href and url commands", function() {
 
     it("should allow single-character URLs", () => {
         expect`\href%end`.toParseLike("\\href{%}end");
-        expect`\href %end`.toParseLike("\\href{%}end");
         expect("\\url%end").toParseLike("\\url{%}end");
-        expect("\\url %end").toParseLike("\\url{%}end");
+        expect("\\url%%end\n").toParseLike("\\url{%}");
         expect("\\url end").toParseLike("\\url{e}nd");
         expect("\\url%end").toParseLike("\\url {%}end");
     });
@@ -2630,6 +2649,10 @@ describe("href and url commands", function() {
         expect(parsed2.href).toBe(url);
     });
 
+    it("should allow comments after URLs", function() {
+        expect("\\url{http://example.com/}%comment\n").toBuild();
+    });
+
     it("should be marked up correctly", function() {
         const markup = katex.renderToString(r`\href{http://example.com/}{example here}`);
         expect(markup).toContain("<a href=\"http://example.com/\">");