Update texcmp to ubuntu 17.04 and avoid mounted host directory (#722)

* Update texcmp to ubuntu 17.04 and avoid mounted host directory

Switch to linux 17.04 in order to have a version of nodejs which understands
ES6 syntax, for the sake of a consistent codebase.
Avoid mounting the host directory, but use “docker cp” instead to transfer
files between host and container.  This should avoid ownership and
permission issues.
Support macros with positional arguments.
Fix one overline example which caused LaTeX failure due to missing braces.

* Extract texcmp results as current user

This allows running the texcmp.sh script using sudo on afs.

* Change texcmp conversion to gray

As per #708, this should increase compatibility with older versions of
imagemagick, and might also do a better job of preserving the original sRGB
color space.

* Abandon tar and use plain docker cp instead

Thanks to Erik Demaine for suggesting this.

* Move npm install into creation of texcmp docker image

dockers/texcmp/Dockerfile

@@ -1,11 +1,36 @@
-# convert from PDF to PNG with -flatten looks really bad on 14.04 LTS
-FROM ubuntu:15.04
+FROM ubuntu:17.04
 
 MAINTAINER Martin von Gagern <gagern@ma.tum.de>
 
-# Disable regular updates, but keep security updates
-RUN sed -i 's/^\(deb.*updates\)/#\1/' /etc/apt/sources.list && apt-get update
+ADD https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64 /usr/bin/dumb-init
 
-# Install all required packages, but try not to pull in TOO much
-RUN apt-get -qy --no-install-recommends install \
-  texlive-latex-base etoolbox imagemagick ghostscript nodejs
+# Disable regular updates, keep security updates, avoid intermediate layers
+RUN sed -i 's/^\(deb.*updates\)/#\1/' /etc/apt/sources.list \
+ && apt-get update \
+ && apt-get upgrade -y \
+ && DEBIAN_FRONTEND=noninteractive \
+    apt-get install -qy --no-install-recommends \
+        ca-certificates \
+        etoolbox \
+        ghostscript \
+        imagemagick \
+        nodejs \
+        npm \
+        texlive-fonts-recommended \
+        texlive-latex-base \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/* \
+ && chmod +x /usr/bin/dumb-init
+
+ENTRYPOINT ["/usr/bin/dumb-init", "--"]
+
+RUN useradd --create-home --home-dir /KaTeX katex \
+ && mkdir -p /KaTeX/dockers/texcmp /KaTeX/test/screenshotter
+
+ADD package.json /KaTeX/dockers/texcmp/package.json
+
+RUN ( cd /KaTeX/dockers/texcmp; npm install; ) \
+ && ( cd /KaTeX/test/screenshotter; npm install js-yaml; ) \
+ && chown -R katex:katex /KaTeX
+
+USER katex

dockers/texcmp/package.json

@@ -3,6 +3,7 @@
   "description": "KaTeX helper to compare LaTeX output against screenshots",
   "license": "MIT",
   "dependencies": {
+    "ndarray": "1.0.18",
     "ndarray-fft": "1.0.0",
     "pngparse": "2.0.1",
     "q": "1.4.1"

dockers/texcmp/texcmp.js

@@ -8,7 +8,7 @@ const path = require("path");
 const Q = require("q"); // To debug, pass Q_DEBUG=1 in the environment
 const pngparse = require("pngparse");
 const fft = require("ndarray-fft");
-const ndarray = require("ndarray-fft/node_modules/ndarray");
+const ndarray = require("ndarray");
 
 const data = require("../../test/screenshotter/ss_data");
 
@@ -69,6 +69,23 @@ function processTestCase(key) {
     if (itm.post) {
         tex = tex + itm.post.replace("<br>", "\\\\");
     }
+    if (itm.macros) {
+        tex = Object.keys(itm.macros).map(name => {
+            const expansion = itm.macros[name];
+            let numArgs = 0;
+            if (expansion.indexOf("#") !== -1) {
+                const stripped = expansion.replace(/##/g, "");
+                while (stripped.indexOf("#" + (numArgs + 1)) !== -1) {
+                    ++numArgs;
+                }
+            }
+            let args = "";
+            for (let i = 1; i <= numArgs; ++i) {
+                args += "#" + i;
+            }
+            return "\\def" + name + args + "{" + expansion + "}\n";
+        }).join("") + tex;
+    }
     tex = template.replace(/\$.*\$/, tex.replace(/\$/g, "$$$$"));
     const texFile = path.join(tmpDir, key + ".tex");
     const pdfFile = path.join(tmpDir, key + ".pdf");
@@ -156,11 +173,11 @@ function processTestCase(key) {
         const uh = Math.max(browser.height + by, latex.height + ly); // u. h.
         return execFile("convert", [
             // First image: latex rendering, converted to grayscale and padded
-            "(", pngFile, "-grayscale", "Rec709Luminance",
+            "(", pngFile, "-colorspace", "Gray",
             "-extent", uw + "x" + uh + "-" + lx + "-" + ly,
             ")",
             // Second image: browser screenshot, to grayscale and padded
-            "(", browserFile, "-grayscale", "Rec709Luminance",
+            "(", browserFile, "-colorspace", "Gray",
             "-extent", uw + "x" + uh + "-" + bx + "-" + by,
             ")",
             // Third image: the per-pixel minimum of the first two images

dockers/texcmp/texcmp.sh

@@ -2,16 +2,33 @@
 
 set -x
 imgname=katex/texcmp
-tag=1.1
+tag=1.3
 imgid=$(docker images | awk "/${imgname//\//\\/} *${tag//./\\.}/{print \$3}")
 cd "$(dirname "$0")" || exit $?
-npm install || exit $?
 if [[ -z ${imgid} ]]; then
     docker build -t "${imgname}:${tag}" . || exit $?
 fi
-base=$(cd ../..; pwd)
-docker run --rm \
-       -v "${base}":/KaTeX \
-       -w /KaTeX/dockers/texcmp \
-       "${imgname}:${tag}" \
-       nodejs texcmp.js "$@"
+
+cd ../..
+mkdir -p test/screenshotter/{tex,diff}
+
+cleanup() {
+    [[ "${container}" ]] \
+        && docker stop "${container}" >/dev/null \
+        && docker rm "${container}" >/dev/null
+    container=
+}
+container=$(docker create -t -u 0:0 \
+    -w /KaTeX/dockers/texcmp \
+    "${imgname}:${tag}" \
+    nodejs texcmp.js "$@")
+trap cleanup EXIT
+tar c dockers/texcmp/{texcmp.js,package.json} \
+    test/screenshotter/{ss_data.{js,yaml},images/*-firefox.png,test.tex} \
+    | docker cp - "${container}:/KaTeX"
+docker start -a "${container}"
+docker cp "${container}:/KaTeX/test/screenshotter/tex" test/screenshotter \
+    || exit $?
+docker cp "${container}:/KaTeX/test/screenshotter/diff" test/screenshotter \
+    || exit $?
+cleanup

test/screenshotter/ss_data.yaml

@@ -104,7 +104,7 @@ LimitControls: |
     \displaystyle\int\limits_2^3 3x^2\,dx + \sum\nolimits^n_{i=1}i +
     \textstyle\int\limits_x^y z
 MathAtom: a\mathrel{\mathop{=}\limits^{\blue ?}}b
-MathAtom2: \mathop{\overline\mathrm{lim}}\limits_{x\to\infty}f(x)
+MathAtom2: \mathop{\overline{\mathrm{lim}}}\limits_{x\to\infty}f(x)
 MathDefaultFonts: Ax2k\breve{a}\omega\Omega\imath+\KaTeX
 MathBb: \mathbb{Ax2k\breve{a}\omega\Omega\imath+\KaTeX}
 MathBf: \mathbf{Ax2k\breve{a}\omega\Omega\imath+\KaTeX}