Collect code coverage in screenshotter tests (#1644)

* Collect code coverage in screenshotter tests

* Using instanbul, no additional dependencies required as they are part 
of jest
* Collect in CircleCI builds and upload to Codecov

* Remove clover report

.circleci/config.yml

@@ -31,6 +31,11 @@ yarn_install: &yarn_install
     name: Install dependencies
     command: yarn
 
+codecov: &codecov
+  run:
+    name: Upload code coverage reports to Codecov
+    command: ./node_modules/.bin/codecov
+
 screenshotter: &screenshotter
   steps:
     - checkout
@@ -47,7 +52,8 @@ screenshotter: &screenshotter
 
     - run:
         name: Verify screenshots and generate diffs and new screenshots
-        command: node dockers/screenshotter/screenshotter.js --selenium-ip localhost -b $CIRCLE_JOB --verify --diff --new
+        command: node dockers/screenshotter/screenshotter.js --selenium-ip localhost -b $CIRCLE_JOB --verify --diff --new --coverage
+    - *codecov
 
     - store_artifacts:
         path: test/screenshotter/new
@@ -98,9 +104,7 @@ jobs:
       - run:
           name: Run tests
           command: yarn test --coverage
-      - run:
-          name: Upload code coverage reports to Codecov
-          command: ./node_modules/.bin/codecov
+      - *codecov
 
       - run:
           name: Build KaTeX

dockers/screenshotter/screenshotter.js

@@ -11,6 +11,9 @@ const path = require("path");
 const selenium = require("selenium-webdriver");
 const firefox = require("selenium-webdriver/firefox");
 
+const istanbulApi = require('istanbul-api');
+const istanbulLibCoverage = require('istanbul-lib-coverage');
+
 const webpack = require('webpack');
 const WebpackDevServer = require("webpack-dev-server");
 const webpackConfig = require("../../webpack.dev")[0];
@@ -48,6 +51,7 @@ const opts = require("commander")
     .option("--diff", "With `--verify`, produce image diffs when match fails")
     .option("--new",
         "With `--verify`, generate new screenshots when match fails")
+    .option("--coverage", "Collect and report test coverage information")
     .option("--attempts <n>",
         "Retry this many times before reporting failure", 5, parseInt)
     .option("--wait <secs>",
@@ -151,6 +155,7 @@ let attempts = 0;
 // Start up development server
 
 let devServer = null;
+let coverageMap;
 const minPort = 32768;
 const maxPort = 61000;
 
@@ -160,6 +165,19 @@ function startServer() {
         return;
     }
     const port = Math.floor(Math.random() * (maxPort - minPort)) + minPort;
+
+    if (opts.coverage) {
+        coverageMap = istanbulLibCoverage.createCoverageMap({});
+        webpackConfig.module.rules[0].use = {
+            loader: 'babel-loader',
+            options: {
+                plugins: [['istanbul', {
+                    include: ["src/**/*.js"],
+                    exclude: ["src/unicodeMake.js"],
+                }]],
+            },
+        };
+    }
     const compiler = webpack(webpackConfig);
     const wds = new WebpackDevServer(compiler, webpackConfig.devServer);
     const server = wds.listen(port);
@@ -364,11 +382,25 @@ function takeScreenshot(key) {
                     "handle_search_string(" +
                     JSON.stringify("?" + itm.query) + ", callback);")
                 .then(waitThenScreenshot);
+        } else if (opts.coverage) {
+            // collect coverage before reloading
+            collectCoverage().then(function() {
+                return driver.get(url).then(waitThenScreenshot);
+            });
         } else {
             driver.get(url).then(waitThenScreenshot);
         }
     }
 
+    function collectCoverage() {
+        return driver.executeScript('return window.__coverage__;')
+            .then(function(result) {
+                if (result) {
+                    coverageMap.merge(result);
+                }
+            });
+    }
+
     function waitThenScreenshot() {
         driverReady = true;
         if (opts.wait) {
@@ -476,6 +508,16 @@ function takeScreenshot(key) {
             if (opts.new) {
                 console.log("New screenshots have been generated in: " + newDir);
             }
+            if (opts.coverage) {
+                collectCoverage().then(function() {
+                    const reporter = istanbulApi.createReporter();
+                    reporter.addAll(['json', 'text', 'lcov']);
+                    reporter.write(coverageMap);
+
+                    process.exit(exitStatus);
+                });
+                return;
+            }
             // devServer.close(cb) will take too long.
             process.exit(exitStatus);
         }

package.json

@@ -21,6 +21,7 @@
     "babel-eslint": "^8.1.2",
     "babel-jest": "^23.0.1",
     "babel-loader": "^7.1.4",
+    "babel-plugin-istanbul": "^4.1.6",
     "babel-plugin-transform-class-properties": "^6.23.0",
     "babel-plugin-transform-runtime": "^6.15.0",
     "babel-plugin-version-inline": "^1.0.0",
@@ -40,6 +41,8 @@
     "fs-extra": "^7.0.0",
     "greenkeeper-lockfile": "^1.15.1",
     "husky": "^1.0.0-rc.8",
+    "istanbul-api": "^1.3.1",
+    "istanbul-lib-coverage": "^1.2.0",
     "jest": "^23.0.1",
     "jest-serializer-html": "^5.0.0",
     "js-yaml": "^3.10.0",