chore(screenshotter): support Browserstack and test on Safari 13.1 (#2306)

* Support Browserstack in screenshotter

* Support Browserstack in CircleCI

* Test screenshots on Safari 13.1

* Don't run Safari screenshotter on forked PRs

* Update a screenshot

* Ignore *.log in .gitignore

.circleci/config.yml

@@ -1,6 +1,9 @@
 version: 2.1
 
 executors:
+  node:
+    docker:
+      - image: circleci/node:10
   firefox:
     docker:
       - image: circleci/node:10
@@ -44,10 +47,19 @@ commands:
             - .yarn
             - .pnp.js
 
+  screenshot:
+    parameters:
+      flags:
+        type: string
+        default: ""
+    steps:
+      - run:
+          name: Verify screenshots and generate diffs and new screenshots
+          command: yarn node dockers/screenshotter/screenshotter.js -b $CIRCLE_JOB --verify --diff --new << parameters.flags >>
+
 jobs:
   test:
-    docker:
-      - image: circleci/node:10
+    executor: node
     steps:
       - run:
           name: Check whether the build is running on the main repository
@@ -80,6 +92,9 @@ jobs:
     parameters:
       browser:
         type: executor
+      browserstack:
+        type: string
+        default: ""
     executor: << parameters.browser >>
     steps:
       - checkout_repo
@@ -87,9 +102,17 @@ jobs:
           filter: '^docs/|^static/|^website/|^LICENSE|\.md$'
       - install_dependencies
 
-      - run:
-          name: Verify screenshots and generate diffs and new screenshots
-          command: yarn node dockers/screenshotter/screenshotter.js --selenium-ip localhost -b $CIRCLE_JOB --verify --diff --new
+      - when:
+          condition: << parameters.browserstack >>
+          steps:
+            - screenshot:
+                flags: --browserstack --selenium-capabilities '<< parameters.browserstack >>'
+      - unless:
+          condition: << parameters.browserstack >>
+          steps:
+            - screenshot:
+                flags: --selenium-ip localhost
+
       - store_artifacts:
           path: test/screenshotter/new
           destination: new
@@ -107,3 +130,17 @@ workflows:
       - screenshotter:
           name: chrome
           browser: chrome
+      - screenshotter:
+          name: safari
+          browser: node
+          browserstack: |
+              {
+                "browserName": "Safari",
+                "browser_version": "13.1",
+                "os": "OS X",
+                "os_version": "Catalina"
+              }
+          filters:
+            branches:
+              # Forked pull requests don't have access to Browserstack credentials
+              ignore: /pull\/[0-9]+/

.gitignore

@@ -1,8 +1,7 @@
 katex.tar.gz
 katex.zip
 node_modules
-npm-debug.log
-debug.log
+*.log
 last.png
 diff.png
 /.npm-install.stamp

dockers/screenshotter/screenshotter.js

@@ -15,6 +15,8 @@ const istanbulLibCoverage = require('istanbul-lib-coverage');
 const istanbulLibReport = require('istanbul-lib-report');
 const istanbulReports = require('istanbul-reports');
 
+const browserstack = require('browserstack-local');
+
 const webpack = require('webpack');
 const WebpackDevServer = require("webpack-dev-server");
 const webpackConfig = require("../../webpack.dev")[0];
@@ -59,6 +61,9 @@ const opts = require("commander")
         "Retry this many times before reporting failure", 5, parseInt)
     .option("--wait <secs>",
         "Wait this many seconds between page load and screenshot", parseFloat)
+    .option("--browserstack", "Use Browserstack. The username and access key"
+        + " should be set as enviroment variable BROWSERSTACK_USER and"
+        + " BROWSERSTACK_ACCESS_KEY")
     .parse(process.argv);
 
 let listOfCases;
@@ -81,6 +86,24 @@ let katexURL = opts.katexUrl;
 let katexIP = opts.katexIp;
 let katexPort = opts.katexPort;
 
+let bsLocal;
+if (opts.browserstack) {
+    // https://www.browserstack.com/automate/node
+    if (!seleniumURL) {
+        seleniumURL = "http://hub-cloud.browserstack.com/wd/hub";
+    }
+    // https://www.browserstack.com/local-testing/automate#test-localhost-websites
+    if (!katexIP && opts.browser === "safari") {
+        katexIP = "bs-local.com";
+    }
+    opts.seleniumCapabilities = Object.assign({
+        resolution: "1280x1024",
+        "browserstack.user": process.env.BROWSERSTACK_USER,
+        "browserstack.key": process.env.BROWSERSTACK_ACCESS_KEY,
+        "browserstack.local": true,
+    }, opts.seleniumCapabilities);
+}
+
 //////////////////////////////////////////////////////////////////////
 // Work out connection to selenium docker container
 
@@ -188,7 +211,7 @@ function startServer() {
         devServer = wds;
         katexPort = port;
         attempts = 0;
-        process.nextTick(tryConnect);
+        process.nextTick(opts.browserstack ? startBrowserstackLocal : tryConnect);
     });
     server.on("error", function(err) {
         if (devServer !== null) { // error after we started listening
@@ -201,6 +224,21 @@ function startServer() {
     });
 }
 
+// Start Browserstack Local connection
+function startBrowserstackLocal() {
+    // unique identifier for the session
+    const localIdentifier = process.env.CIRCLE_BUILD_NUM || "p" + katexPort;
+    opts.seleniumCapabilities["browserstack.localIdentifier"] = localIdentifier;
+
+    bsLocal = new browserstack.Local();
+    bsLocal.start({localIdentifier}, function(err) {
+        if (err) {
+            throw err;
+        }
+        process.nextTick(tryConnect);
+    });
+}
+
 //////////////////////////////////////////////////////////////////////
 // Wait for container to become ready
 
@@ -546,7 +584,13 @@ function takeScreenshot(key) {
     function done() {
         // devServer.close(cb) will take too long.
         driver.quit().then(() => {
-            process.exit(exitStatus);
+            if (bsLocal) {
+                bsLocal.stop(() => {
+                    process.exit(exitStatus);
+                });
+            } else {
+                process.exit(exitStatus);
+            }
         });
     }
 }

dockers/screenshotter/screenshotter.sh

@@ -17,7 +17,7 @@ cleanup() {
 container=
 trap cleanup EXIT
 status=0
-for browserTag in firefox:2.48.2 chrome:2.48.2; do
+for browserTag in firefox:3.141.59-20200525 chrome:3.141.59-20200525; do
     browser=${browserTag%:*}
     image=selenium/standalone-${browserTag}
     echo "Starting container for ${image}"
@@ -34,4 +34,22 @@ for browserTag in firefox:2.48.2 chrome:2.48.2; do
     echo "${res} taking screenshots, stopping and removing ${container:0:12}"
     cleanup
 done
+
+if [[ $BROWSERSTACK_USER ]]; then
+    echo "Creating screenshots for Safari..."
+    if yarn node "$(dirname "$0")"/screenshotter.js \
+            --browser=safari --browserstack --selenium-capabilities '{
+                "browserName": "Safari",
+                "browser_version": "13.1",
+                "os": "OS X",
+                "os_version": "Catalina"
+            }' "$@"; then
+        res=Done
+    else
+        res=Failed
+        status=1
+    fi
+    echo "${res} taking screenshots"
+fi
+
 exit ${status}

package.json

@@ -34,6 +34,7 @@
     "babel-plugin-version-inline": "^1.0.0",
     "benchmark": "^2.1.4",
     "browserslist": "^4.13.0",
+    "browserstack-local": "^1.4.5",
     "caniuse-lite": "^1.0.30001102",
     "codecov": "^3.7.1",
     "css-loader": "^4.0.0",