Create visual test system

contributor_docs/unit_testing.md

@@ -118,3 +118,24 @@ test('keyIsPressed is a boolean', function() {
 
 Similarly we can use `assert.strictEqual(myp5.keyIsPressed, true)` to assert if the value is true. You can read more about chai's assert [here](https://www.chaijs.com/api/assert/)
 Now that you have written the tests, run them and see if the method behaves as expected. If not, create an issue for the same and if you want, you can even work on fixing it!
+
+## Visual tests
+
+Visual tests are a way to make sure sketches do not unexpectedly change when we change the implementation of p5.js features. Each visual test file lives in the `test/unit/visual/cases` folder. Inside each file there are multiple visual test cases. Each case creates a sample sketch, and then calls `screenshot()` to check how the sketch looks.
+
+```js
+visualTest('2D objects maintain correct size', function(p5, screenshot) {
+  p5.createCanvas(50, 50, p5.WEBGL);
+  p5.noStroke();
+  p5.fill('red');
+  p5.rectMode(p5.CENTER);
+  p5.rect(0, 0, p5.width/2, p5.height/2);
+  screenshot();
+});
+```
+
+If you need to add a new test file, add it to that folder, then add the filename to the list in `test/visual/visualTestList.js`. Additionally, if you want that file to be run automatically as part of continuous integration on every pull request, add the filename to the `visual` list in `test/unit/spec.js`.
+
+When you add a new test, running `npm test` will generate new screenshots for any visual tests that do not yet have them. Those screenshots will then be used as a reference the next time tests run to make sure the sketch looks the same. If a test intentionally needs to look different, you can delete the folder matching the test name in the `test/unit/visual/screenshots` folder, and then re-run `npm test` to generate a new one.
+
+To manually inspect all visual tests, run `grunt yui:dev` to launch a local server, then go to http://127.0.0.1:9001/test/visual.html to see a list of all test cases.

tasks/test/mocha-chrome.js

@@ -4,6 +4,7 @@ const puppeteer = require('puppeteer');
 const util = require('util');
 const mapSeries = require('promise-map-series');
 const fs = require('fs');
+const path = require('path');
 const EventEmitter = require('events');
 
 const mkdir = util.promisify(fs.mkdir);
@@ -28,6 +29,28 @@ module.exports = function(grunt) {
         const page = await browser.newPage();
 
         try {
+          // Set up visual tests
+          await page.evaluateOnNewDocument(function(shouldGenerateScreenshots) {
+            window.shouldGenerateScreenshots = shouldGenerateScreenshots;
+          }, !process.env.CI);
+
+          await page.exposeFunction('writeImageFile', function(filename, base64Data) {
+            fs.mkdirSync('test/' + path.dirname(filename), { recursive: true });
+            const prefix = /^data:image\/\w+;base64,/;
+            fs.writeFileSync(
+              'test/' + filename,
+              base64Data.replace(prefix, ''),
+              'base64'
+            );
+          });
+          await page.exposeFunction('writeTextFile', function(filename, data) {
+            fs.mkdirSync('test/' + path.dirname(filename), { recursive: true });
+            fs.writeFileSync(
+              'test/' + filename,
+              data
+            );
+          });
+
           // Using eval to start the test in the browser
           // A 'mocha:end' event will be triggered with test runner end
           await page.evaluateOnNewDocument(`

test/unit/spec.js

@@ -47,8 +47,16 @@ var spec = {
     'p5.Shader',
     'p5.Texture',
     'light'
+  ],
+  'visual/cases': [
+    // Add the visual tests that you want run as part of CI here. Feel free
+    // to omit some for speed if they should only be run manually.
+    'webgl'
   ]
 };
+document.write(
+  '<script src="unit/visual/visualTest.js" type="text/javascript"></script>'
+);
 Object.keys(spec).map(function(folder) {
   spec[folder].map(function(file) {
     var string = [

test/unit/visual/cases/webgl.js

@@ -0,0 +1,37 @@
+visualSuite('WebGL', function() {
+  visualSuite('Camera', function() {
+    visualTest('2D objects maintain correct size', function(p5, screenshot) {
+      p5.createCanvas(50, 50, p5.WEBGL);
+      p5.noStroke();
+      p5.fill('red');
+      p5.rectMode(p5.CENTER);
+      p5.rect(0, 0, p5.width/2, p5.height/2);
+      screenshot();
+    });
+
+    visualTest('Custom camera before and after resize', function(p5, screenshot) {
+      p5.createCanvas(25, 50, p5.WEBGL);
+      const cam = p5.createCamera();
+      p5.setCamera(cam);
+      cam.setPosition(-10, -10, 800);
+      p5.strokeWeight(4);
+      p5.box(20);
+      screenshot();
+
+      p5.resizeCanvas(50, 25);
+      p5.box(20);
+      screenshot();
+    });
+  });
+
+  visualSuite('Lights', function() {
+    visualTest('Fill color and default ambient material', function(p5, screenshot) {
+      p5.createCanvas(50, 50, p5.WEBGL);
+      p5.noStroke();
+      p5.lights();
+      p5.fill('red');
+      p5.sphere(20);
+      screenshot();
+    });
+  });
+});

test/unit/visual/screenshots/WebGL/Camera/2D objects maintain correct size/metadata.json

@@ -0,0 +1,3 @@
+{
+  "numScreenshots": 1
+}

test/unit/visual/screenshots/WebGL/Camera/Custom camera before and after resize/metadata.json

@@ -0,0 +1,3 @@
+{
+  "numScreenshots": 2
+}

test/unit/visual/screenshots/WebGL/Lights/Fill color and default ambient material/metadata.json

@@ -0,0 +1,3 @@
+{
+  "numScreenshots": 1
+}

test/unit/visual/visualTest.js

@@ -0,0 +1,196 @@
+/**
+ * A helper class to contain an error and also the screenshot data that
+ * caused the error.
+ */
+class ScreenshotError extends Error {
+  constructor(message, actual, expected) {
+    super(message);
+    this.actual = actual;
+    this.expected = expected;
+  }
+}
+
+function toBase64(img) {
+  return img.canvas.toDataURL();
+}
+
+function escapeName(name) {
+  // Encode slashes as `encodeURIComponent('/')`
+  return name.replace(/\//g, '%2F');
+}
+
+let namePrefix = '';
+
+/**
+ * A helper to define a category of visual tests.
+ *
+ * @param name The name of the category of test.
+ * @param callback A callback that calls `visualTest` a number of times to define
+ * visual tests within this suite.
+ * @param [options] An options object with optional additional settings. Set its
+ * key `focus` to true to only run this test, or its `skip` key to skip it.
+ */
+window.visualSuite = function(
+  name,
+  callback,
+  { focus = false, skip = false } = {}
+) {
+  const lastPrefix = namePrefix;
+  namePrefix += escapeName(name) + '/';
+
+  let suiteFn = suite;
+  if (focus) {
+    suiteFn = suiteFn.only;
+  }
+  if (skip) {
+    suiteFn = suiteFn.skip;
+  }
+  suiteFn(name, callback);
+
+  namePrefix = lastPrefix;
+};
+
+window.checkMatch = function(actual, expected, p5) {
+  const maxSide = 50;
+  const scale = Math.min(maxSide/expected.width, maxSide/expected.height);
+  for (const img of [actual, expected]) {
+    img.resize(
+      Math.ceil(img.width * scale),
+      Math.ceil(img.height * scale)
+    );
+  }
+  const diff = p5.createImage(actual.width, actual.height);
+  diff.drawingContext.drawImage(actual.canvas, 0, 0);
+  diff.drawingContext.globalCompositeOperation = 'difference';
+  diff.drawingContext.drawImage(expected.canvas, 0, 0);
+  diff.filter(p5.ERODE, false);
+  diff.loadPixels();
+
+  let ok = true;
+  for (let i = 0; i < diff.pixels.length; i++) {
+    if (i % 4 === 3) continue; // Skip alpha checks
+    if (Math.abs(diff.pixels[i]) > 10) {
+      ok = false;
+      break;
+    }
+  }
+  return { ok, diff };
+};
+
+/**
+ * A helper to define a visual test, where we will assert that a sketch matches
+ * screenshots saved ahead of time of what the test should look like.
+ *
+ * When defining a new test, run the tests once to generate initial screenshots.
+ *
+ * To regenerate screenshots for a test, delete its screenshots folder in
+ * the test/unit/visual/screenshots directory, and rerun the tests.
+ *
+ * @param testName The display name of a test. This also links the test to its
+ * expected screenshot, so make sure to rename the screenshot folder after
+ * renaming a test.
+ * @param callback A callback to set up the test case. It takes two parameters:
+ * first is `p5`, a reference to the p5 instance, and second is `screenshot`, a
+ * function to grab a screenshot of the canvas. It returns either nothing, or a
+ * Promise that resolves when all screenshots have been taken.
+ * @param [options] An options object with optional additional settings. Set its
+ * key `focus` to true to only run this test, or its `skip` key to skip it.
+ */
+window.visualTest = function(
+  testName,
+  callback,
+  { focus = false, skip = false } = {}
+) {
+  const name = namePrefix + escapeName(testName);
+  let suiteFn = suite;
+  if (focus) {
+    suiteFn = suiteFn.only;
+  }
+  if (skip) {
+    suiteFn = suiteFn.skip;
+  }
+
+  suiteFn(testName, function() {
+    let myp5;
+
+    setup(function() {
+      return new Promise(res => {
+        myp5 = new p5(function(p) {
+          p.setup = function() {
+            res();
+          };
+        });
+      });
+    });
+
+    teardown(function() {
+      myp5.remove();
+    });
+
+    test('matches expected screenshots', async function() {
+      let expectedScreenshots;
+      try {
+        metadata = await fetch(
+          `unit/visual/screenshots/${name}/metadata.json`
+        ).then(res => res.json());
+        expectedScreenshots = metadata.numScreenshots;
+      } catch (e) {
+        expectedScreenshots = 0;
+      }
+
+      if (!window.shouldGenerateScreenshots && !expectedScreenshots) {
+        // If running on CI, all expected screenshots should already
+        // be generated
+        throw new Error('No expected screenshots found');
+      }
+
+      const actual = [];
+
+      // Generate screenshots
+      await callback(myp5, () => {
+        actual.push(myp5.get());
+      });
+
+      if (expectedScreenshots && actual.length !== expectedScreenshots) {
+        throw new Error(
+          `Expected ${expectedScreenshots} screenshot(s) but generated ${actual.length}`
+        );
+      }
+      if (!expectedScreenshots) {
+        writeTextFile(
+          `unit/visual/screenshots/${name}/metadata.json`,
+          JSON.stringify({ numScreenshots: actual.length }, null, 2)
+        );
+      }
+
+      const expectedFilenames = actual.map(
+        (_, i) => `unit/visual/screenshots/${name}/${i.toString().padStart(3, '0')}.png`
+      );
+      const expected = expectedScreenshots
+        ? (
+          await Promise.all(
+            expectedFilenames.map(path => new Promise((resolve, reject) => {
+              myp5.loadImage(path, resolve, reject);
+            }))
+          )
+        )
+        : [];
+
+      for (let i = 0; i < actual.length; i++) {
+        if (expected[i]) {
+          if (!checkMatch(actual[i], expected[i], myp5).ok) {
+            throw new ScreenshotError(
+              `Screenshots do not match! Expected:\n${toBase64(expected[i])}\n\nReceived:\n${toBase64(actual[i])}\n\n` +
+              'If this is unexpected, paste these URLs into your browser to inspect them, or run grunt yui:dev and go to http://127.0.0.1:9001/test/visual.html.\n\n' +
+              `If this change is expected, please delete the test/unit/visual/screenshots/${name} folder and run tests again to generate a new screenshot.`,
+              actual[i],
+              expected[i]
+            );
+          }
+        } else {
+          writeImageFile(expectedFilenames[i], toBase64(actual[i]));
+        }
+      }
+    });
+  });
+};

test/visual.html

@@ -0,0 +1,16 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta http-equiv="Content-type" content="text/html; charset=utf-8">
+    <title>p5.js Visual Test Runner</title>
+    <link rel="stylesheet" type="text/css" href="visual/style.css" />
+  </head>
+  <body>
+    <h1>p5.js Visual Test Runner</h1>
+    <p id="metrics"></p>
+    <script src="../../lib/p5.js" type="text/javascript"></script>
+    <script src="unit/visual/visualTest.js" type="text/javascript"></script>
+    <script src="visual/visualTestRunner.js" type="text/javascript"></script>
+    <script src="visual/visualTestList.js" type="text/javascript"></script>
+  </body>
+</html>

test/visual/style.css

@@ -0,0 +1,45 @@
+body {
+  font-family: sans-serif;
+}
+
+h4 {
+  font-weight: normal;
+  margin-bottom: 10px;
+  margin-top: 0;
+}
+
+#metrics {
+  color: #777;
+  text-decoration: italic;
+}
+
+.suite {
+  padding-left: 10px;
+  border-left: 2px solid rgba(0,0,0,0.2);
+  margin-bottom: 30px;
+}
+.skipped {
+  opacity: 0.5;
+}
+.suite.focused {
+  border-left-color: #2B2;
+}
+.suite.failed {
+  border-left-color: #F00;
+}
+
+.failed {
+  color: #F00;
+}
+
+.screenshot img {
+  border: 2px solid #000;
+  margin-right: 5px;
+}
+.screenshot.failed img {
+  border-color: #F00;
+}
+
+.diff {
+  background: #000;
+}