feat: Ability to snapshot a directory of webpages. Usage: `percy snap…

…shot directory/` (#137)

* Add wip static snpashot service

* Clean trailing slashes off passed in asset dir and baseUrl

* Install walk package

* Remove errant semicolon

* WIP static snapshot service

* Add dummy static site app for testing

* Refactor static snapshot service

* Add tests for static snapshot service

* Clean up misses from rebase

* Code cleanup

* Make the ignore regex required for the static snapshot service

* Add baseURL to snapshot url walking and clean up regex logic

* Add test static site

* Add test command for snapshot multiplatform integration testing

* Refactor snapshot file capture and ignore logic

* Comment cleanup

* Fix typos

* Add debugger logging

* Incorporate PR feedback

* Update testing static site folder names

* Build the local static site url in a seperate function

* Test the actual urls instead of just a count of urls

* Uninstall walk and install globby

* Update snapshot command to new glob syntax and naming

* Update static snapshot options interface

* Update static snapshot service to use globby instead of walk

* Add nested file for unit testing

* Update unit tests

* Remove walk types

* Code cleanup

* Implement ignore glob

* Test ignore glob

* Incorporate PR feedback

package.json

@@ -30,11 +30,12 @@
     "colors": "^1.3.2",
     "cors": "^2.8.4",
     "express": "^4.16.3",
+    "globby": "^9.2.0",
     "js-yaml": "^3.13.1",
     "percy-client": "^3.0.3",
     "puppeteer": "^1.13.0",
-    "winston": "^2.0.0",
-    "retry-axios": "^0.5.0"
+    "retry-axios": "^0.5.0",
+    "winston": "^2.0.0"
   },
   "devDependencies": {
     "@babel/core": "^7.4.0",
@@ -47,6 +48,7 @@
     "@oclif/tslint": "^3",
     "@percy/tslint": "^1.0.0",
     "@semantic-release/changelog": "^3.0.2",
+    "@semantic-release/git": "^7.0.8",
     "@types/chai": "^4.1.4",
     "@types/chai-http": "^3.0.5",
     "@types/cors": "^2.8.4",
@@ -56,24 +58,22 @@
     "@types/node": "^10.7.1",
     "@types/sinon": "^5.0.1",
     "@types/sinon-chai": "^3.2.0",
-    "@semantic-release/git": "^7.0.8",
     "browserify": "^16.2.3",
     "chai": "^4.1.2",
     "chai-http": "^4.0.0",
     "eslint-config-oclif": "^3.0.0",
-    "globby": "^9.0.0",
     "http-server": "^0.11.1",
     "husky": "^1.0.0-rc.13",
     "mocha": "^6.1.3",
     "nock": "^10.0.6",
     "npm-watch": "^0.6.0",
     "prettier": "1.17.0",
     "pryjs": "^1.0.3",
-    "semantic-release": "^15.13.3",
     "rollup": "^1.8.0",
     "rollup-plugin-babel": "^4.3.2",
     "rollup-plugin-commonjs": "^9.2.3",
     "rollup-plugin-node-resolve": "^4.0.1",
+    "semantic-release": "^15.13.3",
     "sinon": "7.2.3",
     "sinon-chai": "^3.2.0",
     "stdout-stderr": "^0.1.9",
@@ -128,6 +128,7 @@
     "test": "npm run build-client && PERCY_TOKEN=abc mocha --forbid-only \"test/**/*.test.ts\" --exclude \"test/percy-agent-client/**/*.test.ts\" --exclude \"test/integration/**/*\"",
     "test-client": "mkdir -p dist-test/ && npm run build-client-test && testem ci --file ./test/percy-agent-client/testem.js",
     "test-integration": "npm run build-client && node ./bin/run exec -- mocha test/integration/**/*.test.ts",
+    "test-snapshot-command": "./bin/run snapshot test/integration/test-static-site -b /dummy-base-url -i '(red-keep)' -c '\\.(html)$'",
     "version": "oclif-dev readme && git add README.md",
     "watch": "npm-watch"
   },

src/commands/snapshot.ts

@@ -18,18 +18,18 @@ export default class Snapshot extends PercyCommand {
   static examples = [
     '$ percy snapshot _site/',
     '$ percy snapshot _site/ --base-url "/blog"',
-    '$ percy snapshot _site/ --ignore-files "\.(blog|docs)$"',
+    '$ percy snapshot _site/ --ignore-files "/blog/drafts/**"',
   ]
 
   static flags = {
     'snapshot-files': flags.string({
-      char: 'c',
-      description: 'Regular expression for matching the files to snapshot.',
-      default: '\.(html|htm)$',
+      char: 's',
+      description: 'Glob or comma-seperated string of globs for matching the files and directories to snapshot.',
+      default: '**/*.html,**/*.htm',
     }),
     'ignore-files': flags.string({
       char: 'i',
-      description: 'Regular expression for matching the files to ignore.',
+      description: 'Glob or comma-seperated string of globs for matching the files and directories to ignore.',
       default: '',
     }),
     'base-url': flags.string({
@@ -56,20 +56,22 @@ export default class Snapshot extends PercyCommand {
 
     const {args, flags} = this.parse(Snapshot)
 
-    const isWindows = process.platform === 'win32'
-
     const snapshotDirectory = args.snapshotDirectory as string
     const port = flags.port as number
     const staticServerPort = port + 1
     const networkIdleTimeout = flags['network-idle-timeout'] as number
     const baseUrl = flags['base-url'] as string
-    const ignoreFilesRegex = flags['ignore-files'] as string
-    const snapshotFilesRegex = flags['snapshot-files'] as string
+    const rawIgnoreGlob = flags['ignore-files'] as string
+    const rawSnapshotGlob = flags['snapshot-files'] as string
+
+    const snapshotGlobs = rawSnapshotGlob.split(',')
+
+    const ignoreGlobs = rawIgnoreGlob ? rawIgnoreGlob.split(',') : []
 
     // exit gracefully if percy will not run
     if (!this.percyWillRun()) { this.exit(0) }
 
-    // check that base url starts with a slash and exit if it is missing
+    // check that the base url passed in starts with a slash and exit if it is missing
     if (baseUrl[0] !== '/') {
       logger.warn('The base-url flag must begin with a slash.')
       this.exit(1)
@@ -83,8 +85,8 @@ export default class Snapshot extends PercyCommand {
       port: staticServerPort,
       snapshotDirectory,
       baseUrl,
-      snapshotFilesRegex,
-      ignoreFilesRegex,
+      snapshotGlobs,
+      ignoreGlobs,
     }
 
     const staticSnapshotService = new StaticSnapshotService(options)

src/services/static-snapshot-options.ts

@@ -2,6 +2,6 @@ export interface StaticSnapshotOptions {
     snapshotDirectory: string,
     port: number,
     baseUrl: string,
-    snapshotFilesRegex: string,
-    ignoreFilesRegex?: string,
+    snapshotGlobs: string[],
+    ignoreGlobs: string[],
   }

src/services/static-snapshot-service.ts

@@ -1,20 +1,93 @@
+import * as bodyParser from 'body-parser'
+import * as cors from 'cors'
+import * as express from 'express'
+import * as globby from 'globby'
+import {Server} from 'http'
+import * as puppeteer from 'puppeteer'
 import logger from '../utils/logger'
+import {agentJsFilename} from '../utils/sdk-utils'
 import {StaticSnapshotOptions} from './static-snapshot-options'
 
+// Use this instead of importing PercyAgent - we only want the compiled version
+declare var PercyAgent: any
+
 export default class StaticSnapshotService {
+  readonly options: StaticSnapshotOptions
+  private readonly app: express.Application
+  private server: Server | null = null
+
   constructor(options: StaticSnapshotOptions) {
-    // logger.info('calling constructor...')
+    this.app = express()
+    this.options = options
+
+    this.app.use(cors())
+    this.app.use(bodyParser.urlencoded({extended: true}))
+    this.app.use(bodyParser.json({limit: '50mb'}))
+
+    this.app.use(options.baseUrl, express.static(options.snapshotDirectory))
   }
 
   async start() {
-    // logger.info('starting static snapshot service...')
+    logger.info(`serving static site at ${this._buildLocalUrl()}`)
+    this.server = await this.app.listen(this.options.port)
   }
 
   async snapshotAll() {
-    // logger.info('taking snapshots of the static site...')
+    logger.debug('taking snapshots of static site')
+
+    const browser = await puppeteer.launch({
+      args: ['--no-sandbox'],
+      handleSIGINT : false,
+    })
+
+    const percyAgentClientFilename = agentJsFilename()
+    const page = await browser.newPage()
+
+    const pageUrls = await this._buildPageUrls()
+
+    for (const url of pageUrls) {
+      logger.debug(`visiting ${url}`)
+
+      await page.goto(url)
+
+      await page.addScriptTag({
+        path: percyAgentClientFilename,
+      })
+
+      await page.evaluate((name) => {
+        const percyAgentClient = new PercyAgent()
+        return percyAgentClient.snapshot(name)
+      }, url)
+    }
+
+    browser.close()
   }
 
   async stop() {
-    // logger.info('stopping static snapshot service...')
+    if (this.server) { await this.server.close() }
+
+    logger.info(`shutting down static site at ${this._buildLocalUrl()}`)
+  }
+
+  _buildLocalUrl() {
+    return `http://localhost:${this.options.port}${this.options.baseUrl}`
+  }
+
+  async _buildPageUrls() {
+    const baseUrl = this._buildLocalUrl()
+    const pageUrls = [] as any
+
+    const globOptions = {
+      cwd: this.options.snapshotDirectory,
+      ignore: this.options.ignoreGlobs,
+    }
+
+    const paths = await globby(this.options.snapshotGlobs, globOptions)
+
+    for (const path of paths) {
+      pageUrls.push(baseUrl + path)
+    }
+
+    return pageUrls
   }
 }

test/commands/snapshot.test.ts

@@ -49,8 +49,6 @@ describe('snapshot', () => {
       chai.expect(staticSnapshotServiceStub.snapshotAll).to.have.callCount(1)
       chai.expect(stdout).to.match(/\[percy\] percy has started./)
     })
-
-    xit('starts the snapshot service on the correct port')
   })
 
   describe('snapshot command', () => {

test/fixtures/services/static-snapshot-service/_dummy-testing-app/about-us.html

@@ -0,0 +1 @@
+This is another dummy page.

test/fixtures/services/static-snapshot-service/_dummy-testing-app/blog/index.html

@@ -0,0 +1 @@
+This is a nested file for unit testing.

test/fixtures/services/static-snapshot-service/_dummy-testing-app/index.html

@@ -0,0 +1 @@
+This is a test file.

test/fixtures/services/static-snapshot-service/_dummy-testing-app/index.js

@@ -0,0 +1 @@
+// this is a dummy file that should not be picked up in testing

test/fixtures/services/static-snapshot-service/_dummy-testing-app/styles.css

@@ -0,0 +1 @@
+/* this is a dummy file that should not be picked up in testing */

test/integration/test-static-site/assets/css/app.css

@@ -0,0 +1,8 @@
+.icon {
+  width: 20px;
+  height: 20px;
+}
+.icon.lg {
+  width: 26px;
+  height: 26px;
+}

test/integration/test-static-site/families/greyjoy/dead.htm

@@ -0,0 +1,15 @@
+<!doctype html>
+<html>
+  <head>
+    <title>Greyjoy Deceased List</title>
+    <meta charset="utf-8"/>
+    <link rel="stylesheet" href="/dummy-base-url/assets/css/app.css">
+  </head>
+  <body>
+      <p><strong>Deceased Greyjoys</strong></p>
+    <ul>
+      <li>Balon Greyjoy</li>
+    </ul>
+    <p><img src="/dummy-base-url/assets/images/red-x.png" class="icon"> <strong>percy-agent should not snapshot this page.</strong></p>
+  </body>
+</html>

test/integration/test-static-site/families/greyjoy/members.html

@@ -0,0 +1,17 @@
+<!doctype html>
+<html>
+  <head>
+    <title>Greyjoy Alive List</title>
+    <meta charset="utf-8"/>
+    <link rel="stylesheet" href="/dummy-base-url/assets/css/app.css">
+  </head>
+  <body>
+      <p><strong>Living Greyjoys</strong></p>
+    <ul>
+      <li>Theon Greyjoy</li>
+      <li>Euron Greyjoy</li>
+      <li>Yara Greyjoy - Unconfirmed</li>
+    </ul>
+    <p><img src="/dummy-base-url/assets/images/check.png" class="icon"> <strong>percy-agent should snapshot this page.</strong></p>
+  </body>
+</html>

test/integration/test-static-site/families/greyjoy/pyke.html

@@ -0,0 +1,15 @@
+<!doctype html>
+<html>
+  <head>
+    <title>Pyke - Greyjoy Castle</title>
+    <meta charset="utf-8"/>
+    <link rel="stylesheet" href="/dummy-base-url/assets/css/app.css">
+  </head>
+  <body>
+    <p><strong>Pyke - Greyjoy Castle (via <a href="https://www.deviantart.com/feliche">feliche</a>)</strong></p>
+    <p>
+        <img src="/dummy-base-url/assets/images/pyke.jpg" alt="">
+    </p>
+    <p><img src="/dummy-base-url/assets/images/check.png" class="icon"> <strong>percy-agent should snapshot this page.</strong></p>
+  </body>
+</html>

test/integration/test-static-site/families/lannister/casterly-rock.html

@@ -0,0 +1,15 @@
+<!doctype html>
+<html>
+  <head>
+    <title>Casterly Rock - Lannister Castle</title>
+    <meta charset="utf-8"/>
+    <link rel="stylesheet" href="/dummy-base-url/assets/css/app.css">
+  </head>
+  <body>
+    <p><strong>Casterly Rock - Lannister Castle (via <a href="https://www.deviantart.com/feliche">feliche</a>)</strong></p>
+    <p>
+        <img src="/dummy-base-url/assets/images/casterly-rock.jpg" alt="">
+    </p>
+    <p><img src="/dummy-base-url/assets/images/check.png" class="icon"> <strong>percy-agent should snapshot this page.</strong></p>
+  </body>
+</html>

test/integration/test-static-site/families/lannister/dead-members/list.htm

@@ -0,0 +1,18 @@
+<!doctype html>
+<html>
+  <head>
+    <title>Lannister Deceased List</title>
+    <meta charset="utf-8"/>
+    <link rel="stylesheet" href="/dummy-base-url/assets/css/app.css">
+  </head>
+  <body>
+      <p><strong>Deceased Lannisters</strong></p>
+    <p>Half of the Lannisters are dead. They're all pretty close anyway.</p>
+    <ul>
+        <li>Tywin Lannister</li>
+        <li>Kevan Lannister</li>
+        <li>Lancel Lannister</li>
+    </ul> 
+    <p><img src="/dummy-base-url/assets/images/red-x.png" class="icon"> <strong>percy-agent should not snapshot this page.</strong></p>
+  </body>
+</html>

test/integration/test-static-site/families/lannister/members.html

@@ -0,0 +1,17 @@
+<!doctype html>
+<html>
+  <head>
+    <title>Lannister Alive List</title>
+    <meta charset="utf-8"/>
+    <link rel="stylesheet" href="/dummy-base-url/assets/css/app.css">
+  </head>
+  <body>
+      <p><strong>Living Lannisters</strong></p>
+    <ul>
+      <li>Cersei Lannister</li>
+      <li>Jamie Lannister</li>
+      <li>Tyrion Lannister</li>
+    </ul>
+    <p><img src="/dummy-base-url/assets/images/check.png" class="icon"> <strong>percy-agent should snapshot this page.</strong></p>
+  </body>
+</html>

test/integration/test-static-site/families/stark/dead-members/list.htm

@@ -0,0 +1,19 @@
+<!doctype html>
+<html>
+  <head>
+    <title>Stark Deceased List</title>
+    <meta charset="utf-8"/>
+    <link rel="stylesheet" href="/dummy-base-url/assets/css/app.css">
+  </head>
+  <body>
+      <p><strong>Deceased Starks</strong></p>
+    <p>Half of the Starks are dead. The rest are pretty close.</p>
+    <ul>
+        <li>Ned Stark</li>
+        <li>Catelyn Stark</li>
+        <li>Robb Stark</li>
+        <li>Rickon Stark</li>
+    </ul> 
+    <p><img src="/dummy-base-url/assets/images/red-x.png" class="icon"> <strong>percy-agent should not snapshot this page.</strong></p>
+  </body>
+</html>