diff --git a/.circleci/config.yml b/.circleci/config.yml
index acc7bc447f4..102d0981ade 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -175,6 +175,9 @@ jobs:
       - run:
           name: Test certain bundles against function constructors
           command: npm run no-new-func
+      - run:
+          name: Test plotly bundles against es6
+          command: npm run no-es6-dist
 
 workflows:
   version: 2
diff --git a/package.json b/package.json
index b4e152f16a1..08e2371031a 100644
--- a/package.json
+++ b/package.json
@@ -34,6 +34,7 @@
     "no-new-func": "eslint --no-ignore --no-eslintrc --no-inline-config --rule '{no-new-func: error}' $(find dist -type f -iname '*.js' | grep -v plotly-gl* | grep -v plotly-with-meta.* | grep -v plotly.js | grep -v plotly.min.js | grep -v MathJax.js)",
     "no-bad-char": "eslint --no-ignore --no-eslintrc --no-inline-config --rule '{no-misleading-character-class: error}' $(find dist -type f -iname '*.js' | grep plotly)",
     "no-dup-keys": "eslint --no-ignore --no-eslintrc --no-inline-config --rule '{no-dupe-keys: error}' $(find dist -type f -iname '*.js' | grep plotly)",
+    "no-es6-dist": "node tasks/no_es6_dist.js",
     "docker": "node tasks/docker.js",
     "pretest": "node tasks/pretest.js",
     "test-jasmine": "karma start test/jasmine/karma.conf.js",
diff --git a/tasks/no_es6_dist.js b/tasks/no_es6_dist.js
new file mode 100644
index 00000000000..955dfe975bf
--- /dev/null
+++ b/tasks/no_es6_dist.js
@@ -0,0 +1,54 @@
+var eslint = require('eslint');
+var constants = require('./util/constants');
+var EXIT_CODE = 0;
+
+assertES5();
+
+// Ensure no ES6 has snuck through into the build:
+function assertES5() {
+    var CLIEngine = eslint.CLIEngine;
+
+    var cli = new CLIEngine({
+        allowInlineConfig: false,
+        useEslintrc: false,
+        ignore: false,
+        parserOptions: {
+            ecmaVersion: 5
+        }
+    });
+
+    var files = constants.partialBundlePaths.map(function(f) { return f.dist; });
+    files.unshift(constants.pathToPlotlyDist);
+
+    var report = cli.executeOnFiles(files);
+    var formatter = cli.getFormatter();
+
+    var errors = [];
+    if(report.errorCount > 0) {
+        console.log(formatter(report.results));
+
+        // It doesn't work well to pass formatted logs into this,
+        // so instead pass the empty string in a way that causes
+        // the test to fail
+        errors.push('');
+    }
+
+    log('es5-only syntax', errors);
+}
+
+
+function log(name, logs) {
+    if(logs.length) {
+        console.error('test-syntax error [' + name + ']');
+        console.error(logs.join('\n'));
+        EXIT_CODE = 1;
+    } else {
+        console.log('ok ' + name);
+    }
+}
+
+process.on('exit', function() {
+    if(EXIT_CODE) {
+        throw new Error('test syntax failed.');
+    }
+});
diff --git a/tasks/test_syntax.js b/tasks/test_syntax.js
index e7e53122e65..f6983905f30 100644
--- a/tasks/test_syntax.js
+++ b/tasks/test_syntax.js
@@ -5,7 +5,6 @@ var falafel = require('falafel');
 var glob = require('glob');
 var madge = require('madge');
 var readLastLines = require('read-last-lines');
-var eslint = require('eslint');
 var trueCasePath = require('true-case-path').trueCasePathSync;
 
 var common = require('./util/common');
@@ -27,7 +26,6 @@ assertSrcContents();
 assertFileNames();
 assertTrailingNewLine();
 assertCircularDeps();
-assertES5();
 
 
 // check for for focus and exclude jasmine blocks
@@ -299,39 +297,6 @@ function assertCircularDeps() {
     });
 }
 
-// Ensure no ES6 has snuck through into the build:
-function assertES5() {
-    var CLIEngine = eslint.CLIEngine;
-
-    var cli = new CLIEngine({
-        allowInlineConfig: false,
-        useEslintrc: false,
-        ignore: false,
-        parserOptions: {
-            ecmaVersion: 5
-        }
-    });
-
-    var files = constants.partialBundlePaths.map(function(f) { return f.dist; });
-    files.unshift(constants.pathToPlotlyBuild, constants.pathToPlotlyDist);
-
-    var report = cli.executeOnFiles(files);
-    var formatter = cli.getFormatter();
-
-    var errors = [];
-    if(report.errorCount > 0) {
-        console.log(formatter(report.results));
-
-        // It doesn't work well to pass formatted logs into this,
-        // so instead pass the empty string in a way that causes
-        // the test to fail
-        errors.push('');
-    }
-
-    log('es5-only syntax', errors);
-}
-
-
 function combineGlobs(arr) {
     return '{' + arr.join(',') + '}';
 }