Merge pull request #394 from thefrontside/cli-exit-code

Set exit code based on result of running suite

.changeset/cli-exit-code.md

@@ -0,0 +1,4 @@
+---
+"@bigtest/cli": "minor"
+---
+Set exit code based on result of running test suite

packages/cli/package.json

@@ -46,7 +46,7 @@
     "@bigtest/performance": "^0.5.0",
     "@bigtest/project": "^0.5.1",
     "@bigtest/server": "^0.8.0",
-    "@effection/node": "^0.6.5",
+    "@effection/node": "^0.8.0",
     "capture-console": "^1.0.1",
     "deepmerge": "^4.2.2",
     "effection": "^0.7.0",

packages/cli/src/cli.ts

@@ -20,9 +20,11 @@ export function * CLI(argv: string[]): Operation {
     yield;
   } else if (command === 'test') {
     yield runTest(config, lines);
-  } else {
+  } else if (command === 'ci') {
     yield startServer(config);
     yield runTest(config, lines);
+  } else {
+    throw new Error(`unknown command: ${command}`);
   }
 }
 
@@ -35,6 +37,10 @@ function parseOptions(config: ProjectOptions, argv: readonly string[]): Command
         choices: Object.keys(config.drivers),
         default: config.launch,
       })
+      .option('test-files', {
+        describe: 'file globs which form the test suite',
+        type: 'array'
+      })
   };
 
   let rawOptions = yargs({})
@@ -57,6 +63,9 @@ function parseOptions(config: ProjectOptions, argv: readonly string[]): Command
   if(rawOptions['launch']) {
     config.launch = rawOptions['launch'];
   }
+  if(rawOptions['testFiles']) {
+    config.testFiles = rawOptions['testFiles'] as string[];
+  }
 
   return rawOptions._[0] as Command;
 }

packages/cli/src/index.ts

@@ -1,6 +1,4 @@
 import { CLI } from './cli';
 import { main } from '@effection/node';
 
-main(function* boot() {
-  yield CLI(process.argv.slice(2));
-});
+main(CLI(process.argv.slice(2)));

packages/cli/src/run-test.ts

@@ -3,10 +3,11 @@ import { ProjectOptions } from '@bigtest/project';
 import { performance } from '@bigtest/performance';
 import { ResultStatus } from '@bigtest/suite';
 import { Client } from '@bigtest/server';
+import { MainError } from '@effection/node';
 import * as query from './query';
 import { StreamingFormatter } from './format-helpers';
 
-export function* runTest(config: ProjectOptions, formatter: StreamingFormatter): Operation<ResultStatus> {
+export function* runTest(config: ProjectOptions, formatter: StreamingFormatter): Operation<void> {
   let client: Client = yield Client.create(`ws://localhost:${config.port}`);
 
   let subscription = yield client.subscription(query.run());
@@ -46,5 +47,7 @@ export function* runTest(config: ProjectOptions, formatter: StreamingFormatter):
     assertionCounts,
   });
 
-  return testRunStatus || 'failed';
+  if(testRunStatus !== 'ok') {
+    throw new MainError({ exitCode: 1 });
+  }
 }

packages/cli/test/cli.test.ts

@@ -38,7 +38,7 @@ describe('@bigtest/cli', function() {
       let runChild: Process;
 
       beforeEach(async () => {
-        startChild = await World.spawn(run('server', '--launch', 'chrome.headless'));
+        startChild = await World.spawn(run('server', '--launch', 'chrome.headless', '--test-files', './test/fixtures/passing.test.ts'));
 
         await World.spawn(startChild.stdout?.waitFor("[orchestrator] running!"));
 
@@ -56,14 +56,38 @@ describe('@bigtest/cli', function() {
         expect(runChild.stdout?.output).toContain("SUCCESS")
       });
     });
+
+    describe('running the suite with failures', () => {
+      let startChild: Process;
+      let runChild: Process;
+
+      beforeEach(async () => {
+        startChild = await World.spawn(run('server', '--launch', 'chrome.headless', '--test-files', './test/fixtures/failing.test.ts'));
+
+        await World.spawn(startChild.stdout?.waitFor("[orchestrator] running!"));
+
+        runChild = await World.spawn(run('test'));
+
+        await World.spawn(runChild.join());
+      });
+
+      afterEach(async () => {
+        await World.spawn(startChild.close());
+      });
+
+      it('exits with error code', async () => {
+        expect(runChild.code).toEqual(1);
+        expect(runChild.stdout?.output).toContain("FAILURE")
+      });
+    });
   });
 
   describe('ci', () => {
     describe('running the suite successfully', () => {
       let child: Process;
 
       beforeEach(async () => {
-        child = await World.spawn(run('ci', '--launch', 'chrome.headless', '--log-level', 'debug'));
+        child = await World.spawn(run('ci', '--launch', 'chrome.headless', '--test-files', './test/fixtures/passing.test.ts'));
         await World.spawn(child.stdout?.waitFor("[orchestrator] running!"));
         await World.spawn(child.join());
       });
@@ -79,6 +103,28 @@ describe('@bigtest/cli', function() {
         expect(child.stdout?.output).toContain("✓ SUCCESS")
       });
     });
+
+    describe('running the suite with failures', () => {
+      let child: Process;
+
+      beforeEach(async () => {
+        child = await World.spawn(run('ci', '--launch', 'chrome.headless', '--test-files', './test/fixtures/failing.test.ts'));
+        await World.spawn(child.stdout?.waitFor("[orchestrator] running!"));
+        await World.spawn(child.join());
+      });
+
+      afterEach(async () => {
+        await World.spawn(child.close());
+      });
+
+      it('exits with error code', async () => {
+        expect(child.code).toEqual(1);
+        expect(child.stdout?.output).toContain("✓ [step]       Failing Test -> first step")
+        expect(child.stdout?.output).toContain("✓ [assertion]  Failing Test -> check the thing")
+        expect(child.stdout?.output).toContain("⨯ [step]       Failing Test -> child -> child second step")
+        expect(child.stdout?.output).toContain("⨯ FAILURE")
+      });
+    });
   });
 });
 

packages/cli/test/fixtures/failing.test.ts

@@ -0,0 +1,16 @@
+import { test } from '@bigtest/suite';
+
+const delay = (time = 50) =>
+  async () => { await new Promise(resolve => setTimeout(resolve, time)) };
+
+export default test('Failing Test')
+  .step("first step", delay())
+  .step("second step", delay())
+  .step("third step", delay())
+  .assertion("check the thing", delay(3))
+  .child(
+    "child", test => test
+      .step("child first step", delay())
+      .step("child second step", async () => { throw new Error('moo') })
+      .step("child third step", delay())
+      .assertion("child first assertion", delay(5)));

yarn.lock

@@ -2208,6 +2208,14 @@
     "@effection/events" "^0.7.4"
     effection "^0.7.0"
 
+"@effection/node@^0.8.0":
+  version "0.8.0"
+  resolved "https://registry.yarnpkg.com/@effection/node/-/node-0.8.0.tgz#a73069ecd60c628d1aa2b9e61a8ff930be8faa7c"
+  integrity sha512-p+6QQINBvGO0ev05LKRMlcsC567ubfBfEefkzNRBNWb1drumOMtD8FMjZjWCB/brPLPO5un7CAOErorfCHvkaA==
+  dependencies:
+    "@effection/events" "^0.7.4"
+    effection "^0.7.0"
+
 "@effection/subscription@^0.8.0":
   version "0.8.1"
   resolved "https://registry.yarnpkg.com/@effection/subscription/-/subscription-0.8.1.tgz#aa0286549b2bb833f211010db33c0324f6b59d81"