From dddbcd993dfcafe21e6f4064cdc72298e10b5e18 Mon Sep 17 00:00:00 2001 From: Jonas Nicklas Date: Wed, 8 Jul 2020 15:41:57 +0200 Subject: [PATCH 1/4] Set exit code based on result of running suite --- packages/cli/package.json | 2 +- packages/cli/src/cli.ts | 11 ++++- packages/cli/src/index.ts | 4 +- packages/cli/src/run-test.ts | 7 ++- packages/cli/src/suite-error.ts | 9 ++++ packages/cli/test/cli.test.ts | 50 +++++++++++++++++++++- packages/cli/test/fixtures/failing.test.ts | 16 +++++++ yarn.lock | 8 ++++ 8 files changed, 98 insertions(+), 9 deletions(-) create mode 100644 packages/cli/src/suite-error.ts create mode 100644 packages/cli/test/fixtures/failing.test.ts diff --git a/packages/cli/package.json b/packages/cli/package.json index 22a2e6132..122df1326 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -46,7 +46,7 @@ "@bigtest/performance": "^0.5.0", "@bigtest/project": "^0.5.1", "@bigtest/server": "^0.8.0", - "@effection/node": "^0.6.5", + "@effection/node": "^0.8.0", "capture-console": "^1.0.1", "deepmerge": "^4.2.2", "effection": "^0.7.0", diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index 28b3fb632..e312bfe0e 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -20,9 +20,11 @@ export function * CLI(argv: string[]): Operation { yield; } else if (command === 'test') { yield runTest(config, lines); - } else { + } else if (command === 'ci') { yield startServer(config); yield runTest(config, lines); + } else { + throw new Error(`unknown command: ${command}`); } } @@ -35,6 +37,10 @@ function parseOptions(config: ProjectOptions, argv: readonly string[]): Command choices: Object.keys(config.drivers), default: config.launch, }) + .option('test-files', { + describe: 'file globs which form the test suite', + type: 'array' + }) }; let rawOptions = yargs({}) @@ -57,6 +63,9 @@ function parseOptions(config: ProjectOptions, argv: readonly string[]): Command if(rawOptions['launch']) { config.launch = rawOptions['launch']; } + if(rawOptions['testFiles']) { + config.testFiles = rawOptions['testFiles'] as string[]; + } return rawOptions._[0] as Command; } diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index 21608da2e..e0c2638a5 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -1,6 +1,4 @@ import { CLI } from './cli'; import { main } from '@effection/node'; -main(function* boot() { - yield CLI(process.argv.slice(2)); -}); +main(CLI(process.argv.slice(2))); diff --git a/packages/cli/src/run-test.ts b/packages/cli/src/run-test.ts index d76c7897a..5e60ad4c4 100644 --- a/packages/cli/src/run-test.ts +++ b/packages/cli/src/run-test.ts @@ -5,8 +5,9 @@ import { ResultStatus } from '@bigtest/suite'; import { Client } from '@bigtest/server'; import * as query from './query'; import { StreamingFormatter } from './format-helpers'; +import { SuiteError } from './suite-error'; -export function* runTest(config: ProjectOptions, formatter: StreamingFormatter): Operation { +export function* runTest(config: ProjectOptions, formatter: StreamingFormatter): Operation { let client: Client = yield Client.create(`ws://localhost:${config.port}`); let subscription = yield client.subscription(query.run()); @@ -46,5 +47,7 @@ export function* runTest(config: ProjectOptions, formatter: StreamingFormatter): assertionCounts, }); - return testRunStatus || 'failed'; + if(testRunStatus !== 'ok') { + throw new SuiteError(); + } } diff --git a/packages/cli/src/suite-error.ts b/packages/cli/src/suite-error.ts new file mode 100644 index 000000000..1e0cf7470 --- /dev/null +++ b/packages/cli/src/suite-error.ts @@ -0,0 +1,9 @@ +export class SuiteError extends Error { + constructor() { + super("suite failed") + } + + name = "SuiteError"; + effectionSilent = true; + effectionExitCode = 1; +} diff --git a/packages/cli/test/cli.test.ts b/packages/cli/test/cli.test.ts index ed9682361..179d16669 100644 --- a/packages/cli/test/cli.test.ts +++ b/packages/cli/test/cli.test.ts @@ -38,7 +38,7 @@ describe('@bigtest/cli', function() { let runChild: Process; beforeEach(async () => { - startChild = await World.spawn(run('server', '--launch', 'chrome.headless')); + startChild = await World.spawn(run('server', '--launch', 'chrome.headless', '--test-files', './test/fixtures/passing.test.ts')); await World.spawn(startChild.stdout?.waitFor("[orchestrator] running!")); @@ -56,6 +56,30 @@ describe('@bigtest/cli', function() { expect(runChild.stdout?.output).toContain("SUCCESS") }); }); + + describe('running the suite with failures', () => { + let startChild: Process; + let runChild: Process; + + beforeEach(async () => { + startChild = await World.spawn(run('server', '--launch', 'chrome.headless', '--test-files', './test/fixtures/failing.test.ts')); + + await World.spawn(startChild.stdout?.waitFor("[orchestrator] running!")); + + runChild = await World.spawn(run('test')); + + await World.spawn(runChild.join()); + }); + + afterEach(async () => { + await World.spawn(startChild.close()); + }); + + it('exits with error code', async () => { + expect(runChild.code).toEqual(1); + expect(runChild.stdout?.output).toContain("FAILURE") + }); + }); }); describe('ci', () => { @@ -63,7 +87,7 @@ describe('@bigtest/cli', function() { let child: Process; beforeEach(async () => { - child = await World.spawn(run('ci', '--launch', 'chrome.headless', '--log-level', 'debug')); + child = await World.spawn(run('ci', '--launch', 'chrome.headless', '--test-files', './test/fixtures/passing.test.ts')); await World.spawn(child.stdout?.waitFor("[orchestrator] running!")); await World.spawn(child.join()); }); @@ -79,6 +103,28 @@ describe('@bigtest/cli', function() { expect(child.stdout?.output).toContain("✓ SUCCESS") }); }); + + describe('running the suite with failures', () => { + let child: Process; + + beforeEach(async () => { + child = await World.spawn(run('ci', '--launch', 'chrome.headless', '--test-files', './test/fixtures/failing.test.ts')); + await World.spawn(child.stdout?.waitFor("[orchestrator] running!")); + await World.spawn(child.join()); + }); + + afterEach(async () => { + await World.spawn(child.close()); + }); + + it('exits with error code', async () => { + expect(child.code).toEqual(1); + expect(child.stdout?.output).toContain("✓ [step] Failing Test -> first step") + expect(child.stdout?.output).toContain("✓ [assertion] Failing Test -> check the thing") + expect(child.stdout?.output).toContain("⨯ [step] Failing Test -> child -> child second step") + expect(child.stdout?.output).toContain("✓ FAILURE") + }); + }); }); }); diff --git a/packages/cli/test/fixtures/failing.test.ts b/packages/cli/test/fixtures/failing.test.ts new file mode 100644 index 000000000..45182f5b3 --- /dev/null +++ b/packages/cli/test/fixtures/failing.test.ts @@ -0,0 +1,16 @@ +import { test } from '@bigtest/suite'; + +const delay = (time = 50) => + async () => { await new Promise(resolve => setTimeout(resolve, time)) }; + +export default test('Failing Test') + .step("first step", delay()) + .step("second step", delay()) + .step("third step", delay()) + .assertion("check the thing", delay(3)) + .child( + "child", test => test + .step("child first step", delay()) + .step("child second step", async () => { throw new Error('moo') }) + .step("child third step", delay()) + .assertion("child first assertion", delay(5))); diff --git a/yarn.lock b/yarn.lock index 9076af699..abe7a25ad 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2208,6 +2208,14 @@ "@effection/events" "^0.7.4" effection "^0.7.0" +"@effection/node@^0.8.0": + version "0.8.0" + resolved "https://registry.yarnpkg.com/@effection/node/-/node-0.8.0.tgz#a73069ecd60c628d1aa2b9e61a8ff930be8faa7c" + integrity sha512-p+6QQINBvGO0ev05LKRMlcsC567ubfBfEefkzNRBNWb1drumOMtD8FMjZjWCB/brPLPO5un7CAOErorfCHvkaA== + dependencies: + "@effection/events" "^0.7.4" + effection "^0.7.0" + "@effection/subscription@^0.8.0": version "0.8.1" resolved "https://registry.yarnpkg.com/@effection/subscription/-/subscription-0.8.1.tgz#aa0286549b2bb833f211010db33c0324f6b59d81" From 80bb1b96e4e42f59a17373439ba76326e3d424dd Mon Sep 17 00:00:00 2001 From: Jonas Nicklas Date: Thu, 9 Jul 2020 14:02:39 +0200 Subject: [PATCH 2/4] Use `MainError` from @effection/node --- packages/cli/src/run-test.ts | 4 ++-- packages/cli/src/suite-error.ts | 9 --------- 2 files changed, 2 insertions(+), 11 deletions(-) delete mode 100644 packages/cli/src/suite-error.ts diff --git a/packages/cli/src/run-test.ts b/packages/cli/src/run-test.ts index 5e60ad4c4..b925b3321 100644 --- a/packages/cli/src/run-test.ts +++ b/packages/cli/src/run-test.ts @@ -3,9 +3,9 @@ import { ProjectOptions } from '@bigtest/project'; import { performance } from '@bigtest/performance'; import { ResultStatus } from '@bigtest/suite'; import { Client } from '@bigtest/server'; +import { MainError } from '@effection/node'; import * as query from './query'; import { StreamingFormatter } from './format-helpers'; -import { SuiteError } from './suite-error'; export function* runTest(config: ProjectOptions, formatter: StreamingFormatter): Operation { let client: Client = yield Client.create(`ws://localhost:${config.port}`); @@ -48,6 +48,6 @@ export function* runTest(config: ProjectOptions, formatter: StreamingFormatter): }); if(testRunStatus !== 'ok') { - throw new SuiteError(); + throw new MainError({ exitCode: 1 }); } } diff --git a/packages/cli/src/suite-error.ts b/packages/cli/src/suite-error.ts deleted file mode 100644 index 1e0cf7470..000000000 --- a/packages/cli/src/suite-error.ts +++ /dev/null @@ -1,9 +0,0 @@ -export class SuiteError extends Error { - constructor() { - super("suite failed") - } - - name = "SuiteError"; - effectionSilent = true; - effectionExitCode = 1; -} From 013e6b73f32d5b967a468e4eb890ebddc14bd64f Mon Sep 17 00:00:00 2001 From: Jonas Nicklas Date: Thu, 9 Jul 2020 14:06:23 +0200 Subject: [PATCH 3/4] Fix typo --- packages/cli/test/cli.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cli/test/cli.test.ts b/packages/cli/test/cli.test.ts index 179d16669..ed4cb3676 100644 --- a/packages/cli/test/cli.test.ts +++ b/packages/cli/test/cli.test.ts @@ -122,7 +122,7 @@ describe('@bigtest/cli', function() { expect(child.stdout?.output).toContain("✓ [step] Failing Test -> first step") expect(child.stdout?.output).toContain("✓ [assertion] Failing Test -> check the thing") expect(child.stdout?.output).toContain("⨯ [step] Failing Test -> child -> child second step") - expect(child.stdout?.output).toContain("✓ FAILURE") + expect(child.stdout?.output).toContain("⨯ FAILURE") }); }); }); From c07415a4addd5004829d118cb668d2fc79e8c019 Mon Sep 17 00:00:00 2001 From: Jonas Nicklas Date: Mon, 13 Jul 2020 15:36:13 +0200 Subject: [PATCH 4/4] Add changeset --- .changeset/cli-exit-code.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .changeset/cli-exit-code.md diff --git a/.changeset/cli-exit-code.md b/.changeset/cli-exit-code.md new file mode 100644 index 000000000..02575d9ad --- /dev/null +++ b/.changeset/cli-exit-code.md @@ -0,0 +1,4 @@ +--- +"@bigtest/cli": "minor" +--- +Set exit code based on result of running test suite