test: collect stdout/stderr in tests

test-runner/src/reporters/json.ts

@@ -54,9 +54,17 @@ class JSONReporter extends BaseReporter {
       slow: test.slow,
       duration: test.duration,
       timeout: test.timeout,
-      error: test.error
+      error: test.error,
+      stdout: test.stdout.map(s => stdioEntry(s)),
+      stderr: test.stderr.map(s => stdioEntry(s)),
     };
   }
 }
 
+function stdioEntry(s: string | Buffer): any {
+  if (typeof s === 'string')
+    return { text: s };
+  return { buffer: s.toString('base64') }
+}
+
 export default JSONReporter;

test-runner/src/runner.ts

@@ -160,13 +160,21 @@ export class Runner {
     });
     worker.on('fail', params => {
       ++this.stats.failures;
-      const out = worker.takeOut();
-      if (out.length)
-        params.test.error.stack += '\n\x1b[33mstdout: ' + out.join('\n') + '\x1b[0m';
-      const err = worker.takeErr();
-      if (err.length)
-        params.test.error.stack += '\n\x1b[33mstderr: ' + err.join('\n') + '\x1b[0m';
-        this._reporter.onFail(this._updateTest(params.test));
+      this._reporter.onFail(this._updateTest(params.test));
+    });
+    worker.on('stdout', params => {
+      const chunk = chunkFromParams(params);
+      if (!this._config.quiet)
+        process.stdout.write(chunk);
+      const test = this._testById.get(params.testId);
+      test.stdout.push(chunk);
+    });
+    worker.on('stderr', params => {
+      const chunk = chunkFromParams(params);
+      if (!this._config.quiet)
+        process.stderr.write(chunk);
+      const test = this._testById.get(params.testId);
+      test.stderr.push(chunk);
     });
     worker.on('exit', () => {
       this._workers.delete(worker);
@@ -238,20 +246,6 @@ class OopWorker extends Worker {
       const { method, params } = message;
       this.emit(method, params);
     });
-    this.stdout = [];
-    this.stderr = [];
-    this.on('stdout', params => {
-      const chunk = chunkFromParams(params);
-      if (!runner._config.quiet)
-        process.stdout.write(chunk);
-      this.stdout.push(chunk);
-    });
-    this.on('stderr', params => {
-      const chunk = chunkFromParams(params);
-      if (!runner._config.quiet)
-        process.stderr.write(chunk);
-      this.stderr.push(chunk);
-    });
   }
 
   async init() {
@@ -267,18 +261,6 @@ class OopWorker extends Worker {
   stop() {
     this.process.send({ method: 'stop' });
   }
-
-  takeOut() {
-    const result = this.stdout;
-    this.stdout = [];
-    return result;
-  }
-
-  takeErr() {
-    const result = this.stderr;
-    this.stderr = [];
-    return result;
-  }
 }
 
 class InProcessWorker extends Worker {
@@ -298,7 +280,7 @@ class InProcessWorker extends Worker {
     delete require.cache[entry.file];
     const { TestRunner } = require('./testRunner');
     const testRunner = new TestRunner(entry, this.runner._config, 0);
-    for (const event of ['test', 'pending', 'pass', 'fail', 'done'])
+    for (const event of ['test', 'pending', 'pass', 'fail', 'done', 'stdout', 'stderr'])
       testRunner.on(event, this.emit.bind(this, event));
     testRunner.run();
   }
@@ -317,9 +299,9 @@ class InProcessWorker extends Worker {
   }
 }
 
-function chunkFromParams(params: string | { buffer: string }): string | Buffer {
-  if (typeof params === 'string')
-    return params;
+function chunkFromParams(params: { testId: string, buffer?: string, text?: string }): string | Buffer {
+  if (typeof params.text === 'string')
+    return params.text;
   return Buffer.from(params.buffer, 'base64');
 }
 

test-runner/src/test.ts

@@ -27,6 +27,8 @@ export class Test {
   timeout = 0;
   fn: Function;
   error: any;
+  stdout: (string | Buffer)[] = [];
+  stderr: (string | Buffer)[] = [];
 
   _ordinal: number;
   _overriddenFn: Function;

test-runner/src/testRunner.ts

@@ -20,6 +20,7 @@ import { setCurrentTestFile } from './expect';
 import { Test, Suite, Configuration, serializeError } from './test';
 import { spec } from './spec';
 import { RunnerConfig } from './runnerConfig';
+import * as util from 'util';
 
 export const fixturePool = new FixturePool<RunnerConfig>();
 
@@ -31,6 +32,14 @@ export type TestRunnerEntry = {
   hash: string;
 };
 
+function chunkToParams(chunk: Buffer | string):  { text?: string, buffer?: string } {
+  if (chunk instanceof Buffer)
+    return { buffer: chunk.toString('base64') };
+  if (typeof chunk !== 'string')
+    return { text: util.inspect(chunk) };
+  return { text: chunk };
+}
+
 export class TestRunner extends EventEmitter {
   private _currentOrdinal = -1;
   private _failedWithError: any | undefined;
@@ -75,6 +84,14 @@ export class TestRunner extends EventEmitter {
     this._reportDone();
   }
 
+  stdout(chunk: string | Buffer) {
+    this.emit('stdout', { testId: this._testId(), ...chunkToParams(chunk) })
+  }
+
+  stderr(chunk: string | Buffer) {
+    this.emit('stderr', { testId: this._testId(), ...chunkToParams(chunk) })
+  }
+
   async run() {
     setParameters(this._parsedGeneratorConfiguration);
 
@@ -169,9 +186,13 @@ export class TestRunner extends EventEmitter {
     return fixturePool.wrapTestCallback(test.fn, timeout, { ...this._config }, test);
   }
 
+  private _testId() {
+    return `${this._test._ordinal}@${this._configuredFile}`;
+  }
+
   private _serializeTest() {
     return {
-      id: `${this._test._ordinal}@${this._configuredFile}`,
+      id: this._testId(),
       error: this._test.error,
       duration: Date.now() - this._test._startTime,
     };

test-runner/src/worker.ts

@@ -16,27 +16,20 @@
 
 import { initializeImageMatcher } from './expect';
 import { TestRunner, fixturePool } from './testRunner';
-import * as util from 'util';
 
 let closed = false;
 
 sendMessageToParent('ready');
 
-function chunkToParams(chunk) {
-  if (chunk instanceof Buffer)
-    return { buffer: chunk.toString('base64') };
-  if (typeof chunk !== 'string')
-    return util.inspect(chunk);
-  return chunk;
-}
-
 process.stdout.write = chunk => {
-  sendMessageToParent('stdout', chunkToParams(chunk));
+  if (testRunner && !closed)
+    testRunner.stdout(chunk);
   return true;
 };
 
 process.stderr.write = chunk => {
-  sendMessageToParent('stderr', chunkToParams(chunk));
+  if (testRunner && !closed)
+    testRunner.stderr(chunk);
   return true;
 };
 
@@ -69,7 +62,7 @@ process.on('message', async message => {
   }
   if (message.method === 'run') {
     testRunner = new TestRunner(message.params.entry, message.params.config, workerId);
-    for (const event of ['test', 'pending', 'pass', 'fail', 'done'])
+    for (const event of ['test', 'pending', 'pass', 'fail', 'done', 'stdout', 'stderr'])
       testRunner.on(event, sendMessageToParent.bind(null, event));
     await testRunner.run();
     testRunner = null;

test-runner/test/assets/stdio.js

@@ -0,0 +1,24 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+require('../../..');
+
+it('stdio', () => {
+  process.stdout.write('stdout text');
+  process.stdout.write(Buffer.from('stdout buffer'));
+  process.stderr.write('stderr text');
+  process.stderr.write(Buffer.from('stderr buffer'));
+});

test-runner/test/exit-code.spec.ts

@@ -56,6 +56,16 @@ it('should handle worker fixture error', async() => {
   expect(result.output).toContain('Worker failed');
 });
 
+it('should collect stdio', async() => {
+  const result = await runTest('stdio.js');
+  expect(result.exitCode).toBe(0);
+  const data = JSON.parse(fs.readFileSync(path.join(__dirname, 'test-results', 'results.json')).toString());
+  const test = data.suites[0].tests[0];
+  const { stdout, stderr } = test;
+  expect(stdout).toEqual([{ text: 'stdout text' }, { buffer: Buffer.from('stdout buffer').toString('base64') }]);
+  expect(stderr).toEqual([{ text: 'stderr text' }, { buffer: Buffer.from('stderr buffer').toString('base64') }]);
+});
+
 async function runTest(filePath: string, timeout = 10000) {
   const outputDir = path.join(__dirname, 'test-results')
   await removeFolderAsync(outputDir).catch(e => {});
@@ -64,8 +74,14 @@ async function runTest(filePath: string, timeout = 10000) {
     path.join(__dirname, '..', 'cli.js'),
     path.join(__dirname, 'assets', filePath),
     '--output=' + outputDir,
-    '--timeout=' + timeout
-  ]);
+    '--timeout=' + timeout,
+    '--reporter=dot,json'
+  ], {
+    env: {
+      ...process.env,
+      PWRUNNER_JSON_REPORT: path.join(__dirname, 'test-results', 'results.json'),
+    }
+  });
   const passed = (/(\d+) passed/.exec(output.toString()) || [])[1];
   const failed = (/(\d+) failed/.exec(output.toString()) || [])[1];
   return {