feat(testrunner): introduce pytest-style reporter

microsoft · Aug 24, 2020 · 1f2183a · 1f2183a
1 parent 2b3a1ae
commit 1f2183a
Show file tree

Hide file tree

Showing 9 changed files with 317 additions and 50 deletions.
diff --git a/test-runner/src/cli.ts b/test-runner/src/cli.ts
@@ -21,18 +21,20 @@ import { collectTests, runTests, RunnerConfig } from '.';
 import { DotReporter } from './reporters/dot';
 import { ListReporter } from './reporters/list';
 import { JSONReporter } from './reporters/json';
+import { PytestReporter } from './reporters/pytest';
 
 export const reporters = {
   'dot': DotReporter,
   'list': ListReporter,
-  'json': JSONReporter
+  'json': JSONReporter,
+  'pytest': PytestReporter,
 };
 
 program
   .version('Version ' + /** @type {any} */ (require)('../package.json').version)
   .option('--forbid-only', 'Fail if exclusive test(s) encountered', false)
   .option('-g, --grep <grep>', 'Only run tests matching this string or regexp', '.*')
-  .option('-j, --jobs <jobs>', 'Number of concurrent jobs for --parallel; use 1 to run in serial, default: (number of CPU cores / 2)', Math.ceil(require('os').cpus().length / 2).toString())
+  .option('-j, --jobs <jobs>', 'Number of concurrent jobs for --parallel; use 1 to run in serial, default: (number of CPU cores / 2)', Math.ceil(require('os').cpus().length / 2) as any)
   .option('--reporter <reporter>', 'Specify reporter to use', '')
   .option('--trial-run', 'Only collect the matching tests and report them as passing')
   .option('--quiet', 'Suppress stdio', false)

diff --git a/test-runner/src/reporters/base.ts b/test-runner/src/reporters/base.ts
@@ -32,6 +32,7 @@ export class BaseReporter implements Reporter  {
   pending: Test[] = [];
   passes: Test[] = [];
   failures: Test[] = [];
+  timeouts: Test[] = [];
   duration = 0;
   startTime: number;
   config: RunnerConfig;
@@ -62,7 +63,10 @@ export class BaseReporter implements Reporter  {
   }
 
   onFail(test: Test) {
-    this.failures.push(test);
+    if (test.duration >= test.timeout)
+      this.timeouts.push(test);
+    else
+      this.failures.push(test);
   }
 
   onEnd() {
@@ -72,43 +76,53 @@ export class BaseReporter implements Reporter  {
   epilogue() {
     console.log('');
 
-    console.log(colors.green(`  ${this.passes.length} passing`) + colors.dim(` (${milliseconds(this.duration)})`));  
+    console.log(colors.green(`  ${this.passes.length} passed`) + colors.dim(` (${milliseconds(this.duration)})`));  
 
     if (this.pending.length)
       console.log(colors.yellow(`  ${this.pending.length} skipped`));
 
-    if (this.failures.length) {  
-      console.log(colors.red(`  ${this.failures.length} failing`));
+    if (this.failures.length) {
+      console.log(colors.red(`  ${this.failures.length} failed`));
       console.log('');
-      this.failures.forEach((failure, index) => {
-        const relativePath = path.relative(process.cwd(), failure.file);
-        const header = `  ${index +1}. ${terminalLink(relativePath, `file://${os.hostname()}${failure.file}`)} › ${failure.title}`;
-        console.log(colors.bold(colors.red(header)));
-        const stack = failure.error.stack;
-        if (stack) {
-          console.log('');
-          const messageLocation = failure.error.stack.indexOf(failure.error.message);
-          const preamble = failure.error.stack.substring(0, messageLocation + failure.error.message.length);
-          console.log(indent(preamble, '    '));
-          const position = positionInFile(stack, failure.file);
-          if (position) {
-            const source = fs.readFileSync(failure.file, 'utf8');
-            console.log('');
-            console.log(indent(codeFrameColumns(source, {
-                start: position,
-              },
-              { highlightCode: true}
-            ), '    '));
-          }
-          console.log('');
-          console.log(indent(colors.dim(stack.substring(preamble.length + 1)), '    '));
-        } else {
+      this._printFailures(this.failures);
+    }
+
+    if (this.timeouts.length) {
+      console.log(colors.red(`  ${this.timeouts.length} timed out`));
+      console.log('');
+      this._printFailures(this.timeouts);
+    }
+  }
+
+  private _printFailures(failures: Test[]) {
+    failures.forEach((failure, index) => {
+      const relativePath = path.relative(process.cwd(), failure.file);
+      const header = `  ${index +1}. ${terminalLink(relativePath, `file://${os.hostname()}${failure.file}`)} › ${failure.title}`;
+      console.log(colors.bold(colors.red(header)));
+      const stack = failure.error.stack;
+      if (stack) {
+        console.log('');
+        const messageLocation = failure.error.stack.indexOf(failure.error.message);
+        const preamble = failure.error.stack.substring(0, messageLocation + failure.error.message.length);
+        console.log(indent(preamble, '    '));
+        const position = positionInFile(stack, failure.file);
+        if (position) {
+          const source = fs.readFileSync(failure.file, 'utf8');
           console.log('');
-          console.log(indent(String(failure.error), '    '));
+          console.log(indent(codeFrameColumns(source, {
+              start: position,
+            },
+            { highlightCode: true}
+          ), '    '));
         }
         console.log('');
-      });
-    }
+        console.log(indent(colors.dim(stack.substring(preamble.length + 1)), '    '));
+      } else {
+        console.log('');
+        console.log(indent(String(failure.error), '    '));
+      }
+      console.log('');
+    });
   }
 }
 

diff --git a/test-runner/src/reporters/dot.ts b/test-runner/src/reporters/dot.ts
@@ -26,7 +26,7 @@ export class DotReporter extends BaseReporter {
 
   onPass(test: Test) {
     super.onPass(test);
-    process.stdout.write(colors.green('\u00B7'));
+    process.stdout.write(colors.green('·'));
   }
 
   onFail(test: Test) {

diff --git a/test-runner/src/reporters/list.ts b/test-runner/src/reporters/list.ts
@@ -36,21 +36,21 @@ export class ListReporter extends BaseReporter {
     super.onPending(test);
     process.stdout.write(colors.green('  - ') + colors.cyan(test.fullTitle()));
     process.stdout.write('\n');
-}
+  }
 
   onPass(test: Test) {
     super.onPass(test);
     process.stdout.write('\u001b[2K\u001b[0G');
     process.stdout.write(colors.green('  ✓ ') + colors.gray(test.fullTitle()));
     process.stdout.write('\n');
-}
+  }
 
   onFail(test: Test) {
     super.onFail(test);
     process.stdout.write('\u001b[2K\u001b[0G');
     process.stdout.write(colors.red(`  ${++this._failure}) ` + test.fullTitle()));
     process.stdout.write('\n');
-}
+  }
 
   onEnd() {
     super.onEnd();