Make ThrowableCollector configurable

This commit generalizes `ThrowableCollector` to take a predicate that
is used to decide whether a `Throwable` is aborting or failing. The
Jupiter engines uses a specialized implementation that treats OTA's
`TestAbortedExceptions` as aborting and everything else as failing:
`OpenTest4JAwareThrowableCollector`.

In addition, this commit introduces `ThrowableCollector.Factory` and
lets `HierarchicalTestEngines` create them in order to allow the engine
to decide how to configure its `ThrowableCollectors`. For backwards
compatibility, the default implementation returns a factory that
always creates instances of `OpenTest4JAwareThrowableCollector`.

Issue: #1313

junit-jupiter-engine/src/main/java/org/junit/jupiter/engine/descriptor/ClassTestDescriptor.java

@@ -56,6 +56,7 @@
 import org.junit.platform.engine.UniqueId;
 import org.junit.platform.engine.support.descriptor.ClassSource;
 import org.junit.platform.engine.support.hierarchical.ExclusiveResource;
+import org.junit.platform.engine.support.hierarchical.OpenTest4JAwareThrowableCollector;
 import org.junit.platform.engine.support.hierarchical.ThrowableCollector;
 
 /**
@@ -153,7 +154,7 @@ public JupiterEngineExecutionContext prepare(JupiterEngineExecutionContext conte
 		registerAfterEachMethodAdapters(registry);
 
 		Lifecycle lifecycle = getTestInstanceLifecycle(this.testClass, context.getConfigurationParameters());
-		ThrowableCollector throwableCollector = new ThrowableCollector();
+		ThrowableCollector throwableCollector = new OpenTest4JAwareThrowableCollector();
 		ClassExtensionContext extensionContext = new ClassExtensionContext(context.getExtensionContext(),
 			context.getExecutionListener(), this, lifecycle, context.getConfigurationParameters(), throwableCollector);
 
@@ -201,7 +202,8 @@ public JupiterEngineExecutionContext before(JupiterEngineExecutionContext contex
 	@Override
 	public void after(JupiterEngineExecutionContext context) {
 
-		Throwable previousThrowable = context.getThrowableCollector().getThrowable();
+		ThrowableCollector throwableCollector = context.getThrowableCollector();
+		Throwable previousThrowable = throwableCollector.getThrowable();
 
 		if (context.beforeAllMethodsExecuted()) {
 			invokeAfterAllMethods(context);
@@ -216,7 +218,9 @@ public void after(JupiterEngineExecutionContext context) {
 		// the execution of this Node. If an exception was already thrown, any
 		// later exceptions were added as suppressed exceptions to that original
 		// exception unless a more severe exception occurred in the meantime.
-		context.getThrowableCollector().assertNotSame(previousThrowable);
+		if (previousThrowable != throwableCollector.getThrowable()) {
+			throwableCollector.assertEmpty();
+		}
 	}
 
 	private TestInstanceFactory resolveTestInstanceFactory(ExtensionRegistry registry) {

junit-jupiter-engine/src/main/java/org/junit/jupiter/engine/descriptor/TestMethodTestDescriptor.java

@@ -34,6 +34,7 @@
 import org.junit.platform.commons.util.ExceptionUtils;
 import org.junit.platform.engine.TestDescriptor;
 import org.junit.platform.engine.UniqueId;
+import org.junit.platform.engine.support.hierarchical.OpenTest4JAwareThrowableCollector;
 import org.junit.platform.engine.support.hierarchical.ThrowableCollector;
 import org.junit.platform.engine.support.hierarchical.ThrowableCollector.Executable;
 
@@ -80,7 +81,7 @@ public JupiterEngineExecutionContext prepare(JupiterEngineExecutionContext conte
 		ExtensionRegistry registry = populateNewExtensionRegistry(context);
 		Object testInstance = context.getTestInstanceProvider().getTestInstance(Optional.of(registry));
 
-		ThrowableCollector throwableCollector = new ThrowableCollector();
+		ThrowableCollector throwableCollector = new OpenTest4JAwareThrowableCollector();
 		ExtensionContext extensionContext = new MethodExtensionContext(context.getExtensionContext(),
 			context.getExecutionListener(), this, context.getConfigurationParameters(), testInstance,
 			throwableCollector);

junit-jupiter-engine/src/main/java/org/junit/jupiter/engine/execution/ExtensionValuesStore.java

@@ -26,6 +26,7 @@
 import org.junit.jupiter.api.extension.ExtensionContext;
 import org.junit.jupiter.api.extension.ExtensionContext.Namespace;
 import org.junit.jupiter.api.extension.ExtensionContextException;
+import org.junit.platform.engine.support.hierarchical.OpenTest4JAwareThrowableCollector;
 import org.junit.platform.engine.support.hierarchical.ThrowableCollector;
 
 /**
@@ -51,7 +52,7 @@ public ExtensionValuesStore(ExtensionValuesStore parentStore) {
 	 * does not close values in parent stores.
 	 */
 	public void closeAllStoredCloseableValues() {
-		ThrowableCollector throwableCollector = new ThrowableCollector();
+		ThrowableCollector throwableCollector = new OpenTest4JAwareThrowableCollector();
 		for (Supplier<Object> supplier : storedValues.values()) {
 			Object value = supplier.get();
 			if (value instanceof ExtensionContext.Store.CloseableResource) {

junit-jupiter-engine/src/test/java/org/junit/jupiter/engine/descriptor/TestFactoryTestDescriptorTests.java

@@ -36,7 +36,7 @@
 import org.junit.platform.engine.support.descriptor.FileSource;
 import org.junit.platform.engine.support.descriptor.UriSource;
 import org.junit.platform.engine.support.hierarchical.Node;
-import org.junit.platform.engine.support.hierarchical.ThrowableCollector;
+import org.junit.platform.engine.support.hierarchical.OpenTest4JAwareThrowableCollector;
 
 /**
  * Unit tests for {@link TestFactoryTestDescriptor}.
@@ -121,7 +121,7 @@ void before() throws Exception {
 			isClosed = false;
 
 			context = new JupiterEngineExecutionContext(null, null).extend().withThrowableCollector(
-				new ThrowableCollector()).withExtensionContext(extensionContext).build();
+				new OpenTest4JAwareThrowableCollector()).withExtensionContext(extensionContext).build();
 
 			Method testMethod = CustomStreamTestCase.class.getDeclaredMethod("customStream");
 			descriptor = new TestFactoryTestDescriptor(UniqueId.forEngine("engine"), CustomStreamTestCase.class,

junit-jupiter-engine/src/test/java/org/junit/jupiter/engine/execution/ExtensionContextTests.java

@@ -43,7 +43,7 @@
 import org.junit.platform.engine.TestDescriptor;
 import org.junit.platform.engine.UniqueId;
 import org.junit.platform.engine.reporting.ReportEntry;
-import org.junit.platform.engine.support.hierarchical.ThrowableCollector;
+import org.junit.platform.engine.support.hierarchical.OpenTest4JAwareThrowableCollector;
 import org.mockito.ArgumentCaptor;
 import org.mockito.Mockito;
 
@@ -131,7 +131,7 @@ void tagsCanBeRetrievedInExtensionContext() {
 		assertThat(nestedExtensionContext.getRoot()).isSameAs(outerExtensionContext);
 
 		MethodExtensionContext methodExtensionContext = new MethodExtensionContext(outerExtensionContext, null,
-			methodTestDescriptor, configParams, new OuterClass(), new ThrowableCollector());
+			methodTestDescriptor, configParams, new OuterClass(), new OpenTest4JAwareThrowableCollector());
 		assertThat(methodExtensionContext.getTags()).containsExactlyInAnyOrder("outer-tag", "method-tag");
 		assertThat(methodExtensionContext.getRoot()).isSameAs(outerExtensionContext);
 	}
@@ -152,7 +152,7 @@ void fromMethodTestDescriptor() {
 		ClassExtensionContext classExtensionContext = new ClassExtensionContext(engineExtensionContext, null,
 			classTestDescriptor, configParams, null);
 		MethodExtensionContext methodExtensionContext = new MethodExtensionContext(classExtensionContext, null,
-			methodTestDescriptor, configParams, testInstance, new ThrowableCollector());
+			methodTestDescriptor, configParams, testInstance, new OpenTest4JAwareThrowableCollector());
 
 		// @formatter:off
 		assertAll("methodContext",
@@ -205,7 +205,7 @@ void usingStore() {
 		ClassTestDescriptor classTestDescriptor = outerClassDescriptor(methodTestDescriptor);
 		ExtensionContext parentContext = new ClassExtensionContext(null, null, classTestDescriptor, configParams, null);
 		MethodExtensionContext childContext = new MethodExtensionContext(parentContext, null, methodTestDescriptor,
-			configParams, new OuterClass(), new ThrowableCollector());
+			configParams, new OuterClass(), new OpenTest4JAwareThrowableCollector());
 
 		ExtensionContext.Store childStore = childContext.getStore(Namespace.GLOBAL);
 		ExtensionContext.Store parentStore = parentContext.getStore(Namespace.GLOBAL);

junit-platform-engine/src/main/java/org/junit/platform/engine/support/hierarchical/HierarchicalTestEngine.java

@@ -45,7 +45,10 @@ public abstract class HierarchicalTestEngine<C extends EngineExecutionContext> i
 	@Override
 	public final void execute(ExecutionRequest request) {
 		try (HierarchicalTestExecutorService executorService = createExecutorService(request)) {
-			new HierarchicalTestExecutor<>(request, createExecutionContext(request), executorService).execute().get();
+			C executionContext = createExecutionContext(request);
+			ThrowableCollector.Factory throwableCollectorFactory = createThrowableCollectorFactory(request);
+			new HierarchicalTestExecutor<>(request, executionContext, executorService,
+				throwableCollectorFactory).execute().get();
 		}
 		catch (Exception exception) {
 			throw new JUnitException("Error executing tests for engine " + getId(), exception);
@@ -74,6 +77,29 @@ protected HierarchicalTestExecutorService createExecutorService(ExecutionRequest
 		return new SameThreadHierarchicalTestExecutorService();
 	}
 
+	/**
+	 * Create the {@linkplain ThrowableCollector.Factory factory} for creating
+	 * {@link ThrowableCollector} instances used to handle exceptions that occur
+	 * during execution of this engine's tests.
+	 *
+	 * <p>An engine may use the information in the supplied <em>request</em>
+	 * such as the contained
+	 * {@linkplain ExecutionRequest#getConfigurationParameters() configuration parameters}
+	 * to decide what kind of factory to return or how to configure it.
+	 *
+	 * <p>By default, this method returns a factory that always creates instances of
+	 * {@link OpenTest4JAwareThrowableCollector}.
+	 *
+	 * @param request the request about to be executed
+	 * @see OpenTest4JAwareThrowableCollector
+	 * @see ThrowableCollector
+	 * @since 1.3
+	 */
+	@API(status = EXPERIMENTAL, since = "1.3")
+	protected ThrowableCollector.Factory createThrowableCollectorFactory(ExecutionRequest request) {
+		return OpenTest4JAwareThrowableCollector::new;
+	}
+
 	/**
 	 * Create the initial execution context for executing the supplied
 	 * {@linkplain ExecutionRequest request}.

junit-platform-engine/src/main/java/org/junit/platform/engine/support/hierarchical/HierarchicalTestExecutor.java

@@ -36,11 +36,14 @@ class HierarchicalTestExecutor<C extends EngineExecutionContext> {
 	private final ExecutionRequest request;
 	private final C rootContext;
 	private final HierarchicalTestExecutorService executorService;
+	private final ThrowableCollector.Factory throwableCollectorFactory;
 
-	HierarchicalTestExecutor(ExecutionRequest request, C rootContext, HierarchicalTestExecutorService executorService) {
+	HierarchicalTestExecutor(ExecutionRequest request, C rootContext, HierarchicalTestExecutorService executorService,
+			ThrowableCollector.Factory throwableCollectorFactory) {
 		this.request = request;
 		this.rootContext = rootContext;
 		this.executorService = executorService;
+		this.throwableCollectorFactory = throwableCollectorFactory;
 	}
 
 	Future<Void> execute() {
@@ -52,7 +55,8 @@ Future<Void> execute() {
 	NodeTestTask<C> prepareNodeTestTaskTree() {
 		TestDescriptor rootTestDescriptor = this.request.getRootTestDescriptor();
 		EngineExecutionListener executionListener = this.request.getEngineExecutionListener();
-		NodeTestTask<C> rootTestTask = new NodeTestTask<>(rootTestDescriptor, executionListener, this.executorService);
+		NodeTestTask<C> rootTestTask = new NodeTestTask<>(rootTestDescriptor, executionListener, this.executorService,
+			this.throwableCollectorFactory);
 		new NodeTestTaskWalker().walk(rootTestTask);
 		return rootTestTask;
 	}

junit-platform-engine/src/main/java/org/junit/platform/engine/support/hierarchical/NodeTestTask.java

@@ -31,10 +31,10 @@
  */
 class NodeTestTask<C extends EngineExecutionContext> implements TestTask {
 
-	private final ThrowableCollector throwableCollector = new ThrowableCollector();
 	private final TestDescriptor testDescriptor;
 	private final EngineExecutionListener listener;
 	private final HierarchicalTestExecutorService executorService;
+	private final ThrowableCollector.Factory throwableCollectorFactory;
 	private final Node<C> node;
 	private final ExecutionMode executionMode;
 	private final Set<ExclusiveResource> exclusiveResources;
@@ -48,18 +48,20 @@ class NodeTestTask<C extends EngineExecutionContext> implements TestTask {
 
 	private SkipResult skipResult;
 	private boolean started;
+	private ThrowableCollector throwableCollector;
 
 	NodeTestTask(TestDescriptor testDescriptor, EngineExecutionListener listener,
-			HierarchicalTestExecutorService executorService) {
+			HierarchicalTestExecutorService executorService, ThrowableCollector.Factory throwableCollectorFactory) {
 		this.testDescriptor = testDescriptor;
 		this.listener = listener;
 		this.executorService = executorService;
+		this.throwableCollectorFactory = throwableCollectorFactory;
 		node = asNode(testDescriptor);
 		executionMode = node.getExecutionMode();
 		exclusiveResources = node.getExclusiveResources();
 		// @formatter:off
 		children = testDescriptor.getChildren().stream()
-				.map(descriptor -> new NodeTestTask<C>(descriptor, listener, executorService))
+				.map(descriptor -> new NodeTestTask<C>(descriptor, listener, executorService, throwableCollectorFactory))
 				.collect(toCollection(ArrayList::new));
 		// @formatter:on
 	}
@@ -96,6 +98,7 @@ public void setParentContext(C parentContext) {
 
 	@Override
 	public void execute() {
+		throwableCollector = throwableCollectorFactory.create();
 		prepare();
 		if (throwableCollector.isEmpty()) {
 			checkWhetherSkipped();
@@ -142,7 +145,8 @@ private void executeRecursively() {
 
 	private void executeDynamicTest(TestDescriptor dynamicTestDescriptor, List<Future<?>> futures) {
 		listener.dynamicTestRegistered(dynamicTestDescriptor);
-		NodeTestTask<C> nodeTestTask = new NodeTestTask<>(dynamicTestDescriptor, listener, executorService);
+		NodeTestTask<C> nodeTestTask = new NodeTestTask<>(dynamicTestDescriptor, listener, executorService,
+			throwableCollectorFactory);
 		Set<ExclusiveResource> exclusiveResources = nodeTestTask.getExclusiveResources();
 		if (!exclusiveResources.isEmpty()) {
 			listener.executionStarted(dynamicTestDescriptor);
@@ -169,6 +173,7 @@ private void reportCompletion() {
 			listener.executionStarted(testDescriptor);
 		}
 		listener.executionFinished(testDescriptor, throwableCollector.toTestExecutionResult());
+		throwableCollector = null;
 	}
 
 	@SuppressWarnings("unchecked")

junit-platform-engine/src/main/java/org/junit/platform/engine/support/hierarchical/OpenTest4JAwareThrowableCollector.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright 2015-2018 the original author or authors.
+ *
+ * All rights reserved. This program and the accompanying materials are
+ * made available under the terms of the Eclipse Public License v2.0 which
+ * accompanies this distribution and is available at
+ *
+ * http://www.eclipse.org/legal/epl-v20.html
+ */
+
+package org.junit.platform.engine.support.hierarchical;
+
+import static org.apiguardian.api.API.Status.MAINTAINED;
+
+import org.apiguardian.api.API;
+import org.opentest4j.TestAbortedException;
+
+/**
+ * Specialization of {@link ThrowableCollector} that treats instances of
+ * {@link TestAbortedException} as <em>aborting</em>.
+ *
+ * @see ThrowableCollector
+ * @since 5.3
+ */
+@API(status = MAINTAINED, since = "5.3")
+public class OpenTest4JAwareThrowableCollector extends ThrowableCollector {
+
+	public OpenTest4JAwareThrowableCollector() {
+		super(TestAbortedException.class::isInstance);
+	}
+
+}