Skip to content

Commit 02145de

Browse files
marcphilippmarcphilipp
committed
Report problematic selector resolution results as discovery issues
Instead of aborting test discovery on the first problematic selector resolution result, they are now converted to discovery issues with `ERROR` severity and reported during execution. Issue: #242
1 parent 57fc651 commit 02145de

File tree

7 files changed

+103
-87
lines changed

7 files changed

+103
-87
lines changed

‎platform-tests/src/test/java/org/junit/platform/launcher/listeners/discovery/AbstractLauncherDiscoveryListenerTests.java ‎junit-platform-engine/src/testFixtures/java/org/junit/platform/fakes/FaultyTestEngines.java

+22-6
Original file line numberDiff line numberDiff line change
@@ -8,19 +8,20 @@
88
* https://www.eclipse.org/legal/epl-v20.html
99
*/
1010

11-
package org.junit.platform.launcher.listeners.discovery;
11+
package org.junit.platform.fakes;
1212

1313
import org.junit.platform.engine.DiscoverySelector;
1414
import org.junit.platform.engine.EngineDiscoveryRequest;
15+
import org.junit.platform.engine.ExecutionRequest;
1516
import org.junit.platform.engine.SelectorResolutionResult;
1617
import org.junit.platform.engine.TestDescriptor;
18+
import org.junit.platform.engine.TestExecutionResult;
1719
import org.junit.platform.engine.UniqueId;
1820
import org.junit.platform.engine.support.descriptor.EngineDescriptor;
19-
import org.junit.platform.fakes.TestEngineStub;
2021

21-
abstract class AbstractLauncherDiscoveryListenerTests {
22+
public class FaultyTestEngines {
2223

23-
protected TestEngineStub createEngineThatCannotResolveAnything(String engineId) {
24+
public static TestEngineStub createEngineThatCannotResolveAnything(String engineId) {
2425
return new TestEngineStub(engineId) {
2526
@Override
2627
public TestDescriptor discover(EngineDiscoveryRequest discoveryRequest, UniqueId uniqueId) {
@@ -29,10 +30,18 @@ public TestDescriptor discover(EngineDiscoveryRequest discoveryRequest, UniqueId
2930
selector, SelectorResolutionResult.unresolved()));
3031
return new EngineDescriptor(uniqueId, "Some Engine");
3132
}
33+
34+
@Override
35+
public void execute(ExecutionRequest request) {
36+
var listener = request.getEngineExecutionListener();
37+
var rootTestDescriptor = request.getRootTestDescriptor();
38+
listener.executionStarted(rootTestDescriptor);
39+
listener.executionFinished(rootTestDescriptor, TestExecutionResult.successful());
40+
}
3241
};
3342
}
3443

35-
protected TestEngineStub createEngineThatFailsToResolveAnything(String engineId, RuntimeException rootCause) {
44+
public static TestEngineStub createEngineThatFailsToResolveAnything(String engineId, Throwable rootCause) {
3645
return new TestEngineStub(engineId) {
3746
@Override
3847
public TestDescriptor discover(EngineDiscoveryRequest discoveryRequest, UniqueId uniqueId) {
@@ -41,7 +50,14 @@ public TestDescriptor discover(EngineDiscoveryRequest discoveryRequest, UniqueId
4150
selector, SelectorResolutionResult.failed(rootCause)));
4251
return new EngineDescriptor(uniqueId, "Some Engine");
4352
}
53+
54+
@Override
55+
public void execute(ExecutionRequest request) {
56+
var listener = request.getEngineExecutionListener();
57+
var rootTestDescriptor = request.getRootTestDescriptor();
58+
listener.executionStarted(rootTestDescriptor);
59+
listener.executionFinished(rootTestDescriptor, TestExecutionResult.successful());
60+
}
4461
};
4562
}
46-
4763
}

‎junit-platform-launcher/src/main/java/org/junit/platform/launcher/core/DiscoveryIssueCollector.java

+21
Original file line numberDiff line numberDiff line change
@@ -10,12 +10,18 @@
1010

1111
package org.junit.platform.launcher.core;
1212

13+
import static org.junit.platform.engine.SelectorResolutionResult.Status.FAILED;
14+
import static org.junit.platform.engine.SelectorResolutionResult.Status.UNRESOLVED;
15+
1316
import java.util.ArrayList;
1417
import java.util.List;
1518

1619
import org.junit.platform.engine.DiscoveryIssue;
1720
import org.junit.platform.engine.DiscoveryIssue.Severity;
21+
import org.junit.platform.engine.DiscoverySelector;
22+
import org.junit.platform.engine.SelectorResolutionResult;
1823
import org.junit.platform.engine.UniqueId;
24+
import org.junit.platform.engine.discovery.UniqueIdSelector;
1925
import org.junit.platform.launcher.LauncherDiscoveryListener;
2026

2127
class DiscoveryIssueCollector implements LauncherDiscoveryListener {
@@ -27,6 +33,21 @@ public void engineDiscoveryStarted(UniqueId engineId) {
2733
this.issues.clear();
2834
}
2935

36+
@Override
37+
public void selectorProcessed(UniqueId engineId, DiscoverySelector selector, SelectorResolutionResult result) {
38+
if (result.getStatus() == FAILED) {
39+
this.issues.add(DiscoveryIssue.builder(Severity.ERROR, selector + " resolution failed") //
40+
.cause(result.getThrowable()) //
41+
.build());
42+
}
43+
else if (result.getStatus() == UNRESOLVED && selector instanceof UniqueIdSelector) {
44+
UniqueId uniqueId = ((UniqueIdSelector) selector).getUniqueId();
45+
if (uniqueId.hasPrefix(engineId)) {
46+
this.issues.add(DiscoveryIssue.create(Severity.ERROR, selector + " could not be resolved"));
47+
}
48+
}
49+
}
50+
3051
@Override
3152
public void issueEncountered(UniqueId engineId, DiscoveryIssue issue) {
3253
this.issues.add(issue);

‎junit-platform-launcher/src/main/java/org/junit/platform/launcher/listeners/discovery/AbortOnFailureLauncherDiscoveryListener.java

-20
Original file line numberDiff line numberDiff line change
@@ -10,15 +10,8 @@
1010

1111
package org.junit.platform.launcher.listeners.discovery;
1212

13-
import static org.junit.platform.engine.SelectorResolutionResult.Status.FAILED;
14-
import static org.junit.platform.engine.SelectorResolutionResult.Status.UNRESOLVED;
15-
16-
import org.junit.platform.commons.JUnitException;
1713
import org.junit.platform.commons.util.ExceptionUtils;
18-
import org.junit.platform.engine.DiscoverySelector;
19-
import org.junit.platform.engine.SelectorResolutionResult;
2014
import org.junit.platform.engine.UniqueId;
21-
import org.junit.platform.engine.discovery.UniqueIdSelector;
2215
import org.junit.platform.launcher.EngineDiscoveryResult;
2316
import org.junit.platform.launcher.LauncherDiscoveryListener;
2417

@@ -33,19 +26,6 @@ public void engineDiscoveryFinished(UniqueId engineId, EngineDiscoveryResult res
3326
result.getThrowable().ifPresent(ExceptionUtils::throwAsUncheckedException);
3427
}
3528

36-
@Override
37-
public void selectorProcessed(UniqueId engineId, DiscoverySelector selector, SelectorResolutionResult result) {
38-
if (result.getStatus() == FAILED) {
39-
throw new JUnitException(selector + " resolution failed", result.getThrowable().orElse(null));
40-
}
41-
if (result.getStatus() == UNRESOLVED && selector instanceof UniqueIdSelector) {
42-
UniqueId uniqueId = ((UniqueIdSelector) selector).getUniqueId();
43-
if (uniqueId.hasPrefix(engineId)) {
44-
throw new JUnitException(selector + " could not be resolved");
45-
}
46-
}
47-
}
48-
4929
@Override
5030
public boolean equals(Object obj) {
5131
if (this == obj) {

‎jupiter-tests/src/test/java/org/junit/jupiter/engine/NestedTestClassesTests.java

+8-10
Original file line numberDiff line numberDiff line change
@@ -10,15 +10,14 @@
1010

1111
package org.junit.jupiter.engine;
1212

13-
import static org.assertj.core.api.Assertions.assertThat;
14-
import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
15-
import static org.assertj.core.util.Throwables.getRootCause;
1613
import static org.junit.jupiter.api.Assertions.assertAll;
1714
import static org.junit.jupiter.api.Assertions.assertEquals;
1815
import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass;
1916
import static org.junit.platform.engine.discovery.DiscoverySelectors.selectMethod;
2017
import static org.junit.platform.engine.discovery.DiscoverySelectors.selectUniqueId;
2118
import static org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder.request;
19+
import static org.junit.platform.testkit.engine.EventConditions.finishedWithFailure;
20+
import static org.junit.platform.testkit.engine.TestExecutionResultConditions.message;
2221

2322
import org.junit.jupiter.api.AfterEach;
2423
import org.junit.jupiter.api.Assertions;
@@ -28,7 +27,6 @@
2827
import org.junit.jupiter.engine.NestedTestClassesTests.OuterClass.NestedClass;
2928
import org.junit.jupiter.engine.NestedTestClassesTests.OuterClass.NestedClass.RecursiveNestedClass;
3029
import org.junit.jupiter.engine.NestedTestClassesTests.OuterClass.NestedClass.RecursiveNestedSiblingClass;
31-
import org.junit.platform.commons.JUnitException;
3230
import org.junit.platform.engine.TestDescriptor;
3331
import org.junit.platform.launcher.LauncherDiscoveryRequest;
3432
import org.junit.platform.testkit.engine.EngineExecutionResults;
@@ -180,12 +178,12 @@ void individualMethodsWithinRecursiveNestedTestClassHierarchiesAreExecuted() {
180178
}
181179

182180
private void assertNestedCycle(Class<?> start, Class<?> from, Class<?> to) {
183-
assertThatExceptionOfType(JUnitException.class)//
184-
.isThrownBy(() -> executeTestsForClass(start))//
185-
.withCauseExactlyInstanceOf(JUnitException.class)//
186-
.satisfies(ex -> assertThat(getRootCause(ex)).hasMessageMatching(
187-
String.format("Detected cycle in inner class hierarchy between .+%s and .+%s", from.getSimpleName(),
188-
to.getSimpleName())));
181+
var results = executeTestsForClass(start);
182+
var expectedMessage = String.format(
183+
"Cause: org.junit.platform.commons.JUnitException: Detected cycle in inner class hierarchy between %s and %s",
184+
from.getName(), to.getName());
185+
results.containerEvents().assertThatEvents() //
186+
.haveExactly(1, finishedWithFailure(message(it -> it.contains(expectedMessage))));
189187
}
190188

191189
// -------------------------------------------------------------------

‎platform-tests/src/test/java/org/junit/platform/launcher/core/DefaultLauncherTests.java

+48-3
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
package org.junit.platform.launcher.core;
1212

1313
import static java.util.Objects.requireNonNull;
14+
import static java.util.function.UnaryOperator.identity;
1415
import static org.assertj.core.api.Assertions.assertThat;
1516
import static org.junit.jupiter.api.Assertions.assertAll;
1617
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -19,8 +20,11 @@
1920
import static org.junit.platform.commons.util.CollectionUtils.getOnlyElement;
2021
import static org.junit.platform.engine.SelectorResolutionResult.unresolved;
2122
import static org.junit.platform.engine.TestExecutionResult.successful;
23+
import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass;
2224
import static org.junit.platform.engine.discovery.DiscoverySelectors.selectPackage;
2325
import static org.junit.platform.engine.discovery.DiscoverySelectors.selectUniqueId;
26+
import static org.junit.platform.fakes.FaultyTestEngines.createEngineThatCannotResolveAnything;
27+
import static org.junit.platform.fakes.FaultyTestEngines.createEngineThatFailsToResolveAnything;
2428
import static org.junit.platform.launcher.LauncherConstants.DRY_RUN_PROPERTY_NAME;
2529
import static org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder.DEFAULT_DISCOVERY_LISTENER_CONFIGURATION_PROPERTY_NAME;
2630
import static org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder.request;
@@ -38,6 +42,7 @@
3842
import java.time.Instant;
3943
import java.util.Set;
4044
import java.util.concurrent.atomic.AtomicReference;
45+
import java.util.function.UnaryOperator;
4146
import java.util.logging.Level;
4247
import java.util.logging.LogRecord;
4348

@@ -844,7 +849,47 @@ public void execute(ExecutionRequest request) {
844849
.isBetween(result.startTime(), result.finishTime());
845850
}
846851

852+
@Test
853+
void reportsEngineExecutionFailureOnUnresolvedUniqueIdSelectorWithEnginePrefix() {
854+
var engine = createEngineThatCannotResolveAnything("some-engine");
855+
var selector = selectUniqueId(UniqueId.forEngine(engine.getId()));
856+
var result = execute(engine, request -> request.selectors(selector));
857+
858+
assertThat(result.testExecutionResult().getStatus()).isEqualTo(Status.FAILED);
859+
assertThat(result.testExecutionResult().getThrowable().orElseThrow()) //
860+
.hasMessageStartingWith(
861+
"TestEngine with ID 'some-engine' encountered a critical issue during test discovery") //
862+
.hasMessageContaining("(1) [ERROR] %s could not be resolved", selector);
863+
}
864+
865+
@Test
866+
void ignoresUnresolvedUniqueIdSelectorWithoutEnginePrefix() {
867+
var engine = createEngineThatCannotResolveAnything("some-engine");
868+
var selector = selectUniqueId(UniqueId.forEngine("some-other-engine"));
869+
var result = execute(engine, request -> request.selectors(selector));
870+
871+
assertThat(result.testExecutionResult().getStatus()).isEqualTo(Status.SUCCESSFUL);
872+
}
873+
874+
@Test
875+
void reportsEngineExecutionFailureForSelectorResolutionFailure() {
876+
var engine = createEngineThatFailsToResolveAnything("some-engine", new RuntimeException("boom"));
877+
var selector = selectClass(Object.class);
878+
var result = execute(engine, request -> request.selectors(selector));
879+
880+
assertThat(result.testExecutionResult().getStatus()).isEqualTo(Status.FAILED);
881+
assertThat(result.testExecutionResult().getThrowable().orElseThrow()) //
882+
.hasMessageStartingWith(
883+
"TestEngine with ID 'some-engine' encountered a critical issue during test discovery") //
884+
.hasMessageContaining("(1) [ERROR] %s resolution failed", selector) //
885+
.hasMessageContaining("Cause: java.lang.RuntimeException: boom");
886+
}
887+
847888
private static ReportedData execute(TestEngine engine) {
889+
return execute(engine, identity());
890+
}
891+
892+
private static ReportedData execute(TestEngine engine, UnaryOperator<LauncherDiscoveryRequestBuilder> configurer) {
848893
var executionListener = mock(TestExecutionListener.class);
849894

850895
AtomicReference<Instant> startTime = new AtomicReference<>();
@@ -859,9 +904,9 @@ private static ReportedData execute(TestEngine engine) {
859904
return null;
860905
}).when(executionListener).executionFinished(any(), any());
861906

862-
var request = request() //
863-
.configurationParameter(DEFAULT_DISCOVERY_LISTENER_CONFIGURATION_PROPERTY_NAME, "logging") //
864-
.build();
907+
var builder = request() //
908+
.configurationParameter(DEFAULT_DISCOVERY_LISTENER_CONFIGURATION_PROPERTY_NAME, "logging");
909+
var request = configurer.apply(builder).build();
865910
var launcher = createLauncher(engine);
866911

867912
var testPlan = launcher.discover(request);

‎platform-tests/src/test/java/org/junit/platform/launcher/listeners/discovery/AbortOnFailureLauncherDiscoveryListenerTests.java

+1-47
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,7 @@
1111
package org.junit.platform.launcher.listeners.discovery;
1212

1313
import static org.assertj.core.api.Assertions.assertThat;
14-
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
1514
import static org.junit.jupiter.api.Assertions.assertThrows;
16-
import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass;
1715
import static org.junit.platform.engine.discovery.DiscoverySelectors.selectUniqueId;
1816
import static org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder.request;
1917
import static org.junit.platform.launcher.core.LauncherFactoryForTestingPurposesOnly.createLauncher;
@@ -26,51 +24,7 @@
2624
import org.junit.platform.engine.UniqueId;
2725
import org.junit.platform.fakes.TestEngineStub;
2826

29-
class AbortOnFailureLauncherDiscoveryListenerTests extends AbstractLauncherDiscoveryListenerTests {
30-
31-
@Test
32-
void abortsDiscoveryOnUnresolvedUniqueIdSelectorWithEnginePrefix() {
33-
var engine = createEngineThatCannotResolveAnything("some-engine");
34-
var request = request() //
35-
.listeners(abortOnFailure()) //
36-
.selectors(selectUniqueId(UniqueId.forEngine(engine.getId()))) //
37-
.build();
38-
var launcher = createLauncher(engine);
39-
40-
var exception = assertThrows(JUnitException.class, () -> launcher.discover(request));
41-
assertThat(exception).hasMessage("TestEngine with ID 'some-engine' failed to discover tests");
42-
assertThat(exception.getCause()).hasMessage(
43-
"UniqueIdSelector [uniqueId = [engine:some-engine]] could not be resolved");
44-
}
45-
46-
@Test
47-
void doesNotAbortDiscoveryOnUnresolvedUniqueIdSelectorWithoutEnginePrefix() {
48-
var engine = createEngineThatCannotResolveAnything("some-engine");
49-
var request = request() //
50-
.listeners(abortOnFailure()) //
51-
.selectors(selectUniqueId(UniqueId.forEngine("some-other-engine"))) //
52-
.build();
53-
var launcher = createLauncher(engine);
54-
55-
assertDoesNotThrow(() -> launcher.discover(request));
56-
}
57-
58-
@Test
59-
void abortsDiscoveryOnSelectorResolutionFailure() {
60-
var rootCause = new RuntimeException();
61-
var engine = createEngineThatFailsToResolveAnything("some-engine", rootCause);
62-
var request = request() //
63-
.listeners(abortOnFailure()) //
64-
.selectors(selectClass(Object.class)) //
65-
.build();
66-
var launcher = createLauncher(engine);
67-
68-
var exception = assertThrows(JUnitException.class, () -> launcher.discover(request));
69-
assertThat(exception).hasMessage("TestEngine with ID 'some-engine' failed to discover tests");
70-
assertThat(exception.getCause()) //
71-
.hasMessageEndingWith("resolution failed") //
72-
.cause().isSameAs(rootCause);
73-
}
27+
class AbortOnFailureLauncherDiscoveryListenerTests {
7428

7529
@Test
7630
void abortsDiscoveryOnEngineDiscoveryFailure() {

‎platform-tests/src/test/java/org/junit/platform/launcher/listeners/discovery/LoggingLauncherDiscoveryListenerTests.java

+3-1
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@
1313
import static org.assertj.core.api.Assertions.assertThat;
1414
import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass;
1515
import static org.junit.platform.engine.discovery.DiscoverySelectors.selectUniqueId;
16+
import static org.junit.platform.fakes.FaultyTestEngines.createEngineThatCannotResolveAnything;
17+
import static org.junit.platform.fakes.FaultyTestEngines.createEngineThatFailsToResolveAnything;
1618
import static org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder.DEFAULT_DISCOVERY_LISTENER_CONFIGURATION_PROPERTY_NAME;
1719
import static org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder.request;
1820
import static org.junit.platform.launcher.core.LauncherFactoryForTestingPurposesOnly.createLauncher;
@@ -29,7 +31,7 @@
2931
import org.junit.platform.fakes.TestEngineStub;
3032

3133
@TrackLogRecords
32-
public class LoggingLauncherDiscoveryListenerTests extends AbstractLauncherDiscoveryListenerTests {
34+
public class LoggingLauncherDiscoveryListenerTests {
3335

3436
@Test
3537
void logsWarningOnUnresolvedUniqueIdSelectorWithEnginePrefix(LogRecordListener log) {

0 commit comments

Comments
 (0)