Report discovery issues for invalid suite lifecycle methods

junit-platform-suite-engine/src/main/java/org/junit/platform/suite/engine/ClassSelectorResolver.java

@@ -26,6 +26,7 @@
 import org.junit.platform.engine.discovery.ClassSelector;
 import org.junit.platform.engine.discovery.UniqueIdSelector;
 import org.junit.platform.engine.reporting.OutputDirectoryProvider;
+import org.junit.platform.engine.support.discovery.DiscoveryIssueReporter;
 import org.junit.platform.engine.support.discovery.SelectorResolver;
 
 /**
@@ -41,13 +42,16 @@ final class ClassSelectorResolver implements SelectorResolver {
 	private final SuiteEngineDescriptor suiteEngineDescriptor;
 	private final ConfigurationParameters configurationParameters;
 	private final OutputDirectoryProvider outputDirectoryProvider;
+	private final DiscoveryIssueReporter issueReporter;
 
 	ClassSelectorResolver(Predicate<String> classNameFilter, SuiteEngineDescriptor suiteEngineDescriptor,
-			ConfigurationParameters configurationParameters, OutputDirectoryProvider outputDirectoryProvider) {
+			ConfigurationParameters configurationParameters, OutputDirectoryProvider outputDirectoryProvider,
+			DiscoveryIssueReporter issueReporter) {
 		this.classNameFilter = classNameFilter;
 		this.suiteEngineDescriptor = suiteEngineDescriptor;
 		this.configurationParameters = configurationParameters;
 		this.outputDirectoryProvider = outputDirectoryProvider;
+		this.issueReporter = issueReporter;
 	}
 
 	@Override
@@ -106,7 +110,8 @@ private Optional<SuiteTestDescriptor> newSuiteDescriptor(Class<?> suiteClass, Te
 			return Optional.empty();
 		}
 
-		return Optional.of(new SuiteTestDescriptor(id, suiteClass, configurationParameters, outputDirectoryProvider));
+		return Optional.of(
+			new SuiteTestDescriptor(id, suiteClass, configurationParameters, outputDirectoryProvider, issueReporter));
 	}
 
 	private static boolean containsCycle(UniqueId id) {

junit-platform-suite-engine/src/main/java/org/junit/platform/suite/engine/DiscoverySelectorResolver.java

@@ -26,7 +26,8 @@ final class DiscoverySelectorResolver {
 					context.getClassNameFilter(),
 					context.getEngineDescriptor(),
 					context.getDiscoveryRequest().getConfigurationParameters(),
-					context.getDiscoveryRequest().getOutputDirectoryProvider()))
+					context.getDiscoveryRequest().getOutputDirectoryProvider(),
+					context.getIssueReporter()))
 			.build();
 	// @formatter:on
 

junit-platform-suite-engine/src/main/java/org/junit/platform/suite/engine/LifecycleMethodUtils.java

@@ -11,16 +11,20 @@
 package org.junit.platform.suite.engine;
 
 import static org.junit.platform.commons.support.AnnotationSupport.findAnnotatedMethods;
-import static org.junit.platform.commons.util.ReflectionUtils.returnsPrimitiveVoid;
+import static org.junit.platform.commons.util.CollectionUtils.toUnmodifiableList;
+import static org.junit.platform.engine.support.discovery.DiscoveryIssueReporter.Condition.allOf;
 
 import java.lang.annotation.Annotation;
 import java.lang.reflect.Method;
 import java.util.List;
 
-import org.junit.platform.commons.JUnitException;
 import org.junit.platform.commons.support.HierarchyTraversalMode;
 import org.junit.platform.commons.support.ModifierSupport;
-import org.junit.platform.engine.support.hierarchical.ThrowableCollector;
+import org.junit.platform.commons.util.ReflectionUtils;
+import org.junit.platform.engine.DiscoveryIssue;
+import org.junit.platform.engine.DiscoveryIssue.Severity;
+import org.junit.platform.engine.support.descriptor.MethodSource;
+import org.junit.platform.engine.support.discovery.DiscoveryIssueReporter;
 import org.junit.platform.suite.api.AfterSuite;
 import org.junit.platform.suite.api.BeforeSuite;
 
@@ -35,56 +39,68 @@ private LifecycleMethodUtils() {
 		/* no-op */
 	}
 
-	static List<Method> findBeforeSuiteMethods(Class<?> testClass, ThrowableCollector throwableCollector) {
-		return findMethodsAndAssertStaticAndNonPrivate(testClass, BeforeSuite.class, HierarchyTraversalMode.TOP_DOWN,
-			throwableCollector);
+	static List<Method> findBeforeSuiteMethods(Class<?> testClass, DiscoveryIssueReporter issueReporter) {
+		return findMethodsAndCheckStaticAndNonPrivate(testClass, BeforeSuite.class, HierarchyTraversalMode.TOP_DOWN,
+			issueReporter);
 	}
 
-	static List<Method> findAfterSuiteMethods(Class<?> testClass, ThrowableCollector throwableCollector) {
-		return findMethodsAndAssertStaticAndNonPrivate(testClass, AfterSuite.class, HierarchyTraversalMode.BOTTOM_UP,
-			throwableCollector);
+	static List<Method> findAfterSuiteMethods(Class<?> testClass, DiscoveryIssueReporter issueReporter) {
+		return findMethodsAndCheckStaticAndNonPrivate(testClass, AfterSuite.class, HierarchyTraversalMode.BOTTOM_UP,
+			issueReporter);
 	}
 
-	private static List<Method> findMethodsAndAssertStaticAndNonPrivate(Class<?> testClass,
+	private static List<Method> findMethodsAndCheckStaticAndNonPrivate(Class<?> testClass,
 			Class<? extends Annotation> annotationType, HierarchyTraversalMode traversalMode,
-			ThrowableCollector throwableCollector) {
-
-		List<Method> methods = findAnnotatedMethods(testClass, annotationType, traversalMode);
-		throwableCollector.execute(() -> methods.forEach(method -> {
-			assertVoid(annotationType, method);
-			assertStatic(annotationType, method);
-			assertNonPrivate(annotationType, method);
-			assertNoParameters(annotationType, method);
-		}));
-		return methods;
+			DiscoveryIssueReporter issueReporter) {
+
+		return findAnnotatedMethods(testClass, annotationType, traversalMode).stream() //
+				.filter(allOf( //
+					returnsPrimitiveVoid(annotationType, issueReporter), //
+					isStatic(annotationType, issueReporter), //
+					isNotPrivate(annotationType, issueReporter), //
+					hasNoParameters(annotationType, issueReporter) //
+				)) //
+				.collect(toUnmodifiableList());
+	}
+
+	private static DiscoveryIssueReporter.Condition<Method> isStatic(Class<? extends Annotation> annotationType,
+			DiscoveryIssueReporter issueReporter) {
+		return issueReporter.createReportingCondition(ModifierSupport::isStatic, method -> {
+			String message = String.format("@%s method '%s' must be static.", annotationType.getSimpleName(),
+				method.toGenericString());
+			return createError(message, method);
+		});
 	}
 
-	private static void assertStatic(Class<? extends Annotation> annotationType, Method method) {
-		if (ModifierSupport.isNotStatic(method)) {
-			throw new JUnitException(String.format("@%s method '%s' must be static.", annotationType.getSimpleName(),
-				method.toGenericString()));
-		}
+	private static DiscoveryIssueReporter.Condition<Method> isNotPrivate(Class<? extends Annotation> annotationType,
+			DiscoveryIssueReporter issueReporter) {
+		return issueReporter.createReportingCondition(ModifierSupport::isNotPrivate, method -> {
+			String message = String.format("@%s method '%s' must not be private.", annotationType.getSimpleName(),
+				method.toGenericString());
+			return createError(message, method);
+		});
 	}
 
-	private static void assertNonPrivate(Class<? extends Annotation> annotationType, Method method) {
-		if (ModifierSupport.isPrivate(method)) {
-			throw new JUnitException(String.format("@%s method '%s' must not be private.",
-				annotationType.getSimpleName(), method.toGenericString()));
-		}
+	private static DiscoveryIssueReporter.Condition<Method> returnsPrimitiveVoid(
+			Class<? extends Annotation> annotationType, DiscoveryIssueReporter issueReporter) {
+		return issueReporter.createReportingCondition(ReflectionUtils::returnsPrimitiveVoid, method -> {
+			String message = String.format("@%s method '%s' must not return a value.", annotationType.getSimpleName(),
+				method.toGenericString());
+			return createError(message, method);
+		});
 	}
 
-	private static void assertVoid(Class<? extends Annotation> annotationType, Method method) {
-		if (!returnsPrimitiveVoid(method)) {
-			throw new JUnitException(String.format("@%s method '%s' must not return a value.",
-				annotationType.getSimpleName(), method.toGenericString()));
-		}
+	private static DiscoveryIssueReporter.Condition<Method> hasNoParameters(Class<? extends Annotation> annotationType,
+			DiscoveryIssueReporter issueReporter) {
+		return issueReporter.createReportingCondition(method -> method.getParameterCount() == 0, method -> {
+			String message = String.format("@%s method '%s' must not accept parameters.",
+				annotationType.getSimpleName(), method.toGenericString());
+			return createError(message, method);
+		});
 	}
 
-	private static void assertNoParameters(Class<? extends Annotation> annotationType, Method method) {
-		if (method.getParameterCount() > 0) {
-			throw new JUnitException(String.format("@%s method '%s' must not accept parameters.",
-				annotationType.getSimpleName(), method.toGenericString()));
-		}
+	private static DiscoveryIssue createError(String message, Method method) {
+		return DiscoveryIssue.builder(Severity.ERROR, message).source(MethodSource.from(method)).build();
 	}
 
 }

junit-platform-suite-engine/src/main/java/org/junit/platform/suite/engine/SuiteTestDescriptor.java

@@ -29,6 +29,7 @@
 import org.junit.platform.engine.reporting.OutputDirectoryProvider;
 import org.junit.platform.engine.support.descriptor.AbstractTestDescriptor;
 import org.junit.platform.engine.support.descriptor.ClassSource;
+import org.junit.platform.engine.support.discovery.DiscoveryIssueReporter;
 import org.junit.platform.engine.support.hierarchical.OpenTest4JAwareThrowableCollector;
 import org.junit.platform.engine.support.hierarchical.ThrowableCollector;
 import org.junit.platform.launcher.LauncherDiscoveryRequest;
@@ -57,17 +58,19 @@ final class SuiteTestDescriptor extends AbstractTestDescriptor {
 	private final OutputDirectoryProvider outputDirectoryProvider;
 	private final Boolean failIfNoTests;
 	private final Class<?> suiteClass;
+	private final LifecycleMethods lifecycleMethods;
 
 	private LauncherDiscoveryResult launcherDiscoveryResult;
 	private SuiteLauncher launcher;
 
 	SuiteTestDescriptor(UniqueId id, Class<?> suiteClass, ConfigurationParameters configurationParameters,
-			OutputDirectoryProvider outputDirectoryProvider) {
+			OutputDirectoryProvider outputDirectoryProvider, DiscoveryIssueReporter issueReporter) {
 		super(id, getSuiteDisplayName(suiteClass), ClassSource.from(suiteClass));
 		this.configurationParameters = configurationParameters;
 		this.outputDirectoryProvider = outputDirectoryProvider;
 		this.failIfNoTests = getFailIfNoTests(suiteClass);
 		this.suiteClass = suiteClass;
+		this.lifecycleMethods = new LifecycleMethods(suiteClass, issueReporter);
 	}
 
 	private static Boolean getFailIfNoTests(Class<?> suiteClass) {
@@ -134,24 +137,21 @@ void execute(EngineExecutionListener parentEngineExecutionListener) {
 		parentEngineExecutionListener.executionStarted(this);
 		ThrowableCollector throwableCollector = new OpenTest4JAwareThrowableCollector();
 
-		List<Method> beforeSuiteMethods = LifecycleMethodUtils.findBeforeSuiteMethods(suiteClass, throwableCollector);
-		List<Method> afterSuiteMethods = LifecycleMethodUtils.findAfterSuiteMethods(suiteClass, throwableCollector);
-
-		executeBeforeSuiteMethods(beforeSuiteMethods, throwableCollector);
+		executeBeforeSuiteMethods(throwableCollector);
 
 		TestExecutionSummary summary = executeTests(parentEngineExecutionListener, throwableCollector);
 
-		executeAfterSuiteMethods(afterSuiteMethods, throwableCollector);
+		executeAfterSuiteMethods(throwableCollector);
 
 		TestExecutionResult testExecutionResult = computeTestExecutionResult(summary, throwableCollector);
 		parentEngineExecutionListener.executionFinished(this, testExecutionResult);
 	}
 
-	private void executeBeforeSuiteMethods(List<Method> beforeSuiteMethods, ThrowableCollector throwableCollector) {
+	private void executeBeforeSuiteMethods(ThrowableCollector throwableCollector) {
 		if (throwableCollector.isNotEmpty()) {
 			return;
 		}
-		for (Method beforeSuiteMethod : beforeSuiteMethods) {
+		for (Method beforeSuiteMethod : lifecycleMethods.beforeSuite) {
 			throwableCollector.execute(() -> ReflectionSupport.invokeMethod(beforeSuiteMethod, null));
 			if (throwableCollector.isNotEmpty()) {
 				return;
@@ -173,8 +173,8 @@ private TestExecutionSummary executeTests(EngineExecutionListener parentEngineEx
 		return launcher.execute(discoveryResult, parentEngineExecutionListener);
 	}
 
-	private void executeAfterSuiteMethods(List<Method> afterSuiteMethods, ThrowableCollector throwableCollector) {
-		for (Method afterSuiteMethod : afterSuiteMethods) {
+	private void executeAfterSuiteMethods(ThrowableCollector throwableCollector) {
+		for (Method afterSuiteMethod : lifecycleMethods.afterSuite) {
 			throwableCollector.execute(() -> ReflectionSupport.invokeMethod(afterSuiteMethod, null));
 		}
 	}
@@ -198,4 +198,15 @@ public boolean mayRegisterTests() {
 		return true;
 	}
 
+	private static class LifecycleMethods {
+
+		final List<Method> beforeSuite;
+		final List<Method> afterSuite;
+
+		LifecycleMethods(Class<?> suiteClass, DiscoveryIssueReporter issueReporter) {
+			beforeSuite = LifecycleMethodUtils.findBeforeSuiteMethods(suiteClass, issueReporter);
+			afterSuite = LifecycleMethodUtils.findAfterSuiteMethods(suiteClass, issueReporter);
+		}
+	}
+
 }

junit-platform-suite-engine/src/nativeImage/initialize-at-build-time

@@ -1,4 +1,5 @@
 org.junit.platform.suite.engine.SuiteEngineDescriptor
 org.junit.platform.suite.engine.SuiteLauncher
 org.junit.platform.suite.engine.SuiteTestDescriptor
+org.junit.platform.suite.engine.SuiteTestDescriptor$LifecycleMethods
 org.junit.platform.suite.engine.SuiteTestEngine

platform-tests/src/test/java/org/junit/platform/suite/engine/BeforeAndAfterSuiteTests.java

@@ -13,6 +13,7 @@
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Named.named;
 import static org.junit.jupiter.params.provider.Arguments.arguments;
+import static org.junit.platform.commons.util.CollectionUtils.getOnlyElement;
 import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass;
 import static org.junit.platform.suite.engine.SuiteEngineDescriptor.ENGINE_ID;
 import static org.junit.platform.suite.engine.testsuites.LifecycleMethodsSuites.FailingAfterSuite;
@@ -47,7 +48,7 @@
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.Arguments;
 import org.junit.jupiter.params.provider.MethodSource;
-import org.junit.platform.commons.JUnitException;
+import org.junit.platform.engine.DiscoveryIssue.Severity;
 import org.junit.platform.suite.api.AfterSuite;
 import org.junit.platform.suite.api.BeforeSuite;
 import org.junit.platform.suite.engine.testcases.StatefulTestCase;
@@ -192,14 +193,12 @@ void severalFailingBeforeAndAfterSuite() {
 	@ParameterizedTest(name = "{0}")
 	@MethodSource
 	void invalidBeforeOrAfterSuiteMethod(Class<?> testSuiteClass, Predicate<String> failureMessagePredicate) {
-		// @formatter:off
-		executeSuite(testSuiteClass)
-				.allEvents()
-				.assertThatEvents()
-				.haveExactly(1, event(
-						container(testSuiteClass),
-						finishedWithFailure(instanceOf(JUnitException.class), message(failureMessagePredicate))));
-		// @formatter:on
+		var results = engineWithSelectedSuite(testSuiteClass).discover();
+
+		var issue = getOnlyElement(results.getDiscoveryIssues());
+		assertThat(issue.severity()).isEqualTo(Severity.ERROR);
+		assertThat(issue.message()).matches(failureMessagePredicate);
+		assertThat(issue.source()).containsInstanceOf(org.junit.platform.engine.support.descriptor.MethodSource.class);
 	}
 
 	private static Stream<Arguments> invalidBeforeOrAfterSuiteMethod() {
@@ -219,11 +218,19 @@ private static Stream<Arguments> invalidBeforeOrAfterSuiteMethod() {
 	private static Arguments invalidBeforeOrAfterSuiteCase(Class<?> suiteClass, String failureMessageStart,
 			String failureMessageEnd) {
 		return arguments(named(suiteClass.getSimpleName(), suiteClass),
-			(Predicate<String>) s -> s.startsWith(failureMessageStart) && s.endsWith(failureMessageEnd));
+			expectedMessage(failureMessageStart, failureMessageEnd));
+	}
+
+	private static Predicate<String> expectedMessage(String failureMessageStart, String failureMessageEnd) {
+		return message -> message.startsWith(failureMessageStart) && message.endsWith(failureMessageEnd);
 	}
 
 	private static EngineExecutionResults executeSuite(Class<?> suiteClass) {
-		return EngineTestKit.engine(ENGINE_ID).selectors(selectClass(suiteClass)).execute();
+		return engineWithSelectedSuite(suiteClass).execute();
+	}
+
+	private static EngineTestKit.Builder engineWithSelectedSuite(Class<?> suiteClass) {
+		return EngineTestKit.engine(ENGINE_ID).selectors(selectClass(suiteClass));
 	}
 
 }

platform-tests/src/test/java/org/junit/platform/suite/engine/SuiteTestDescriptorTests.java

@@ -13,6 +13,7 @@
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
 import static org.junit.jupiter.api.Assertions.assertAll;
+import static org.mockito.Mockito.mock;
 
 import java.util.Collections;
 import java.util.Optional;
@@ -28,6 +29,7 @@
 import org.junit.platform.engine.TestDescriptor;
 import org.junit.platform.engine.UniqueId;
 import org.junit.platform.engine.reporting.OutputDirectoryProvider;
+import org.junit.platform.engine.support.discovery.DiscoveryIssueReporter;
 import org.junit.platform.launcher.core.OutputDirectoryProviders;
 import org.junit.platform.suite.api.Suite;
 import org.junit.platform.suite.engine.testcases.SingleTestTestCase;
@@ -48,8 +50,9 @@ class SuiteTestDescriptorTests {
 
 	final ConfigurationParameters configurationParameters = new EmptyConfigurationParameters();
 	final OutputDirectoryProvider outputDirectoryProvider = OutputDirectoryProviders.dummyOutputDirectoryProvider();
+	final DiscoveryIssueReporter discoveryIssueReporter = DiscoveryIssueReporter.create(mock(), engineId);
 	final SuiteTestDescriptor suite = new SuiteTestDescriptor(suiteId, TestSuite.class, configurationParameters,
-		outputDirectoryProvider);
+		outputDirectoryProvider, discoveryIssueReporter);
 
 	@Test
 	void suiteIsEmptyBeforeDiscovery() {
@@ -68,7 +71,7 @@ void suiteDiscoversTestsFromClass() {
 	}
 
 	@Test
-	void suitDiscoversTestsFromUniqueId() {
+	void suiteDiscoversTestsFromUniqueId() {
 		suite.addDiscoveryRequestFrom(methodId);
 		suite.discover();
 

platform-tests/src/test/java/org/junit/platform/suite/engine/testsuites/LifecycleMethodsSuites.java

@@ -29,6 +29,7 @@
  *
  * @since 1.11
  */
+@SuppressWarnings("NewClassNamingConvention")
 public class LifecycleMethodsSuites {
 
 	@Retention(RetentionPolicy.RUNTIME)