diff --git a/src/Adapter/MSTest.TestAdapter/Execution/ClassCleanupManager.cs b/src/Adapter/MSTest.TestAdapter/Execution/ClassCleanupManager.cs index 8f6d5d75f2..6cd4c78279 100644 --- a/src/Adapter/MSTest.TestAdapter/Execution/ClassCleanupManager.cs +++ b/src/Adapter/MSTest.TestAdapter/Execution/ClassCleanupManager.cs @@ -62,4 +62,19 @@ public void MarkTestComplete(TestMethodInfo testMethodInfo, TestMethod testMetho ShouldRunEndOfAssemblyCleanup = _remainingTestsByClass.IsEmpty; } } + + internal static void ForceCleanup(TypeCache typeCache) + { + IEnumerable classInfoCache = typeCache.ClassInfoListWithExecutableCleanupMethods; + foreach (TestClassInfo classInfo in classInfoCache) + { + classInfo.ExecuteClassCleanup(); + } + + IEnumerable assemblyInfoCache = typeCache.AssemblyInfoListWithExecutableCleanupMethods; + foreach (TestAssemblyInfo assemblyInfo in assemblyInfoCache) + { + assemblyInfo.ExecuteAssemblyCleanup(); + } + } } diff --git a/src/Adapter/MSTest.TestAdapter/Execution/TestClassInfo.cs b/src/Adapter/MSTest.TestAdapter/Execution/TestClassInfo.cs index 07423c3608..fe29c5998e 100644 --- a/src/Adapter/MSTest.TestAdapter/Execution/TestClassInfo.cs +++ b/src/Adapter/MSTest.TestAdapter/Execution/TestClassInfo.cs @@ -252,6 +252,7 @@ public void RunClassInitialize(TestContext testContext) // If no class initialize and no base class initialize, return if (ClassInitializeMethod is null && BaseClassInitMethods.Count == 0) { + IsClassInitializeExecuted = true; return; } @@ -558,8 +559,10 @@ internal void ExecuteClassCleanup() lock (_testClassExecuteSyncObject) { if (IsClassCleanupExecuted - // If there is a ClassInitialize method and it has not been executed, then we should not execute ClassCleanup - || (!IsClassInitializeExecuted && ClassInitializeMethod is not null)) + // If ClassInitialize method has not been executed, then we should not execute ClassCleanup + // Note that if there is no ClassInitialze method at all, we will still set + // IsClassInitializeExecuted to true in RunClassInitialize + || !IsClassInitializeExecuted) { return; } diff --git a/src/Adapter/MSTest.TestAdapter/Execution/TestExecutionManager.cs b/src/Adapter/MSTest.TestAdapter/Execution/TestExecutionManager.cs index dc8c9171e6..b8af1f04b5 100644 --- a/src/Adapter/MSTest.TestAdapter/Execution/TestExecutionManager.cs +++ b/src/Adapter/MSTest.TestAdapter/Execution/TestExecutionManager.cs @@ -399,6 +399,11 @@ private void ExecuteTestsInSource(IEnumerable tests, IRunContext? runC ExecuteTestsWithTestRunner(testsToRun, frameworkHandle, source, sourceLevelParameters, testRunner); } + if (MSTestGracefulStopTestExecutionCapability.Instance.IsStopRequested) + { + testRunner.ForceCleanup(); + } + PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo("Executed tests belonging to source {0}", source); } @@ -419,6 +424,10 @@ private void ExecuteTestsWithTestRunner( foreach (TestCase currentTest in orderedTests) { _testRunCancellationToken?.ThrowIfCancellationRequested(); + if (MSTestGracefulStopTestExecutionCapability.Instance.IsStopRequested) + { + break; + } // If it is a fixture test, add it to the list of fixture tests and do not execute it. // It is executed by test itself. diff --git a/src/Adapter/MSTest.TestAdapter/Execution/UnitTestRunner.cs b/src/Adapter/MSTest.TestAdapter/Execution/UnitTestRunner.cs index 174155f9aa..6d6e854add 100644 --- a/src/Adapter/MSTest.TestAdapter/Execution/UnitTestRunner.cs +++ b/src/Adapter/MSTest.TestAdapter/Execution/UnitTestRunner.cs @@ -169,6 +169,7 @@ internal UnitTestResult[] RunSingleTest(TestMethod testMethod, IDictionary ClassCleanupManager.ForceCleanup(_typeCache); } diff --git a/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/MSTestGracefulStopTestExecutionCapability.cs b/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/MSTestGracefulStopTestExecutionCapability.cs new file mode 100644 index 0000000000..a8ef89ff47 --- /dev/null +++ b/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/MSTestGracefulStopTestExecutionCapability.cs @@ -0,0 +1,25 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Platform.Capabilities.TestFramework; + +namespace Microsoft.VisualStudio.TestTools.UnitTesting; + +#pragma warning disable TPEXP // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. +internal sealed class MSTestGracefulStopTestExecutionCapability : IGracefulStopTestExecutionCapability +#pragma warning restore TPEXP // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. +{ + private MSTestGracefulStopTestExecutionCapability() + { + } + + public static MSTestGracefulStopTestExecutionCapability Instance { get; } = new(); + + public bool IsStopRequested { get; private set; } + + public Task StopTestExecutionAsync(CancellationToken cancellationToken) + { + IsStopRequested = true; + return Task.CompletedTask; + } +} diff --git a/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/TestApplicationBuilderExtensions.cs b/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/TestApplicationBuilderExtensions.cs index 2a86adaf09..3cb51f0732 100644 --- a/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/TestApplicationBuilderExtensions.cs +++ b/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/TestApplicationBuilderExtensions.cs @@ -8,6 +8,7 @@ using Microsoft.Testing.Extensions.VSTestBridge.Helpers; using Microsoft.Testing.Platform.Builder; using Microsoft.Testing.Platform.Capabilities.TestFramework; +using Microsoft.Testing.Platform.Helpers; using Microsoft.Testing.Platform.Services; namespace Microsoft.VisualStudio.TestTools.UnitTesting; @@ -20,12 +21,16 @@ public static void AddMSTest(this ITestApplicationBuilder testApplicationBuilder testApplicationBuilder.AddRunSettingsService(extension); testApplicationBuilder.AddTestCaseFilterService(extension); testApplicationBuilder.AddTestRunParametersService(extension); +#pragma warning disable TPEXP // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + testApplicationBuilder.AddMaximumFailedTestsService(extension); +#pragma warning restore TPEXP // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. testApplicationBuilder.AddRunSettingsEnvironmentVariableProvider(extension); testApplicationBuilder.RegisterTestFramework( serviceProvider => new TestFrameworkCapabilities( new VSTestBridgeExtensionBaseCapabilities(), #pragma warning disable TPEXP // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. - new MSTestBannerCapability(serviceProvider.GetRequiredService())), + new MSTestBannerCapability(serviceProvider.GetRequiredService()), + MSTestGracefulStopTestExecutionCapability.Instance), #pragma warning restore TPEXP // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. (capabilities, serviceProvider) => new MSTestBridgedTestFramework(extension, getTestAssemblies, serviceProvider, capabilities)); } diff --git a/src/Platform/Microsoft.Testing.Platform/Capabilities/TestFramework/IGracefulStopTestExecutionCapability.cs b/src/Platform/Microsoft.Testing.Platform/Capabilities/TestFramework/IGracefulStopTestExecutionCapability.cs new file mode 100644 index 0000000000..ee8a92dee3 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Platform/Capabilities/TestFramework/IGracefulStopTestExecutionCapability.cs @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Testing.Platform.Capabilities.TestFramework; + +/// +/// A capability to support stopping test execution gracefully, without cancelling/aborting everything. +/// This is used to support '--maximum-failed-tests'. +/// +/// +/// Test frameworks can choose to run any needed cleanup when cancellation is requested. +/// +[Experimental("TPEXP", UrlFormat = "https://aka.ms/testingplatform/diagnostics#{0}")] +public interface IGracefulStopTestExecutionCapability : ITestFrameworkCapability +{ + Task StopTestExecutionAsync(CancellationToken cancellationToken); +} diff --git a/src/Platform/Microsoft.Testing.Platform/CommandLine/MaxFailedTestsCommandLineOptionsProvider.cs b/src/Platform/Microsoft.Testing.Platform/CommandLine/MaxFailedTestsCommandLineOptionsProvider.cs new file mode 100644 index 0000000000..e81cc907b7 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Platform/CommandLine/MaxFailedTestsCommandLineOptionsProvider.cs @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Globalization; + +using Microsoft.Testing.Platform.Extensions; +using Microsoft.Testing.Platform.Extensions.CommandLine; +using Microsoft.Testing.Platform.Helpers; +using Microsoft.Testing.Platform.Resources; + +namespace Microsoft.Testing.Platform.CommandLine; + +internal sealed class MaxFailedTestsCommandLineOptionsProvider(IExtension extension) : ICommandLineOptionsProvider +{ + internal const string MaxFailedTestsOptionKey = "maximum-failed-tests"; + + private static readonly IReadOnlyCollection OptionsCache = + [ + new(MaxFailedTestsOptionKey, PlatformResources.PlatformCommandLineMaxFailedTestsOptionDescription, ArgumentArity.ExactlyOne, isHidden: false), + ]; + + public string Uid => extension.Uid; + + public string Version => extension.Version; + + public string DisplayName => extension.DisplayName; + + public string Description => extension.Description; + + public IReadOnlyCollection GetCommandLineOptions() + => OptionsCache; + + public Task IsEnabledAsync() => Task.FromResult(true); + + public Task ValidateCommandLineOptionsAsync(ICommandLineOptions commandLineOptions) + => ValidationResult.ValidTask; + + public Task ValidateOptionArgumentsAsync(CommandLineOption commandOption, string[] arguments) + { + if (commandOption.Name == MaxFailedTestsOptionKey) + { + string arg = arguments[0]; + // We consider --maximum-failed-tests 0 as invalid. + // The idea is that we stop the execution when we *reach* the max failed tests, not when *exceed*. + // So the value 1 means, stop execution on the first failure. + return int.TryParse(arg, out int maxFailedTestsResult) && maxFailedTestsResult > 0 + ? ValidationResult.ValidTask + : ValidationResult.InvalidTask(string.Format(CultureInfo.InvariantCulture, PlatformResources.MaxFailedTestsMustBePositive, arg)); + } + + throw ApplicationStateGuard.Unreachable(); + } +} diff --git a/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs b/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs new file mode 100644 index 0000000000..78e4999b3c --- /dev/null +++ b/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs @@ -0,0 +1,76 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Platform.Capabilities.TestFramework; +using Microsoft.Testing.Platform.CommandLine; +using Microsoft.Testing.Platform.Extensions.Messages; +using Microsoft.Testing.Platform.Extensions.TestHost; +using Microsoft.Testing.Platform.Helpers; +using Microsoft.Testing.Platform.Messages; +using Microsoft.Testing.Platform.Resources; +using Microsoft.Testing.Platform.Services; + +namespace Microsoft.Testing.Platform.Extensions; + +internal sealed class AbortForMaxFailedTestsExtension : IDataConsumer +{ + private readonly int? _maxFailedTests; + private readonly IGracefulStopTestExecutionCapability? _capability; + private readonly IStopPoliciesService _policiesService; + private readonly ITestApplicationCancellationTokenSource _testApplicationCancellationTokenSource; + private int _failCount; + + public AbortForMaxFailedTestsExtension( + ICommandLineOptions commandLineOptions, + IGracefulStopTestExecutionCapability? capability, + IStopPoliciesService policiesService, + ITestApplicationCancellationTokenSource testApplicationCancellationTokenSource) + { + if (commandLineOptions.TryGetOptionArgumentList(MaxFailedTestsCommandLineOptionsProvider.MaxFailedTestsOptionKey, out string[]? args) && + int.TryParse(args[0], out int maxFailedTests) && + maxFailedTests > 0) + { + _maxFailedTests = maxFailedTests; + } + + _capability = capability; + _policiesService = policiesService; + _testApplicationCancellationTokenSource = testApplicationCancellationTokenSource; + } + + public Type[] DataTypesConsumed { get; } = [typeof(TestNodeUpdateMessage)]; + + /// + public string Uid { get; } = nameof(AbortForMaxFailedTestsExtension); + + /// + public string Version { get; } = AppVersion.DefaultSemVer; + + /// + public string DisplayName { get; } = nameof(AbortForMaxFailedTestsExtension); + + /// + public string Description { get; } = PlatformResources.AbortForMaxFailedTestsDescription; + + /// + public Task IsEnabledAsync() => Task.FromResult(_maxFailedTests.HasValue && _capability is not null); + + public async Task ConsumeAsync(IDataProducer dataProducer, IData value, CancellationToken cancellationToken) + { + var node = (TestNodeUpdateMessage)value; + + // If we are called, the extension is enabled, which means both _maxFailedTests and capability are not null. + RoslynDebug.Assert(_maxFailedTests is not null); + RoslynDebug.Assert(_capability is not null); + + TestNodeStateProperty testNodeStateProperty = node.TestNode.Properties.Single(); + if (TestNodePropertiesCategories.WellKnownTestNodeTestRunOutcomeFailedProperties.Any(t => t == testNodeStateProperty.GetType()) && + ++_failCount >= _maxFailedTests.Value && + // If already triggered, don't do it again. + !_policiesService.IsMaxFailedTestsTriggered) + { + await _capability.StopTestExecutionAsync(_testApplicationCancellationTokenSource.CancellationToken); + await _policiesService.ExecuteMaxFailedTestsCallbacksAsync(_maxFailedTests.Value, _testApplicationCancellationTokenSource.CancellationToken); + } + } +} diff --git a/src/Platform/Microsoft.Testing.Platform/Helpers/ExitCodes.cs b/src/Platform/Microsoft.Testing.Platform/Helpers/ExitCodes.cs index 28514ae2c0..79e8fd0864 100644 --- a/src/Platform/Microsoft.Testing.Platform/Helpers/ExitCodes.cs +++ b/src/Platform/Microsoft.Testing.Platform/Helpers/ExitCodes.cs @@ -22,4 +22,5 @@ internal static class ExitCodes public const int TestAdapterTestSessionFailure = 10; public const int DependentProcessExited = 11; public const int IncompatibleProtocolVersion = 12; + public const int TestExecutionStoppedForMaxFailedTests = 13; } diff --git a/src/Platform/Microsoft.Testing.Platform/Helpers/TestApplicationBuilderExtensions.cs b/src/Platform/Microsoft.Testing.Platform/Helpers/TestApplicationBuilderExtensions.cs index 512dc9e6a8..bae0af6c00 100644 --- a/src/Platform/Microsoft.Testing.Platform/Helpers/TestApplicationBuilderExtensions.cs +++ b/src/Platform/Microsoft.Testing.Platform/Helpers/TestApplicationBuilderExtensions.cs @@ -15,4 +15,12 @@ public static class TestApplicationBuilderExtensions { public static void AddTreeNodeFilterService(this ITestApplicationBuilder testApplicationBuilder, IExtension extension) => testApplicationBuilder.CommandLine.AddProvider(() => new TreeNodeFilterCommandLineOptionsProvider(extension)); + + /// + /// Registers the command-line options provider for '--maximum-failed-tests'. + /// + /// The test application builder. + [Experimental("TPEXP", UrlFormat = "https://aka.ms/testingplatform/diagnostics#{0}")] + public static void AddMaximumFailedTestsService(this ITestApplicationBuilder builder, IExtension extension) + => builder.CommandLine.AddProvider(() => new MaxFailedTestsCommandLineOptionsProvider(extension)); } diff --git a/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs b/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs index 9b19eea139..b071f4fb9f 100644 --- a/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs +++ b/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs @@ -206,6 +206,10 @@ public async Task BuildAsync( // Set the concrete command line options to the proxy. commandLineOptionsProxy.SetCommandLineOptions(commandLineHandler); + // This is needed by output device. + var policiesService = new StopPoliciesService(testApplicationCancellationTokenSource); + serviceProvider.AddService(policiesService); + bool hasServerFlag = commandLineHandler.TryGetOptionArgumentList(PlatformCommandLineProvider.ServerOptionKey, out string[]? protocolName); bool isJsonRpcProtocol = protocolName is null || protocolName.Length == 0 || protocolName[0].Equals(PlatformCommandLineProvider.JsonRpcProtocolName, StringComparison.OrdinalIgnoreCase); @@ -313,9 +317,9 @@ public async Task BuildAsync( // Register the ITestApplicationResult TestApplicationResult testApplicationResult = new( proxyOutputDevice, - serviceProvider.GetTestApplicationCancellationTokenSource(), serviceProvider.GetCommandLineOptions(), - serviceProvider.GetEnvironment()); + serviceProvider.GetEnvironment(), + policiesService); serviceProvider.AddService(testApplicationResult); // ============= SETUP COMMON SERVICE USED IN ALL MODES END ===============// @@ -376,6 +380,8 @@ await LogTestHostCreatedAsync( TestHostOrchestratorConfiguration testHostOrchestratorConfiguration = await TestHostOrchestratorManager.BuildAsync(serviceProvider); if (testHostOrchestratorConfiguration.TestHostOrchestrators.Length > 0 && !commandLineHandler.IsOptionSet(PlatformCommandLineProvider.DiscoverTestsOptionKey)) { + policiesService.ProcessRole = TestProcessRole.TestHostOrchestrator; + await proxyOutputDevice.HandleProcessRoleAsync(TestProcessRole.TestHostOrchestrator); return new TestHostOrchestratorHost(testHostOrchestratorConfiguration, serviceProvider); } @@ -411,6 +417,8 @@ await LogTestHostCreatedAsync( if (testHostControllers.RequireProcessRestart) { testHostControllerInfo.IsCurrentProcessTestHostController = true; + policiesService.ProcessRole = TestProcessRole.TestHostController; + await proxyOutputDevice.HandleProcessRoleAsync(TestProcessRole.TestHostController); TestHostControllersTestHost testHostControllersTestHost = new(testHostControllers, testHostControllersServiceProvider, passiveNode, systemEnvironment, loggerFactory, systemClock); await LogTestHostCreatedAsync( @@ -424,6 +432,8 @@ await LogTestHostCreatedAsync( } // ======= TEST HOST MODE ======== // + policiesService.ProcessRole = TestProcessRole.TestHost; + await proxyOutputDevice.HandleProcessRoleAsync(TestProcessRole.TestHost); // Setup the test host working folder. // Out of the test host controller extension the current working directory is the test host working directory. @@ -724,6 +734,17 @@ private async Task BuildTestFrameworkAsync(TestFrameworkBuilderD dataConsumersBuilder.Add(pushOnlyProtocolDataConsumer); } + var abortForMaxFailedTestsExtension = new AbortForMaxFailedTestsExtension( + serviceProvider.GetCommandLineOptions(), + serviceProvider.GetTestFrameworkCapabilities().GetCapability(), + serviceProvider.GetRequiredService(), + serviceProvider.GetTestApplicationCancellationTokenSource()); + + if (await abortForMaxFailedTestsExtension.IsEnabledAsync()) + { + dataConsumersBuilder.Add(abortForMaxFailedTestsExtension); + } + IDataConsumer[] dataConsumerServices = dataConsumersBuilder.ToArray(); // Build the message bus diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/IPlatformOutputDevice.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/IPlatformOutputDevice.cs index 25d1bf12ae..22705e1968 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/IPlatformOutputDevice.cs +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/IPlatformOutputDevice.cs @@ -15,4 +15,6 @@ internal interface IPlatformOutputDevice : IExtension Task DisplayAfterSessionEndRunAsync(); Task DisplayAsync(IOutputDeviceDataProducer producer, IOutputDeviceData data); + + Task HandleProcessRoleAsync(TestProcessRole processRole); } diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/OutputDeviceManager.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/OutputDeviceManager.cs index e8e7ac88b0..78d4dafb0f 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/OutputDeviceManager.cs +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/OutputDeviceManager.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +using Microsoft.Testing.Platform.Logging; using Microsoft.Testing.Platform.ServerMode; using Microsoft.Testing.Platform.Services; @@ -33,7 +34,9 @@ internal async Task BuildAsync(ServiceProvider serviceProvide return new ProxyOutputDevice( nonServerOutputDevice, useServerModeOutputDevice - ? new ServerModePerCallOutputDevice(serviceProvider) + ? new ServerModePerCallOutputDevice( + serviceProvider.GetService(), + serviceProvider.GetRequiredService()) : null); } @@ -51,5 +54,6 @@ public static TerminalOutputDevice GetDefaultTerminalOutputDevice(ServiceProvide serviceProvider.GetCommandLineOptions(), serviceProvider.GetFileLoggerInformation(), serviceProvider.GetLoggerFactory(), - serviceProvider.GetClock()); + serviceProvider.GetClock(), + serviceProvider.GetRequiredService()); } diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/ProxyOutputDevice.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/ProxyOutputDevice.cs index 6896debd84..b2d0b2ed4c 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/ProxyOutputDevice.cs +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/ProxyOutputDevice.cs @@ -62,4 +62,13 @@ internal async Task InitializeAsync(ServerTestHost serverTestHost) await _serverModeOutputDevice.InitializeAsync(serverTestHost); } } + + internal async Task HandleProcessRoleAsync(TestProcessRole processRole) + { + await OriginalOutputDevice.HandleProcessRoleAsync(processRole); + if (_serverModeOutputDevice is not null) + { + await _serverModeOutputDevice.HandleProcessRoleAsync(processRole); + } + } } diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.cs index f9d521e8a6..f049246cab 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.cs +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.cs @@ -49,6 +49,7 @@ internal sealed partial class TerminalOutputDevice : IHotReloadPlatformOutputDev private readonly IFileLoggerInformation? _fileLoggerInformation; private readonly ILoggerFactory _loggerFactory; private readonly IClock _clock; + private readonly IStopPoliciesService _policiesService; private readonly string? _longArchitecture; private readonly string? _shortArchitecture; @@ -72,7 +73,8 @@ internal sealed partial class TerminalOutputDevice : IHotReloadPlatformOutputDev public TerminalOutputDevice(ITestApplicationCancellationTokenSource testApplicationCancellationTokenSource, IConsole console, ITestApplicationModuleInfo testApplicationModuleInfo, ITestHostControllerInfo testHostControllerInfo, IAsyncMonitor asyncMonitor, IRuntimeFeature runtimeFeature, IEnvironment environment, IProcessHandler process, IPlatformInformation platformInformation, - ICommandLineOptions commandLineOptions, IFileLoggerInformation? fileLoggerInformation, ILoggerFactory loggerFactory, IClock clock) + ICommandLineOptions commandLineOptions, IFileLoggerInformation? fileLoggerInformation, ILoggerFactory loggerFactory, IClock clock, + IStopPoliciesService policiesService) { _testApplicationCancellationTokenSource = testApplicationCancellationTokenSource; _console = console; @@ -87,6 +89,7 @@ public TerminalOutputDevice(ITestApplicationCancellationTokenSource testApplicat _fileLoggerInformation = fileLoggerInformation; _loggerFactory = loggerFactory; _clock = clock; + _policiesService = policiesService; if (_runtimeFeature.IsDynamicCodeSupported) { @@ -110,8 +113,15 @@ public TerminalOutputDevice(ITestApplicationCancellationTokenSource testApplicat } } - public Task InitializeAsync() + public async Task InitializeAsync() { + await _policiesService.RegisterOnAbortCallbackAsync( + () => + { + _terminalTestReporter?.StartCancelling(); + return Task.CompletedTask; + }); + if (_fileLoggerInformation is not null) { _logger = _loggerFactory.CreateLogger(GetType().ToString()); @@ -163,10 +173,6 @@ public Task InitializeAsync() ShowActiveTests = true, ShowProgress = shouldShowProgress, }); - - _testApplicationCancellationTokenSource.CancellationToken.Register(() => _terminalTestReporter.StartCancelling()); - - return Task.CompletedTask; } private string GetShortArchitecture(string runtimeIdentifier) @@ -593,4 +599,14 @@ public async Task ConsumeAsync(IDataProducer dataProducer, IData value, Cancella public void Dispose() => _terminalTestReporter?.Dispose(); + + public async Task HandleProcessRoleAsync(TestProcessRole processRole) + { + if (processRole == TestProcessRole.TestHost) + { + await _policiesService.RegisterOnMaxFailedTestsCallbackAsync( + async (maxFailedTests, _) => await DisplayAsync( + this, new TextOutputDeviceData(string.Format(CultureInfo.InvariantCulture, PlatformResources.ReachedMaxFailedTestsMessage, maxFailedTests)))); + } + } } diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/TestProcessRole.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/TestProcessRole.cs new file mode 100644 index 0000000000..0955e45246 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/TestProcessRole.cs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.Testing.Platform; + +internal enum TestProcessRole +{ + /// + /// Indicates that the currently running process is the test host. + /// + TestHost, + + /// + /// Indicates that the currently running process is the test host controller. + /// + TestHostController, + + /// + /// Indicates that the currently running process is the test host orchestrator. + /// + TestHostOrchestrator, +} diff --git a/src/Platform/Microsoft.Testing.Platform/PublicAPI/PublicAPI.Unshipped.txt b/src/Platform/Microsoft.Testing.Platform/PublicAPI/PublicAPI.Unshipped.txt index 3976e688ef..1d1c2b5ca1 100644 --- a/src/Platform/Microsoft.Testing.Platform/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/Platform/Microsoft.Testing.Platform/PublicAPI/PublicAPI.Unshipped.txt @@ -6,6 +6,8 @@ Microsoft.Testing.Platform.OutputDevice.ErrorMessageOutputDeviceData.Message.get Microsoft.Testing.Platform.OutputDevice.WarningMessageOutputDeviceData Microsoft.Testing.Platform.OutputDevice.WarningMessageOutputDeviceData.Message.get -> string! Microsoft.Testing.Platform.OutputDevice.WarningMessageOutputDeviceData.WarningMessageOutputDeviceData(string! message) -> void +[TPEXP]Microsoft.Testing.Platform.Capabilities.TestFramework.IGracefulStopTestExecutionCapability +[TPEXP]Microsoft.Testing.Platform.Capabilities.TestFramework.IGracefulStopTestExecutionCapability.StopTestExecutionAsync(System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task! [TPEXP]Microsoft.Testing.Platform.Extensions.Messages.StandardOutputProperty [TPEXP]Microsoft.Testing.Platform.Extensions.Messages.StandardOutputProperty.StandardOutput.get -> string! [TPEXP]Microsoft.Testing.Platform.Extensions.Messages.StandardOutputProperty.StandardOutput.init -> void diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/PlatformResources.resx b/src/Platform/Microsoft.Testing.Platform/Resources/PlatformResources.resx index 8a899ed676..4047fa6895 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/PlatformResources.resx +++ b/src/Platform/Microsoft.Testing.Platform/Resources/PlatformResources.resx @@ -700,4 +700,17 @@ Takes one argument as string in the format <value>[h|m|s] where 'value' is Exception during the cancellation of request id '{0}' {0} is the request id - \ No newline at end of file + + Specifies a maximum number of test failures that, when exceeded, will abort the test run. + + + The option '--maximum-failed-tests' must be a positive integer. The value '{0}' is not valid. + + + Extension used to support '--maximum-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + + + Test session is aborting due to reaching failures ('{0}') specified by the '--maximum-failed-tests' option. + {0} is the number of max failed tests. + + diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.cs.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.cs.xlf index 47adbca59a..95c3be44f2 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.cs.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.cs.xlf @@ -2,6 +2,11 @@ + + Extension used to support '--maximum-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + Extension used to support '--maximum-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + + Aborted Přerušeno @@ -371,6 +376,11 @@ Rozhraní ILoggerFactory ještě nebylo sestaveno. + + The option '--maximum-failed-tests' must be a positive integer. The value '{0}' is not valid. + The option '--maximum-failed-tests' must be a positive integer. The value '{0}' is not valid. + + The message bus has not been built yet or is no more usable at this stage. Sběrnice zpráv ještě nebyla sestavena nebo už není v této fázi použitelná. @@ -535,6 +545,11 @@ Dostupné hodnoty jsou Trace, Debug, Information, Warning, Error a Critical.Umožňuje zobrazit informace o testovací aplikaci .NET. + + Specifies a maximum number of test failures that, when exceeded, will abort the test run. + Specifies a maximum number of test failures that, when exceeded, will abort the test run. + + '--list-tests' and '--minimum-expected-tests' are incompatible options --list-tests a --minimum-expected-tests jsou nekompatibilní možnosti. @@ -623,6 +638,11 @@ Může mít jenom jeden argument jako řetězec ve formátu <value>[h|m|s] Proces měl být ukončen před tím, než jsme mohli určit tuto hodnotu. + + Test session is aborting due to reaching failures ('{0}') specified by the '--maximum-failed-tests' option. + Test session is aborting due to reaching failures ('{0}') specified by the '--maximum-failed-tests' option. + {0} is the number of max failed tests. + Retry failed after {0} times Opakování se po {0} pokusech nezdařilo. diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.de.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.de.xlf index 5102d26462..5ad7e52816 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.de.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.de.xlf @@ -2,6 +2,11 @@ + + Extension used to support '--maximum-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + Extension used to support '--maximum-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + + Aborted Abgebrochen @@ -371,6 +376,11 @@ Die ILoggerFactory wurde noch nicht erstellt. + + The option '--maximum-failed-tests' must be a positive integer. The value '{0}' is not valid. + The option '--maximum-failed-tests' must be a positive integer. The value '{0}' is not valid. + + The message bus has not been built yet or is no more usable at this stage. Der Nachrichtenbus wurde noch nicht erstellt oder kann zu diesem Zeitpunkt nicht mehr verwendet werden. @@ -535,6 +545,11 @@ Die verfügbaren Werte sind "Trace", "Debug", "Information", "Warning", "Error" Zeigen Sie .NET-Testanwendungsinformationen an. + + Specifies a maximum number of test failures that, when exceeded, will abort the test run. + Specifies a maximum number of test failures that, when exceeded, will abort the test run. + + '--list-tests' and '--minimum-expected-tests' are incompatible options "--list-tests" und "--minimum-expected-tests" sind inkompatible Optionen. @@ -623,6 +638,11 @@ Nimmt ein Argument als Zeichenfolge im Format <value>[h|m|s], wobei "value Der Prozess hätte beendet werden müssen, bevor dieser Wert ermittelt werden kann + + Test session is aborting due to reaching failures ('{0}') specified by the '--maximum-failed-tests' option. + Test session is aborting due to reaching failures ('{0}') specified by the '--maximum-failed-tests' option. + {0} is the number of max failed tests. + Retry failed after {0} times Wiederholungsfehler nach {0} Mal diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.es.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.es.xlf index c23458d113..58c731750f 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.es.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.es.xlf @@ -2,6 +2,11 @@ + + Extension used to support '--maximum-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + Extension used to support '--maximum-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + + Aborted Anulado @@ -371,6 +376,11 @@ ILoggerFactory aún no se ha compilado. + + The option '--maximum-failed-tests' must be a positive integer. The value '{0}' is not valid. + The option '--maximum-failed-tests' must be a positive integer. The value '{0}' is not valid. + + The message bus has not been built yet or is no more usable at this stage. El bus de mensajes aún no se ha compilado o ya no se puede usar en esta fase. @@ -535,6 +545,11 @@ Los valores disponibles son 'Seguimiento', 'Depurar', 'Información', 'Advertenc Muestre la información de la aplicación de prueba de .NET. + + Specifies a maximum number of test failures that, when exceeded, will abort the test run. + Specifies a maximum number of test failures that, when exceeded, will abort the test run. + + '--list-tests' and '--minimum-expected-tests' are incompatible options “--list-tests” y “--minimum-expected-tests” son opciones incompatibles @@ -623,6 +638,11 @@ Toma un argumento como cadena con el formato <value>[h|m|s] donde 'value' El proceso debería haberse terminado para poder determinar este valor + + Test session is aborting due to reaching failures ('{0}') specified by the '--maximum-failed-tests' option. + Test session is aborting due to reaching failures ('{0}') specified by the '--maximum-failed-tests' option. + {0} is the number of max failed tests. + Retry failed after {0} times Error al reintentar después de {0} veces diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.fr.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.fr.xlf index f4bb8dd0b7..6529f58252 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.fr.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.fr.xlf @@ -2,6 +2,11 @@ + + Extension used to support '--maximum-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + Extension used to support '--maximum-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + + Aborted Abandonné @@ -371,6 +376,11 @@ ILoggerFactory n’a pas encore été généré. + + The option '--maximum-failed-tests' must be a positive integer. The value '{0}' is not valid. + The option '--maximum-failed-tests' must be a positive integer. The value '{0}' is not valid. + + The message bus has not been built yet or is no more usable at this stage. Le bus de messages n’a pas encore été généré ou n’est plus utilisable à ce stade. @@ -535,6 +545,11 @@ Les valeurs disponibles sont « Trace », « Debug », « Information », Afficher les informations de l’application de test .NET. + + Specifies a maximum number of test failures that, when exceeded, will abort the test run. + Specifies a maximum number of test failures that, when exceeded, will abort the test run. + + '--list-tests' and '--minimum-expected-tests' are incompatible options « --list-tests » et « --minimum-expected-tests » sont des options incompatibles @@ -623,6 +638,11 @@ Prend un argument sous forme de chaîne au format <value>[h|m|s] où « v Le processus aurait dû s’arrêter avant que nous puissions déterminer cette valeur + + Test session is aborting due to reaching failures ('{0}') specified by the '--maximum-failed-tests' option. + Test session is aborting due to reaching failures ('{0}') specified by the '--maximum-failed-tests' option. + {0} is the number of max failed tests. + Retry failed after {0} times Échec de la nouvelle tentative après {0} fois diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.it.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.it.xlf index 0de5fa57fa..15d86d4003 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.it.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.it.xlf @@ -2,6 +2,11 @@ + + Extension used to support '--maximum-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + Extension used to support '--maximum-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + + Aborted Operazione interrotta @@ -371,6 +376,11 @@ ILoggerFactory non è stato ancora compilato. + + The option '--maximum-failed-tests' must be a positive integer. The value '{0}' is not valid. + The option '--maximum-failed-tests' must be a positive integer. The value '{0}' is not valid. + + The message bus has not been built yet or is no more usable at this stage. Il bus di messaggi non è stato ancora compilato o non è più utilizzabile in questa fase. @@ -535,6 +545,11 @@ I valori disponibili sono 'Trace', 'Debug', 'Information', 'Warning', 'Error' e Visualizza le informazioni sull'applicazione di test .NET. + + Specifies a maximum number of test failures that, when exceeded, will abort the test run. + Specifies a maximum number of test failures that, when exceeded, will abort the test run. + + '--list-tests' and '--minimum-expected-tests' are incompatible options '--list-tests' e '--minimum-expected-tests' sono opzioni incompatibili @@ -623,6 +638,11 @@ Acquisisce un argomento come stringa nel formato <value>[h|m|s] dove 'valu Il processo dovrebbe essere terminato prima di poter determinare questo valore + + Test session is aborting due to reaching failures ('{0}') specified by the '--maximum-failed-tests' option. + Test session is aborting due to reaching failures ('{0}') specified by the '--maximum-failed-tests' option. + {0} is the number of max failed tests. + Retry failed after {0} times Tentativi non riusciti dopo {0} tentativi diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ja.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ja.xlf index 2e622e075e..18e1d555d3 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ja.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ja.xlf @@ -2,6 +2,11 @@ + + Extension used to support '--maximum-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + Extension used to support '--maximum-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + + Aborted 中止されました @@ -371,6 +376,11 @@ ILoggerFactory はまだ構築されていません。 + + The option '--maximum-failed-tests' must be a positive integer. The value '{0}' is not valid. + The option '--maximum-failed-tests' must be a positive integer. The value '{0}' is not valid. + + The message bus has not been built yet or is no more usable at this stage. メッセージ バスはまだ構築されていないか、この段階ではこれ以上使用できません。 @@ -536,6 +546,11 @@ The available values are 'Trace', 'Debug', 'Information', 'Warning', 'Error', an .NET テスト アプリケーション情報を表示します。 + + Specifies a maximum number of test failures that, when exceeded, will abort the test run. + Specifies a maximum number of test failures that, when exceeded, will abort the test run. + + '--list-tests' and '--minimum-expected-tests' are incompatible options '--list-tests' と '--minimum-expected-tests' は互換性のないオプションです @@ -624,6 +639,11 @@ Takes one argument as string in the format <value>[h|m|s] where 'value' is この値を決定する前にプロセスを終了する必要があります + + Test session is aborting due to reaching failures ('{0}') specified by the '--maximum-failed-tests' option. + Test session is aborting due to reaching failures ('{0}') specified by the '--maximum-failed-tests' option. + {0} is the number of max failed tests. + Retry failed after {0} times 再試行が {0} 回後に失敗しました diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ko.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ko.xlf index 22c32f88ab..c8b3cbeb40 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ko.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ko.xlf @@ -2,6 +2,11 @@ + + Extension used to support '--maximum-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + Extension used to support '--maximum-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + + Aborted 중단됨 @@ -371,6 +376,11 @@ ILoggerFactory가 아직 빌드되지 않았습니다. + + The option '--maximum-failed-tests' must be a positive integer. The value '{0}' is not valid. + The option '--maximum-failed-tests' must be a positive integer. The value '{0}' is not valid. + + The message bus has not been built yet or is no more usable at this stage. 메시지 버스가 아직 빌드되지 않았거나 이 단계에서 더 이상 사용할 수 없습니다. @@ -535,6 +545,11 @@ The available values are 'Trace', 'Debug', 'Information', 'Warning', 'Error', an .NET 테스트 애플리케이션 정보를 표시합니다. + + Specifies a maximum number of test failures that, when exceeded, will abort the test run. + Specifies a maximum number of test failures that, when exceeded, will abort the test run. + + '--list-tests' and '--minimum-expected-tests' are incompatible options '--list-tests' 및 '--minimum-expected-tests'는 호환되지 않는 옵션입니다. @@ -623,6 +638,11 @@ Takes one argument as string in the format <value>[h|m|s] where 'value' is 이 값을 결정하려면 프로세스가 종료되어야 합니다. + + Test session is aborting due to reaching failures ('{0}') specified by the '--maximum-failed-tests' option. + Test session is aborting due to reaching failures ('{0}') specified by the '--maximum-failed-tests' option. + {0} is the number of max failed tests. + Retry failed after {0} times {0}회 후 다시 시도 실패 diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pl.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pl.xlf index c1cd3555bb..010f80b6d0 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pl.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pl.xlf @@ -2,6 +2,11 @@ + + Extension used to support '--maximum-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + Extension used to support '--maximum-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + + Aborted Przerwano @@ -371,6 +376,11 @@ Obiekt ILoggerFactory nie został jeszcze skompilowany. + + The option '--maximum-failed-tests' must be a positive integer. The value '{0}' is not valid. + The option '--maximum-failed-tests' must be a positive integer. The value '{0}' is not valid. + + The message bus has not been built yet or is no more usable at this stage. Magistrala komunikatów nie została jeszcze zbudowana lub nie można jej już na tym etapie użyteczna. @@ -535,6 +545,11 @@ Dostępne wartości to „Trace”, „Debug”, „Information”, „Warning Wyświetl informacje o aplikacji testowej platformy .NET. + + Specifies a maximum number of test failures that, when exceeded, will abort the test run. + Specifies a maximum number of test failures that, when exceeded, will abort the test run. + + '--list-tests' and '--minimum-expected-tests' are incompatible options Opcje „--list-tests” i „--minimum-expected-tests” są niezgodne @@ -623,6 +638,11 @@ Pobiera jeden argument jako ciąg w formacie <value>[h|m|s], gdzie element Proces powinien zakończyć się przed ustaleniem tej wartości + + Test session is aborting due to reaching failures ('{0}') specified by the '--maximum-failed-tests' option. + Test session is aborting due to reaching failures ('{0}') specified by the '--maximum-failed-tests' option. + {0} is the number of max failed tests. + Retry failed after {0} times Ponowna próba nie powiodła się po {0} razach diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pt-BR.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pt-BR.xlf index 398b0f293c..6046c82389 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pt-BR.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pt-BR.xlf @@ -2,6 +2,11 @@ + + Extension used to support '--maximum-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + Extension used to support '--maximum-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + + Aborted Anulado @@ -371,6 +376,11 @@ O ILoggerFactory ainda não foi criado. + + The option '--maximum-failed-tests' must be a positive integer. The value '{0}' is not valid. + The option '--maximum-failed-tests' must be a positive integer. The value '{0}' is not valid. + + The message bus has not been built yet or is no more usable at this stage. O barramento de mensagens ainda não foi criado ou não pode mais ser usado nesse estágio. @@ -535,6 +545,11 @@ Os valores disponíveis são 'Rastreamento', 'Depuração', 'Informação', 'Avi Exibir informações do aplicativo de teste do .NET. + + Specifies a maximum number of test failures that, when exceeded, will abort the test run. + Specifies a maximum number of test failures that, when exceeded, will abort the test run. + + '--list-tests' and '--minimum-expected-tests' are incompatible options '--list-tests' e '--minimum-expected-tests' são opções incompatíveis @@ -623,6 +638,11 @@ Recebe um argumento como cadeia de caracteres no formato <valor>[h|m|s] em O processo deve ter sido encerrado antes que possamos determinar esse valor + + Test session is aborting due to reaching failures ('{0}') specified by the '--maximum-failed-tests' option. + Test session is aborting due to reaching failures ('{0}') specified by the '--maximum-failed-tests' option. + {0} is the number of max failed tests. + Retry failed after {0} times Falha na repetição após o {0} tempo diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ru.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ru.xlf index a48eef9579..59ba762b05 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ru.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ru.xlf @@ -2,6 +2,11 @@ + + Extension used to support '--maximum-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + Extension used to support '--maximum-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + + Aborted Прервано @@ -371,6 +376,11 @@ Параметр ILoggerFactory еще не создан. + + The option '--maximum-failed-tests' must be a positive integer. The value '{0}' is not valid. + The option '--maximum-failed-tests' must be a positive integer. The value '{0}' is not valid. + + The message bus has not been built yet or is no more usable at this stage. Шина сообщений еще не построена или на данном этапе непригодна для использования. @@ -535,6 +545,11 @@ The available values are 'Trace', 'Debug', 'Information', 'Warning', 'Error', an Отображение сведений о тестовом приложении .NET. + + Specifies a maximum number of test failures that, when exceeded, will abort the test run. + Specifies a maximum number of test failures that, when exceeded, will abort the test run. + + '--list-tests' and '--minimum-expected-tests' are incompatible options Параметры "--list-tests" и "--minimum-expected-tests" несовместимы @@ -623,6 +638,11 @@ Takes one argument as string in the format <value>[h|m|s] where 'value' is Процесс должен быть завершен, прежде чем мы сможем определить это значение + + Test session is aborting due to reaching failures ('{0}') specified by the '--maximum-failed-tests' option. + Test session is aborting due to reaching failures ('{0}') specified by the '--maximum-failed-tests' option. + {0} is the number of max failed tests. + Retry failed after {0} times Повторные попытки ({0}) завершились неудачно diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.tr.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.tr.xlf index 4ea66f782c..bbaf34f0ff 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.tr.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.tr.xlf @@ -2,6 +2,11 @@ + + Extension used to support '--maximum-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + Extension used to support '--maximum-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + + Aborted Durduruldu @@ -371,6 +376,11 @@ ILoggerFactory henüz derlenmedi. + + The option '--maximum-failed-tests' must be a positive integer. The value '{0}' is not valid. + The option '--maximum-failed-tests' must be a positive integer. The value '{0}' is not valid. + + The message bus has not been built yet or is no more usable at this stage. İleti veri yolu henüz derlenmedi veya bu aşamada artık kullanılamıyor. @@ -535,6 +545,11 @@ Kullanılabilir değerler: 'Trace', 'Debug', 'Information', 'Warning', 'Error' v .NET test uygulaması bilgilerini görüntüler. + + Specifies a maximum number of test failures that, when exceeded, will abort the test run. + Specifies a maximum number of test failures that, when exceeded, will abort the test run. + + '--list-tests' and '--minimum-expected-tests' are incompatible options '--list-tests' ve '--minimum-expected-tests' uyumsuz seçenekler @@ -623,6 +638,11 @@ Bir bağımsız değişkeni, 'value' değerinin kayan olduğu <value>[h|m| Bu değeri belirleyebilmemiz için süreçten çıkılmış olması gerekir + + Test session is aborting due to reaching failures ('{0}') specified by the '--maximum-failed-tests' option. + Test session is aborting due to reaching failures ('{0}') specified by the '--maximum-failed-tests' option. + {0} is the number of max failed tests. + Retry failed after {0} times Yeniden deneme {0} deneme sonrasında başarısız oldu diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hans.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hans.xlf index 4b98c6928f..1e6719edca 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hans.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hans.xlf @@ -2,6 +2,11 @@ + + Extension used to support '--maximum-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + Extension used to support '--maximum-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + + Aborted 已中止 @@ -371,6 +376,11 @@ 尚未生成 ILoggerFactory。 + + The option '--maximum-failed-tests' must be a positive integer. The value '{0}' is not valid. + The option '--maximum-failed-tests' must be a positive integer. The value '{0}' is not valid. + + The message bus has not been built yet or is no more usable at this stage. 消息总线尚未生成或在此阶段不再可用。 @@ -535,6 +545,11 @@ The available values are 'Trace', 'Debug', 'Information', 'Warning', 'Error', an 显示 .NET 测试应用程序信息。 + + Specifies a maximum number of test failures that, when exceeded, will abort the test run. + Specifies a maximum number of test failures that, when exceeded, will abort the test run. + + '--list-tests' and '--minimum-expected-tests' are incompatible options “--list-tests”和“--minimum-expected-tests”是不兼容的选项 @@ -623,6 +638,11 @@ Takes one argument as string in the format <value>[h|m|s] where 'value' is 在我们确定此值之前，流程应该已退出 + + Test session is aborting due to reaching failures ('{0}') specified by the '--maximum-failed-tests' option. + Test session is aborting due to reaching failures ('{0}') specified by the '--maximum-failed-tests' option. + {0} is the number of max failed tests. + Retry failed after {0} times {0} 次之后重试失败 diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hant.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hant.xlf index ec34238c60..5d8b20aad0 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hant.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hant.xlf @@ -2,6 +2,11 @@ + + Extension used to support '--maximum-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + Extension used to support '--maximum-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + + Aborted 已中止 @@ -371,6 +376,11 @@ 尚未建置 ILoggerFactory。 + + The option '--maximum-failed-tests' must be a positive integer. The value '{0}' is not valid. + The option '--maximum-failed-tests' must be a positive integer. The value '{0}' is not valid. + + The message bus has not been built yet or is no more usable at this stage. 訊息匯流排尚未建置或在此階段無法再使用。 @@ -535,6 +545,11 @@ The available values are 'Trace', 'Debug', 'Information', 'Warning', 'Error', an 顯示 .NET 測試應用程式資訊。 + + Specifies a maximum number of test failures that, when exceeded, will abort the test run. + Specifies a maximum number of test failures that, when exceeded, will abort the test run. + + '--list-tests' and '--minimum-expected-tests' are incompatible options '--list-tests' 和 '--minimum-expected-tests' 是不相容的選項 @@ -623,6 +638,11 @@ Takes one argument as string in the format <value>[h|m|s] where 'value' is 在我們確定此值之前，流程應已結束 + + Test session is aborting due to reaching failures ('{0}') specified by the '--maximum-failed-tests' option. + Test session is aborting due to reaching failures ('{0}') specified by the '--maximum-failed-tests' option. + {0} is the number of max failed tests. + Retry failed after {0} times 在 {0} 次重試之後失敗 diff --git a/src/Platform/Microsoft.Testing.Platform/ServerMode/JsonRpc/ServerModePerCallOutputDevice.cs b/src/Platform/Microsoft.Testing.Platform/ServerMode/JsonRpc/ServerModePerCallOutputDevice.cs index 091f782ad3..8c831b8f7f 100644 --- a/src/Platform/Microsoft.Testing.Platform/ServerMode/JsonRpc/ServerModePerCallOutputDevice.cs +++ b/src/Platform/Microsoft.Testing.Platform/ServerMode/JsonRpc/ServerModePerCallOutputDevice.cs @@ -15,17 +15,21 @@ namespace Microsoft.Testing.Platform.ServerMode; -internal sealed class ServerModePerCallOutputDevice : IPlatformOutputDevice +internal sealed class ServerModePerCallOutputDevice : IPlatformOutputDevice, IOutputDeviceDataProducer { - private readonly IServiceProvider _serviceProvider; + private readonly FileLoggerProvider? _fileLoggerProvider; + private readonly IStopPoliciesService _policiesService; private readonly ConcurrentBag _messages = new(); private IServerTestHost? _serverTestHost; private static readonly string[] NewLineStrings = { "\r\n", "\n" }; - public ServerModePerCallOutputDevice(IServiceProvider serviceProvider) - => _serviceProvider = serviceProvider; + public ServerModePerCallOutputDevice(FileLoggerProvider? fileLoggerProvider, IStopPoliciesService policiesService) + { + _fileLoggerProvider = fileLoggerProvider; + _policiesService = policiesService; + } internal async Task InitializeAsync(IServerTestHost serverTestHost) { @@ -94,7 +98,7 @@ public async Task DisplayBannerAsync(string? bannerMessage) public async Task DisplayBeforeSessionStartAsync() { - if (_serviceProvider.GetService() is { FileLogger.FileName: { } logFileName }) + if (_fileLoggerProvider is { FileLogger.FileName: { } logFileName }) { await LogAsync(LogLevel.Trace, string.Format(CultureInfo.InvariantCulture, PlatformResources.StartingTestSessionWithLogFilePath, logFileName), padding: null); } @@ -149,4 +153,14 @@ private static string GetIndentedMessage(string message, int? padding) return builder.ToString(); } + + public async Task HandleProcessRoleAsync(TestProcessRole processRole) + { + if (processRole == TestProcessRole.TestHost) + { + await _policiesService.RegisterOnMaxFailedTestsCallbackAsync( + async (maxFailedTests, _) => await DisplayAsync( + this, new TextOutputDeviceData(string.Format(CultureInfo.InvariantCulture, PlatformResources.ReachedMaxFailedTestsMessage, maxFailedTests)))); + } + } } diff --git a/src/Platform/Microsoft.Testing.Platform/Services/IStopPoliciesService.cs b/src/Platform/Microsoft.Testing.Platform/Services/IStopPoliciesService.cs new file mode 100644 index 0000000000..d82def8818 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Platform/Services/IStopPoliciesService.cs @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.Testing.Platform.Services; + +internal interface IStopPoliciesService +{ + bool IsMaxFailedTestsTriggered { get; } + + bool IsAbortTriggered { get; } + + Task RegisterOnMaxFailedTestsCallbackAsync(Func callback); + + Task RegisterOnAbortCallbackAsync(Func callback); + + Task ExecuteMaxFailedTestsCallbacksAsync(int maxFailedTests, CancellationToken cancellationToken); + + Task ExecuteAbortCallbacksAsync(); +} diff --git a/src/Platform/Microsoft.Testing.Platform/Services/StopPoliciesService.cs b/src/Platform/Microsoft.Testing.Platform/Services/StopPoliciesService.cs new file mode 100644 index 0000000000..fdd0b5f032 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Platform/Services/StopPoliciesService.cs @@ -0,0 +1,95 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Collections.Concurrent; + +using Microsoft.Testing.Platform.Helpers; + +namespace Microsoft.Testing.Platform.Services; + +internal sealed class StopPoliciesService : IStopPoliciesService +{ + private readonly ITestApplicationCancellationTokenSource _testApplicationCancellationTokenSource; + + private BlockingCollection>? _maxFailedTestsCallbacks; + private BlockingCollection>? _abortCallbacks; + private int _lastMaxFailedTests; + + public StopPoliciesService(ITestApplicationCancellationTokenSource testApplicationCancellationTokenSource) + { + _testApplicationCancellationTokenSource = testApplicationCancellationTokenSource; + +#pragma warning disable VSTHRD101 // Avoid unsupported async delegates + // Note: If cancellation already requested, Register will still invoke the callback. + testApplicationCancellationTokenSource.CancellationToken.Register(async () => await ExecuteAbortCallbacksAsync()); +#pragma warning restore VSTHRD101 // Avoid unsupported async delegates + } + + internal TestProcessRole? ProcessRole { get; set; } + + public bool IsMaxFailedTestsTriggered { get; private set; } + + public bool IsAbortTriggered { get; private set; } + + private static void RegisterCallback(ref BlockingCollection? callbacks, T callback) + => (callbacks ??= new()).Add(callback); + + public async Task ExecuteMaxFailedTestsCallbacksAsync(int maxFailedTests, CancellationToken cancellationToken) + { + _lastMaxFailedTests = maxFailedTests; + IsMaxFailedTestsTriggered = true; + if (_maxFailedTestsCallbacks is null) + { + return; + } + + foreach (Func callback in _maxFailedTestsCallbacks) + { + // For now, we are fine if the callback crashed us. It shouldn't happen for our + // current usage anyway and the APIs around this are all internal for now. + await callback.Invoke(maxFailedTests, cancellationToken); + } + } + + public async Task ExecuteAbortCallbacksAsync() + { + IsAbortTriggered = true; + + if (_abortCallbacks is null) + { + return; + } + + foreach (Func callback in _abortCallbacks) + { + // For now, we are fine if the callback crashed us. It shouldn't happen for our + // current usage anyway and the APIs around this are all internal for now. + await callback.Invoke(); + } + } + + public async Task RegisterOnMaxFailedTestsCallbackAsync(Func callback) + { + if (ProcessRole != TestProcessRole.TestHost) + { + throw ApplicationStateGuard.Unreachable(); + } + + if (IsMaxFailedTestsTriggered) + { + await callback(_lastMaxFailedTests, _testApplicationCancellationTokenSource.CancellationToken); + } + + RegisterCallback(ref _maxFailedTestsCallbacks, callback); + } + + public async Task RegisterOnAbortCallbackAsync(Func callback) + { + if (IsAbortTriggered) + { + await callback(); + } + + RegisterCallback(ref _abortCallbacks, callback); + } +} diff --git a/src/Platform/Microsoft.Testing.Platform/Services/TestApplicationResult.cs b/src/Platform/Microsoft.Testing.Platform/Services/TestApplicationResult.cs index 3304d317de..0b19b09d89 100644 --- a/src/Platform/Microsoft.Testing.Platform/Services/TestApplicationResult.cs +++ b/src/Platform/Microsoft.Testing.Platform/Services/TestApplicationResult.cs @@ -13,20 +13,28 @@ namespace Microsoft.Testing.Platform.Services; -internal sealed class TestApplicationResult( - IOutputDevice outputService, - ITestApplicationCancellationTokenSource testApplicationCancellationTokenSource, - ICommandLineOptions commandLineOptions, - IEnvironment environment) : ITestApplicationProcessExitCode, IOutputDeviceDataProducer +internal sealed class TestApplicationResult : ITestApplicationProcessExitCode, IOutputDeviceDataProducer { - private readonly IOutputDevice _outputService = outputService; - private readonly ITestApplicationCancellationTokenSource _testApplicationCancellationTokenSource = testApplicationCancellationTokenSource; - private readonly ICommandLineOptions _commandLineOptions = commandLineOptions; - private readonly IEnvironment _environment = environment; + private readonly IOutputDevice _outputService; + private readonly ICommandLineOptions _commandLineOptions; + private readonly IEnvironment _environment; + private readonly IStopPoliciesService _policiesService; private readonly List _failedTests = []; private int _totalRanTests; private bool _testAdapterTestSessionFailure; + public TestApplicationResult( + IOutputDevice outputService, + ICommandLineOptions commandLineOptions, + IEnvironment environment, + IStopPoliciesService policiesService) + { + _outputService = outputService; + _commandLineOptions = commandLineOptions; + _environment = environment; + _policiesService = policiesService; + } + /// public string Uid { get; } = nameof(TestApplicationResult); @@ -81,9 +89,10 @@ public Task ConsumeAsync(IDataProducer dataProducer, IData value, CancellationTo public int GetProcessExitCode() { int exitCode = ExitCodes.Success; + exitCode = exitCode == ExitCodes.Success && _policiesService.IsMaxFailedTestsTriggered ? ExitCodes.TestExecutionStoppedForMaxFailedTests : exitCode; exitCode = exitCode == ExitCodes.Success && _testAdapterTestSessionFailure ? ExitCodes.TestAdapterTestSessionFailure : exitCode; exitCode = exitCode == ExitCodes.Success && _failedTests.Count > 0 ? ExitCodes.AtLeastOneTestFailed : exitCode; - exitCode = exitCode == ExitCodes.Success && _testApplicationCancellationTokenSource.CancellationToken.IsCancellationRequested ? ExitCodes.TestSessionAborted : exitCode; + exitCode = exitCode == ExitCodes.Success && _policiesService.IsAbortTriggered ? ExitCodes.TestSessionAborted : exitCode; // If the user has specified the VSTestAdapterMode option, then we don't want to return a non-zero exit code if no tests ran. if (!_commandLineOptions.IsOptionSet(PlatformCommandLineProvider.VSTestAdapterModeOptionKey)) diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/HelpInfoTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/HelpInfoTests.cs index 8894ffbc2d..e64b0928f5 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/HelpInfoTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/HelpInfoTests.cs @@ -71,6 +71,8 @@ Takes one argument as string in the format [h|m|s] where 'value' is float Extension options: --filter Filters tests using the given expression. For more information, see the Filter option details section. For more information and examples on how to use selective unit test filtering, see https://learn.microsoft.com/dotnet/core/testing/selective-unit-tests. + --maximum-failed-tests + Specifies a maximum number of test failures that, when exceeded, will abort the test run. --no-ansi Disable outputting ANSI escape characters to screen. --no-progress @@ -113,6 +115,10 @@ public async Task Info_WhenMSTestExtensionRegistered_OutputInfoContentOfRegister Arity: 1..N Hidden: False Description: Specify or override a key-value pair parameter. For more information and examples, see https://learn.microsoft.com/visualstudio/test/configure-unit-tests-by-using-a-dot-runsettings-file#testrunparameters + --maximum-failed-tests + Arity: 1 + Hidden: False + Description: Specifies a maximum number of test failures that, when exceeded, will abort the test run. """; testHostResult.AssertOutputContains(output); diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/MaxFailedTestsExtensionTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/MaxFailedTestsExtensionTests.cs new file mode 100644 index 0000000000..9c5f2fe087 --- /dev/null +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/MaxFailedTestsExtensionTests.cs @@ -0,0 +1,154 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Globalization; +using System.Text.RegularExpressions; + +using Microsoft.Testing.Platform.Acceptance.IntegrationTests; +using Microsoft.Testing.Platform.Acceptance.IntegrationTests.Helpers; +using Microsoft.Testing.Platform.Helpers; + +namespace MSTest.Acceptance.IntegrationTests; + +[TestGroup] +public sealed class MaxFailedTestsExtensionTests : AcceptanceTestBase +{ + private const string AssetName = nameof(MaxFailedTestsExtensionTests); + private readonly TestAssetFixture _testAssetFixture; + + public MaxFailedTestsExtensionTests(ITestExecutionContext testExecutionContext, TestAssetFixture testAssetFixture) + : base(testExecutionContext) => _testAssetFixture = testAssetFixture; + + [ArgumentsProvider(nameof(TargetFrameworks.All), typeof(TargetFrameworks))] + public async Task SimpleMaxFailedTestsScenario(string tfm) + { + var testHost = TestHost.LocateFrom(_testAssetFixture.TargetAssetPath, AssetName, tfm); + + TestHostResult testHostResult = await testHost.ExecuteAsync("--maximum-failed-tests 3"); + testHostResult.AssertExitCodeIs(ExitCodes.TestExecutionStoppedForMaxFailedTests); + + int total = int.Parse(Regex.Match(testHostResult.StandardOutput, @"total: (\d+)").Groups[1].Value, CultureInfo.InvariantCulture); + + // We can't know the number of tests that will be executed exactly due to the async + // nature of publish/consume on the platform side. But we expect the cancellation to + // happen "fast" enough that we don't execute all tests. + Assert.IsTrue(total < 12); + Assert.IsTrue(total >= 5); + + testHostResult = await testHost.ExecuteAsync(); + testHostResult.AssertExitCodeIs(ExitCodes.AtLeastOneTestFailed); + + total = int.Parse(Regex.Match(testHostResult.StandardOutput, @"total: (\d+)").Groups[1].Value, CultureInfo.InvariantCulture); + Assert.AreEqual(12, total); + } + + [TestFixture(TestFixtureSharingStrategy.PerTestGroup)] + public sealed class TestAssetFixture(AcceptanceFixture acceptanceFixture) : TestAssetFixtureBase(acceptanceFixture.NuGetGlobalPackagesFolder) + { + private const string Sources = """ +#file MaxFailedTestsExtensionTests.csproj + + + $TargetFrameworks$ + true + Exe + enable + preview + + + + + + + +#file UnitTest1.cs +using System.Threading.Tasks; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +[TestClass] +public class UnitTest1 +{ + [TestMethod] + public void Test1() + { + Assert.Fail(); + } + + [TestMethod] + public void Test2() + { + } + + [TestMethod] + public void Test3() + { + } + + [TestMethod] + public void Test4() + { + Assert.Fail(); + } + + [TestMethod] + public void Test5() + { + Assert.Fail(); + } + + [TestMethod] + public async Task Test6() + { + await Task.Delay(10); + } + + [TestMethod] + public async Task Test7() + { + await Task.Delay(10); + } + + [TestMethod] + public async Task Test8() + { + await Task.Delay(10); + } + + [TestMethod] + public async Task Test9() + { + await Task.Delay(10); + } + + [TestMethod] + public async Task Test10() + { + await Task.Delay(10); + } + + [TestMethod] + public async Task Test11() + { + await Task.Delay(10); + } + + [TestMethod] + public async Task Test12() + { + await Task.Delay(10); + } +} +"""; + + public string TargetAssetPath => GetAssetPath(AssetName); + + public override IEnumerable<(string ID, string Name, string Code)> GetAssetsToGenerate() + { + yield return (AssetName, AssetName, + Sources + .PatchTargetFrameworks(TargetFrameworks.All) + .PatchCodeWithReplace("$MicrosoftTestingPlatformVersion$", MicrosoftTestingPlatformVersion) + .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion)); + } + } +} diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MaxFailedTestsExtensionTests.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MaxFailedTestsExtensionTests.cs new file mode 100644 index 0000000000..1f9883dbec --- /dev/null +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MaxFailedTestsExtensionTests.cs @@ -0,0 +1,167 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Platform.Acceptance.IntegrationTests.Helpers; +using Microsoft.Testing.Platform.Helpers; + +namespace Microsoft.Testing.Platform.Acceptance.IntegrationTests; + +[TestGroup] +public class MaxFailedTestsExtensionTests : AcceptanceTestBase +{ + private const string AssetName = nameof(MaxFailedTestsExtensionTests); + private readonly TestAssetFixture _testAssetFixture; + + public MaxFailedTestsExtensionTests(ITestExecutionContext testExecutionContext, TestAssetFixture testAssetFixture) + : base(testExecutionContext) => _testAssetFixture = testAssetFixture; + + public async Task TestMaxFailedTestsShouldCallStopTestExecutionAsync() + { + var testHost = TestInfrastructure.TestHost.LocateFrom(_testAssetFixture.TargetAssetPath, AssetName, TargetFrameworks.NetCurrent.Arguments); + TestHostResult testHostResult = await testHost.ExecuteAsync("--maximum-failed-tests 2"); + + testHostResult.AssertExitCodeIs(ExitCodes.TestExecutionStoppedForMaxFailedTests); + + testHostResult.AssertOutputContains("Test session is aborting due to reaching failures ('2') specified by the '--maximum-failed-tests' option."); + testHostResult.AssertOutputContainsSummary(failed: 3, passed: 3, skipped: 0); + } + + [TestFixture(TestFixtureSharingStrategy.PerTestGroup)] + public sealed class TestAssetFixture(AcceptanceFixture acceptanceFixture) : TestAssetFixtureBase(acceptanceFixture.NuGetGlobalPackagesFolder) + { + private const string Sources = """ +#file MaxFailedTestsExtensionTests.csproj + + + $TargetFrameworks$ + Exe + true + enable + preview + + + + + + +#file Program.cs +using System; +using System.Collections.Generic; +using System.Runtime.InteropServices; +using System.Threading.Tasks; +using System.Threading; +using Microsoft.Testing.Platform.Builder; +using Microsoft.Testing.Platform.Capabilities; +using Microsoft.Testing.Platform.Capabilities.TestFramework; +using Microsoft.Testing.Platform.Extensions.Messages; +using Microsoft.Testing.Platform.Extensions.TestFramework; +using Microsoft.Testing.Platform.Helpers; + +internal sealed class Program +{ + public static async Task Main(string[] args) + { + ITestApplicationBuilder builder = await TestApplication.CreateBuilderAsync(args); + var adapter = new DummyAdapter(); + builder.RegisterTestFramework(_ => new Capabilities(), (_, __) => adapter); + +#pragma warning disable TPEXP // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + builder.AddMaximumFailedTestsService(adapter); +#pragma warning restore TPEXP // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + + using ITestApplication app = await builder.BuildAsync(); + return await app.RunAsync(); + } +} + +internal class DummyAdapter : ITestFramework, IDataProducer +{ + public string Uid => nameof(DummyAdapter); + + public string Version => string.Empty; + + public string DisplayName => string.Empty; + + public string Description => string.Empty; + + public Type[] DataTypesProduced => new[] { typeof(TestNodeUpdateMessage) }; + + public Task CloseTestSessionAsync(CloseTestSessionContext context) => Task.FromResult(new CloseTestSessionResult() { IsSuccess = true }); + + public Task CreateTestSessionAsync(CreateTestSessionContext context) => Task.FromResult(new CreateTestSessionResult() { IsSuccess = true }); + + public async Task ExecuteRequestAsync(ExecuteRequestContext context) + { + // First fail. + await context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage(context.Request.Session.SessionUid, + new TestNode() { Uid = "1", DisplayName = "Test1", Properties = new(new FailedTestNodeStateProperty()) })); + + await context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage(context.Request.Session.SessionUid, + new TestNode() { Uid = "2", DisplayName = "Test2", Properties = new(PassedTestNodeStateProperty.CachedInstance) })); + + await context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage(context.Request.Session.SessionUid, + new TestNode() { Uid = "3", DisplayName = "Test3", Properties = new(PassedTestNodeStateProperty.CachedInstance) })); + + await context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage(context.Request.Session.SessionUid, + new TestNode() { Uid = "4", DisplayName = "Test4", Properties = new(PassedTestNodeStateProperty.CachedInstance) })); + + if (GracefulStop.Instance.IsStopRequested) throw new InvalidOperationException("Unexpected stop request!"); + + // Second fail. + await context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage(context.Request.Session.SessionUid, + new TestNode() { Uid = "5", DisplayName = "Test5", Properties = new(new FailedTestNodeStateProperty()) })); + + await GracefulStop.Instance.TCS.Task; + + if (!GracefulStop.Instance.IsStopRequested) throw new InvalidOperationException("Expected stop request!"); + + // Third fail. + await context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage(context.Request.Session.SessionUid, + new TestNode() { Uid = "6", DisplayName = "Test6", Properties = new(new FailedTestNodeStateProperty()) })); + + context.Complete(); + } + + public Task IsEnabledAsync() => Task.FromResult(true); +} + +internal class Capabilities : ITestFrameworkCapabilities +{ + IReadOnlyCollection ICapabilities.Capabilities => [GracefulStop.Instance]; +} + +#pragma warning disable TPEXP // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. +internal sealed class GracefulStop : IGracefulStopTestExecutionCapability +#pragma warning restore TPEXP // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. +{ + private GracefulStop() + { + } + + public static GracefulStop Instance { get; } = new(); + + public TaskCompletionSource TCS { get; } = new(); + + public bool IsStopRequested { get; private set; } + + public Task StopTestExecutionAsync(CancellationToken cancellationToken) + { + IsStopRequested = true; + TCS.SetResult(); + return Task.CompletedTask; + } +} + +"""; + + public string TargetAssetPath => GetAssetPath(AssetName); + + public override IEnumerable<(string ID, string Name, string Code)> GetAssetsToGenerate() + { + yield return (AssetName, AssetName, + Sources + .PatchTargetFrameworks(TargetFrameworks.NetCurrent) + .PatchCodeWithReplace("$MicrosoftTestingPlatformVersion$", MicrosoftTestingPlatformVersion)); + } + } +} diff --git a/test/UnitTests/MSTestAdapter.UnitTests/Execution/TestClassInfoTests.cs b/test/UnitTests/MSTestAdapter.UnitTests/Execution/TestClassInfoTests.cs index c4840468df..9fc878852d 100644 --- a/test/UnitTests/MSTestAdapter.UnitTests/Execution/TestClassInfoTests.cs +++ b/test/UnitTests/MSTestAdapter.UnitTests/Execution/TestClassInfoTests.cs @@ -419,6 +419,7 @@ public void RunClassCleanupShouldInvokeIfClassCleanupMethod() _testClassInfo.ClassCleanupMethod = typeof(DummyTestClass).GetMethod(nameof(DummyTestClass.ClassCleanupMethod)); // Act + _testClassInfo.RunClassInitialize(null); _testClassInfo.ExecuteClassCleanup(); // Assert @@ -446,6 +447,7 @@ public void RunClassCleanupShouldReturnAssertFailureExceptionDetails() _testClassInfo.ClassCleanupMethod = typeof(DummyTestClass).GetMethod(nameof(DummyTestClass.ClassCleanupMethod)); // Act + _testClassInfo.RunClassInitialize(null); Exception classCleanupException = VerifyThrows(_testClassInfo.ExecuteClassCleanup); // Assert @@ -461,6 +463,7 @@ public void RunClassCleanupShouldReturnAssertInconclusiveExceptionDetails() _testClassInfo.ClassCleanupMethod = typeof(DummyTestClass).GetMethod(nameof(DummyTestClass.ClassCleanupMethod)); // Act + _testClassInfo.RunClassInitialize(null); Exception classCleanupException = VerifyThrows(_testClassInfo.ExecuteClassCleanup); // Assert @@ -476,6 +479,7 @@ public void RunClassCleanupShouldReturnExceptionDetailsOfNonAssertExceptions() _testClassInfo.ClassCleanupMethod = typeof(DummyTestClass).GetMethod(nameof(DummyTestClass.ClassCleanupMethod)); // Act + _testClassInfo.RunClassInitialize(null); Exception classCleanupException = VerifyThrows(_testClassInfo.ExecuteClassCleanup); // Assert @@ -493,6 +497,7 @@ public void RunBaseClassCleanupWithNoDerivedClassCleanupShouldReturnExceptionDet _testClassInfo.BaseClassCleanupMethods.Add(baseClassCleanupMethod); // Act + _testClassInfo.RunClassInitialize(null); Exception classCleanupException = VerifyThrows(_testClassInfo.ExecuteClassCleanup); // Assert @@ -515,6 +520,22 @@ public void RunBaseClassCleanupEvenIfThereIsNoDerivedClassCleanup() // Assert Verify(_testClassInfo.HasExecutableCleanupMethod); + Verify(classCleanupCallCount == 0, "DummyBaseTestClass.CleanupClassMethod call count"); + + // Act 2 + _testClassInfo.RunClassInitialize(null); + _testClassInfo.ExecuteClassCleanup(); + + // Assert 2 + Verify(_testClassInfo.HasExecutableCleanupMethod); + Verify(_testClassInfo.IsClassInitializeExecuted); + Verify(classCleanupCallCount == 1, "DummyBaseTestClass.CleanupClassMethod call count"); + + // Act 3 + _testClassInfo.ExecuteClassCleanup(); + + // Assert 3 + Verify(_testClassInfo.HasExecutableCleanupMethod); Verify(classCleanupCallCount == 1, "DummyBaseTestClass.CleanupClassMethod call count"); } @@ -526,6 +547,7 @@ public void RunClassCleanupShouldThrowTheInnerMostExceptionWhenThereAreMultipleN DummyTestClass.ClassCleanupMethodBody = FailingStaticHelper.DoWork; _testClassInfo.ClassCleanupMethod = typeof(DummyTestClass).GetMethod("ClassCleanupMethod"); + _testClassInfo.RunClassInitialize(null); Exception classCleanupException = VerifyThrows(_testClassInfo.ExecuteClassCleanup); Verify(classCleanupException.Message.StartsWith("Class Cleanup method DummyTestClass.ClassCleanupMethod failed. Error Message: System.InvalidOperationException: I fail..", StringComparison.Ordinal)); diff --git a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Services/TestApplicationResultTests.cs b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Services/TestApplicationResultTests.cs index 6964e6aa45..804d5aecf0 100644 --- a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Services/TestApplicationResultTests.cs +++ b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Services/TestApplicationResultTests.cs @@ -15,7 +15,7 @@ namespace Microsoft.Testing.Platform.UnitTests; public sealed class TestApplicationResultTests : TestBase { private readonly TestApplicationResult _testApplicationResult - = new(new Mock().Object, new Mock().Object, new Mock().Object, new Mock().Object); + = new(new Mock().Object, new Mock().Object, new Mock().Object, new Mock().Object); public TestApplicationResultTests(ITestExecutionContext testExecutionContext) : base(testExecutionContext) @@ -68,15 +68,17 @@ public async Task GetProcessExitCodeAsync_If_Failed_Tests_Returns_AtLeastOneTest public async Task GetProcessExitCodeAsync_If_Canceled_Returns_TestSessionAborted() { Mock testApplicationCancellationTokenSource = new(); + // CTS should not be created in SetupGet so that the mocked ITestApplicationCancellationTokenSource returns the same instance on every access + // which is the case in the real production implementation. + CancellationTokenSource cancellationTokenSource = new(); testApplicationCancellationTokenSource.SetupGet(x => x.CancellationToken).Returns(() => { - CancellationTokenSource cancellationTokenSource = new(); cancellationTokenSource.Cancel(); return cancellationTokenSource.Token; }); TestApplicationResult testApplicationResult - = new(new Mock().Object, testApplicationCancellationTokenSource.Object, new Mock().Object, new Mock().Object); + = new(new Mock().Object, new Mock().Object, new Mock().Object, new StopPoliciesService(testApplicationCancellationTokenSource.Object)); await testApplicationResult.ConsumeAsync(new DummyProducer(), new TestNodeUpdateMessage( default, @@ -108,9 +110,10 @@ public async Task GetProcessExitCodeAsync_If_TestAdapter_Returns_TestAdapterTest public async Task GetProcessExitCodeAsync_If_MinimumExpectedTests_Violated_Returns_MinimumExpectedTestsPolicyViolation() { TestApplicationResult testApplicationResult - = new(new Mock().Object, new Mock().Object, - new CommandLineOption(PlatformCommandLineProvider.MinimumExpectedTestsOptionKey, ["2"]), - new Mock().Object); + = new( + new Mock().Object, + new CommandLineOption(PlatformCommandLineProvider.MinimumExpectedTestsOptionKey, ["2"]), + new Mock().Object, new Mock().Object); await testApplicationResult.ConsumeAsync(new DummyProducer(), new TestNodeUpdateMessage( default, @@ -136,9 +139,10 @@ TestApplicationResult testApplicationResult public async Task GetProcessExitCodeAsync_OnDiscovery_No_Tests_Discovered_Returns_ZeroTests() { TestApplicationResult testApplicationResult - = new(new Mock().Object, new Mock().Object, - new CommandLineOption(PlatformCommandLineProvider.DiscoverTestsOptionKey, []), - new Mock().Object); + = new( + new Mock().Object, + new CommandLineOption(PlatformCommandLineProvider.DiscoverTestsOptionKey, []), + new Mock().Object, new Mock().Object); await testApplicationResult.ConsumeAsync(new DummyProducer(), new TestNodeUpdateMessage( default, @@ -154,9 +158,10 @@ TestApplicationResult testApplicationResult public async Task GetProcessExitCodeAsync_OnDiscovery_Some_Tests_Discovered_Returns_Success() { TestApplicationResult testApplicationResult - = new(new Mock().Object, new Mock().Object, - new CommandLineOption(PlatformCommandLineProvider.DiscoverTestsOptionKey, []), - new Mock().Object); + = new( + new Mock().Object, + new CommandLineOption(PlatformCommandLineProvider.DiscoverTestsOptionKey, []), + new Mock().Object, new Mock().Object); await testApplicationResult.ConsumeAsync(new DummyProducer(), new TestNodeUpdateMessage( default, @@ -188,12 +193,15 @@ public void GetProcessExitCodeAsync_IgnoreExitCodes(string argument, int expecte foreach (TestApplicationResult testApplicationResult in new TestApplicationResult[] { - new(new Mock().Object, new Mock().Object, + new( + new Mock().Object, new CommandLineOption(PlatformCommandLineProvider.IgnoreExitCodeOptionKey, argument is null ? [] : [argument]), - new Mock().Object), - new(new Mock().Object, new Mock().Object, + new Mock().Object, new Mock().Object), + new( + new Mock().Object, new Mock().Object, - environment.Object), + environment.Object, + new Mock().Object), }) { Assert.AreEqual(expectedExitCode, testApplicationResult.GetProcessExitCode());