Support '--maximum-failed-tests' to abort test run when failure threshold is reached

src/Platform/Microsoft.Testing.Platform/Capabilities/TestFramework/IStopTestExecutionCapability.cs

@@ -0,0 +1,16 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace Microsoft.Testing.Platform.Capabilities.TestFramework;
+
+/// <summary>
+/// A capability to support stopping test execution gracefully, without cancelling/aborting everything.
+/// This is used to support '--max-failed-tests'.
+/// </summary>
+/// <remarks>
+/// Test frameworks can choose to run any needed cleanup when cancellation is requested.
+/// </remarks>
+public interface IStopTestExecutionCapability : ITestFrameworkCapability
+{
+    Task StopTestExecutionAsync(CancellationToken cancellationToken);
+}

src/Platform/Microsoft.Testing.Platform/CommandLine/PlatformCommandLineProvider.cs

@@ -32,6 +32,7 @@ internal sealed class PlatformCommandLineProvider : ICommandLineOptionsProvider
     public const string TestHostControllerPIDOptionKey = "internal-testhostcontroller-pid";
     public const string ExitOnProcessExitOptionKey = "exit-on-process-exit";
     public const string ConfigFileOptionKey = "config-file";
+    public const string MaxFailedTestsOptionKey = "max-failed-tests";
 
     public const string ServerOptionKey = "server";
     public const string ClientPortOptionKey = "client-port";
@@ -61,6 +62,7 @@ internal sealed class PlatformCommandLineProvider : ICommandLineOptionsProvider
         new(IgnoreExitCodeOptionKey, PlatformResources.PlatformCommandLineIgnoreExitCodeOptionDescription, ArgumentArity.ExactlyOne, false, isBuiltIn: true),
         new(ExitOnProcessExitOptionKey, PlatformResources.PlatformCommandLineExitOnProcessExitOptionDescription, ArgumentArity.ExactlyOne, false, isBuiltIn: true),
         new(ConfigFileOptionKey, PlatformResources.PlatformCommandLineConfigFileOptionDescription, ArgumentArity.ExactlyOne, false, isBuiltIn: true),
+        new(MaxFailedTestsOptionKey, PlatformResources.PlatformCommandLineMaxFailedTestsOptionDescription, ArgumentArity.ExactlyOne, false, isBuiltIn: true),
 
         // Hidden options
         new(HelpOptionQuestionMark, PlatformResources.PlatformCommandLineHelpOptionDescription, ArgumentArity.Zero, true, isBuiltIn: true),
@@ -141,6 +143,15 @@ public Task<ValidationResult> ValidateOptionArgumentsAsync(CommandLineOption com
             }
         }
 
+        if (commandOption.Name == MaxFailedTestsOptionKey)
+        {
+            string arg = arguments[0];
+            if (!int.TryParse(arg, out int maxFailedTestsResult) || maxFailedTestsResult <= 0)
+            {
+                return ValidationResult.InvalidTask(string.Format(CultureInfo.InvariantCulture, PlatformResources.MaxFailedTestsMustBePositive, arg));
+            }
+        }
+
         // Now validate the minimum expected tests option
         return IsMinimumExpectedTestsOptionValidAsync(commandOption, arguments);
     }

src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs

@@ -0,0 +1,71 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using Microsoft.Testing.Platform.Capabilities.TestFramework;
+using Microsoft.Testing.Platform.CommandLine;
+using Microsoft.Testing.Platform.Extensions.Messages;
+using Microsoft.Testing.Platform.Extensions.TestHost;
+using Microsoft.Testing.Platform.Helpers;
+using Microsoft.Testing.Platform.Messages;
+using Microsoft.Testing.Platform.Resources;
+using Microsoft.Testing.Platform.Services;
+
+namespace Microsoft.Testing.Platform.Extensions;
+
+internal sealed class AbortForMaxFailedTestsExtension : IDataConsumer
+{
+    private readonly int? _maxFailedTests;
+    private readonly IStopTestExecutionCapability? _capability;
+    private readonly PoliciesService _policiesService;
+    private readonly CancellationToken _cancellationToken;
+    private int _failCount;
+
+    public AbortForMaxFailedTestsExtension(ICommandLineOptions commandLineOptions, IStopTestExecutionCapability? capability, PoliciesService policiesService, CancellationToken cancellationToken)
+    {
+        if (commandLineOptions.TryGetOptionArgumentList(PlatformCommandLineProvider.MaxFailedTestsOptionKey, out string[]? args) &&
+            int.TryParse(args[0], out int maxFailedTests) &&
+            maxFailedTests > 0)
+        {
+            _maxFailedTests = maxFailedTests;
+        }
+
+        _capability = capability;
+        _policiesService = policiesService;
+        _cancellationToken = cancellationToken;
+    }
+
+    public Type[] DataTypesConsumed { get; } = [typeof(TestNodeUpdateMessage)];
+
+    /// <inheritdoc />
+    public string Uid { get; } = nameof(AbortForMaxFailedTestsExtension);
+
+    /// <inheritdoc />
+    public string Version { get; } = AppVersion.DefaultSemVer;
+
+    /// <inheritdoc />
+    public string DisplayName { get; } = nameof(AbortForMaxFailedTestsExtension);
+
+    /// <inheritdoc />
+    public string Description { get; } = PlatformResources.AbortForMaxFailedTestsDescription;
+
+    /// <inheritdoc />
+    public Task<bool> IsEnabledAsync() => Task.FromResult(_maxFailedTests.HasValue && _capability is not null);
+
+    public async Task ConsumeAsync(IDataProducer dataProducer, IData value, CancellationToken cancellationToken)
+    {
+        var node = (TestNodeUpdateMessage)value;
+
+        // If we are called, the extension is enabled, which means both _maxFailedTests and _cancellationTokenSource are not null.
+        RoslynDebug.Assert(_maxFailedTests is not null);
+        RoslynDebug.Assert(_capability is not null);
+
+        int maxFailed = _maxFailedTests.Value;
+        TestNodeStateProperty testNodeStateProperty = node.TestNode.Properties.Single<TestNodeStateProperty>();
+        if (TestNodePropertiesCategories.WellKnownTestNodeTestRunOutcomeFailedProperties.Any(t => t == testNodeStateProperty.GetType()) &&
+            Interlocked.Increment(ref _failCount) > maxFailed)
+        {
+            await _capability.StopTestExecutionAsync(_cancellationToken);
+            await _policiesService.ExecuteOnStopTestExecutionCallbacks(_cancellationToken);
+        }
+    }
+}

src/Platform/Microsoft.Testing.Platform/Helpers/ExitCodes.cs

@@ -22,4 +22,5 @@ internal static class ExitCodes
     public const int TestAdapterTestSessionFailure = 10;
     public const int DependentProcessExited = 11;
     public const int IncompatibleProtocolVersion = 12;
+    public const int TestExecutionStopped = 13;
 }

src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs

@@ -206,6 +206,9 @@ public async Task<ITestHost> BuildAsync(
         // Set the concrete command line options to the proxy.
         commandLineOptionsProxy.SetCommandLineOptions(commandLineHandler);
 
+        var policiesService = new PoliciesService();
+        serviceProvider.AddService(policiesService);
+
         bool hasServerFlag = commandLineHandler.TryGetOptionArgumentList(PlatformCommandLineProvider.ServerOptionKey, out string[]? protocolName);
         bool isJsonRpcProtocol = protocolName is null || protocolName.Length == 0 || protocolName[0].Equals(PlatformCommandLineProvider.JsonRpcProtocolName, StringComparison.OrdinalIgnoreCase);
 
@@ -243,6 +246,13 @@ public async Task<ITestHost> BuildAsync(
             await testFrameworkCapabilitiesAsyncInitializable.InitializeAsync();
         }
 
+        TestHost.AddDataConsumer(
+            serviceProvider => new AbortForMaxFailedTestsExtension(
+                serviceProvider.GetCommandLineOptions(),
+                serviceProvider.GetTestFrameworkCapabilities().GetCapability<IStopTestExecutionCapability>(),
+                policiesService,
+                serviceProvider.GetTestApplicationCancellationTokenSource().CancellationToken));
+
         // If command line is not valid we return immediately.
         ValidationResult commandLineValidationResult = await CommandLineOptionsValidator.ValidateAsync(
             loggingState.CommandLineParseResult,
@@ -315,7 +325,8 @@ public async Task<ITestHost> BuildAsync(
             proxyOutputDevice,
             serviceProvider.GetTestApplicationCancellationTokenSource(),
             serviceProvider.GetCommandLineOptions(),
-            serviceProvider.GetEnvironment());
+            serviceProvider.GetEnvironment(),
+            policiesService);
         serviceProvider.AddService(testApplicationResult);
 
         // ============= SETUP COMMON SERVICE USED IN ALL MODES END ===============//

src/Platform/Microsoft.Testing.Platform/OutputDevice/OutputDeviceManager.cs

@@ -30,7 +30,11 @@ internal async Task<ProxyOutputDevice> BuildAsync(ServiceProvider serviceProvide
             nonServerOutputDevice = GetDefaultTerminalOutputDevice(serviceProvider);
         }
 
-        return new ProxyOutputDevice(nonServerOutputDevice, useServerModeOutputDevice ? new ServerModePerCallOutputDevice(serviceProvider) : null);
+        return new ProxyOutputDevice(
+            nonServerOutputDevice,
+            useServerModeOutputDevice
+                ? new ServerModePerCallOutputDevice(serviceProvider)
+                : null);
     }
 
     public static TerminalOutputDevice GetDefaultTerminalOutputDevice(ServiceProvider serviceProvider)

src/Platform/Microsoft.Testing.Platform/PublicAPI/PublicAPI.Unshipped.txt

@@ -1,4 +1,6 @@
 #nullable enable
+Microsoft.Testing.Platform.Capabilities.TestFramework.IStopTestExecutionCapability
+Microsoft.Testing.Platform.Capabilities.TestFramework.IStopTestExecutionCapability.StopTestExecutionAsync(System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task!
 Microsoft.Testing.Platform.Extensions.Messages.TestMetadataProperty.TestMetadataProperty(string! key) -> void
 Microsoft.Testing.Platform.OutputDevice.ErrorMessageOutputDeviceData
 Microsoft.Testing.Platform.OutputDevice.ErrorMessageOutputDeviceData.ErrorMessageOutputDeviceData(string! message) -> void

src/Platform/Microsoft.Testing.Platform/Resources/PlatformResources.resx

@@ -694,4 +694,13 @@ Takes one argument as string in the format <value>[h|m|s] where 'value' is
     <value>Exception during the cancellation of request id '{0}'</value>
     <comment>{0} is the request id</comment>
   </data>
+  <data name="PlatformCommandLineMaxFailedTestsOptionDescription" xml:space="preserve">
+    <value>Specifies a maximum number of test failures that, when exceeded, will abort the test run.</value>
+  </data>
+  <data name="MaxFailedTestsMustBePositive" xml:space="preserve">
+    <value>The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid.</value>
+  </data>
+  <data name="AbortForMaxFailedTestsDescription" xml:space="preserve">
+    <value>Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted.</value>
+  </data>
 </root>

src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.cs.xlf

@@ -2,6 +2,11 @@
 <xliff xmlns="urn:oasis:names:tc:xliff:document:1.2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" version="1.2" xsi:schemaLocation="urn:oasis:names:tc:xliff:document:1.2 xliff-core-1.2-transitional.xsd">
   <file datatype="xml" source-language="en" target-language="cs" original="../PlatformResources.resx">
     <body>
+      <trans-unit id="AbortForMaxFailedTestsDescription">
+        <source>Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted.</source>
+        <target state="new">Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted.</target>
+        <note />
+      </trans-unit>
       <trans-unit id="Aborted">
         <source>Aborted</source>
         <target state="translated">Přerušeno</target>
@@ -371,6 +376,11 @@
         <target state="translated">Rozhraní ILoggerFactory ještě nebylo sestaveno.</target>
         <note />
       </trans-unit>
+      <trans-unit id="MaxFailedTestsMustBePositive">
+        <source>The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid.</source>
+        <target state="new">The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid.</target>
+        <note />
+      </trans-unit>
       <trans-unit id="MessageBusNotReady">
         <source>The message bus has not been built yet or is no more usable at this stage.</source>
         <target state="translated">Sběrnice zpráv ještě nebyla sestavena nebo už není v této fázi použitelná.</target>
@@ -525,6 +535,11 @@ Dostupné hodnoty jsou Trace, Debug, Information, Warning, Error a Critical.</ta
         <target state="translated">Umožňuje zobrazit informace o testovací aplikaci .NET.</target>
         <note />
       </trans-unit>
+      <trans-unit id="PlatformCommandLineMaxFailedTestsOptionDescription">
+        <source>Specifies a maximum number of test failures that, when exceeded, will abort the test run.</source>
+        <target state="new">Specifies a maximum number of test failures that, when exceeded, will abort the test run.</target>
+        <note />
+      </trans-unit>
       <trans-unit id="PlatformCommandLineMinimumExpectedTestsIncompatibleDiscoverTests">
         <source>'--list-tests' and '--minimum-expected-tests' are incompatible options</source>
         <target state="translated">--list-tests a --minimum-expected-tests jsou nekompatibilní možnosti.</target>

src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.de.xlf

@@ -2,6 +2,11 @@
 <xliff xmlns="urn:oasis:names:tc:xliff:document:1.2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" version="1.2" xsi:schemaLocation="urn:oasis:names:tc:xliff:document:1.2 xliff-core-1.2-transitional.xsd">
   <file datatype="xml" source-language="en" target-language="de" original="../PlatformResources.resx">
     <body>
+      <trans-unit id="AbortForMaxFailedTestsDescription">
+        <source>Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted.</source>
+        <target state="new">Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted.</target>
+        <note />
+      </trans-unit>
       <trans-unit id="Aborted">
         <source>Aborted</source>
         <target state="translated">Abgebrochen</target>
@@ -371,6 +376,11 @@
         <target state="translated">Die ILoggerFactory wurde noch nicht erstellt.</target>
         <note />
       </trans-unit>
+      <trans-unit id="MaxFailedTestsMustBePositive">
+        <source>The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid.</source>
+        <target state="new">The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid.</target>
+        <note />
+      </trans-unit>
       <trans-unit id="MessageBusNotReady">
         <source>The message bus has not been built yet or is no more usable at this stage.</source>
         <target state="translated">Der Nachrichtenbus wurde noch nicht erstellt oder kann zu diesem Zeitpunkt nicht mehr verwendet werden.</target>
@@ -525,6 +535,11 @@ Die verfügbaren Werte sind "Trace", "Debug", "Information", "Warning", "Error"
         <target state="translated">Zeigen Sie .NET-Testanwendungsinformationen an.</target>
         <note />
       </trans-unit>
+      <trans-unit id="PlatformCommandLineMaxFailedTestsOptionDescription">
+        <source>Specifies a maximum number of test failures that, when exceeded, will abort the test run.</source>
+        <target state="new">Specifies a maximum number of test failures that, when exceeded, will abort the test run.</target>
+        <note />
+      </trans-unit>
       <trans-unit id="PlatformCommandLineMinimumExpectedTestsIncompatibleDiscoverTests">
         <source>'--list-tests' and '--minimum-expected-tests' are incompatible options</source>
         <target state="translated">"--list-tests" und "--minimum-expected-tests" sind inkompatible Optionen.</target>

src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.es.xlf

@@ -2,6 +2,11 @@
 <xliff xmlns="urn:oasis:names:tc:xliff:document:1.2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" version="1.2" xsi:schemaLocation="urn:oasis:names:tc:xliff:document:1.2 xliff-core-1.2-transitional.xsd">
   <file datatype="xml" source-language="en" target-language="es" original="../PlatformResources.resx">
     <body>
+      <trans-unit id="AbortForMaxFailedTestsDescription">
+        <source>Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted.</source>
+        <target state="new">Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted.</target>
+        <note />
+      </trans-unit>
       <trans-unit id="Aborted">
         <source>Aborted</source>
         <target state="translated">Anulado</target>
@@ -371,6 +376,11 @@
         <target state="translated">ILoggerFactory aún no se ha compilado.</target>
         <note />
       </trans-unit>
+      <trans-unit id="MaxFailedTestsMustBePositive">
+        <source>The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid.</source>
+        <target state="new">The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid.</target>
+        <note />
+      </trans-unit>
       <trans-unit id="MessageBusNotReady">
         <source>The message bus has not been built yet or is no more usable at this stage.</source>
         <target state="translated">El bus de mensajes aún no se ha compilado o ya no se puede usar en esta fase.</target>
@@ -525,6 +535,11 @@ Los valores disponibles son 'Seguimiento', 'Depurar', 'Información', 'Advertenc
         <target state="translated">Muestre la información de la aplicación de prueba de .NET.</target>
         <note />
       </trans-unit>
+      <trans-unit id="PlatformCommandLineMaxFailedTestsOptionDescription">
+        <source>Specifies a maximum number of test failures that, when exceeded, will abort the test run.</source>
+        <target state="new">Specifies a maximum number of test failures that, when exceeded, will abort the test run.</target>
+        <note />
+      </trans-unit>
       <trans-unit id="PlatformCommandLineMinimumExpectedTestsIncompatibleDiscoverTests">
         <source>'--list-tests' and '--minimum-expected-tests' are incompatible options</source>
         <target state="translated">“--list-tests” y “--minimum-expected-tests” son opciones incompatibles</target>