Support '--maximum-failed-tests' to abort test run when failure threshold is reached

src/Platform/Microsoft.Testing.Platform/CommandLine/PlatformCommandLineProvider.cs

@@ -32,6 +32,7 @@ internal sealed class PlatformCommandLineProvider : ICommandLineOptionsProvider
     public const string TestHostControllerPIDOptionKey = "internal-testhostcontroller-pid";
     public const string ExitOnProcessExitOptionKey = "exit-on-process-exit";
     public const string ConfigFileOptionKey = "config-file";
+    public const string MaxFailedTestsOptionKey = "max-failed-tests";
 
     public const string ServerOptionKey = "server";
     public const string ClientPortOptionKey = "client-port";
@@ -61,6 +62,7 @@ internal sealed class PlatformCommandLineProvider : ICommandLineOptionsProvider
         new(IgnoreExitCodeOptionKey, PlatformResources.PlatformCommandLineIgnoreExitCodeOptionDescription, ArgumentArity.ExactlyOne, false, isBuiltIn: true),
         new(ExitOnProcessExitOptionKey, PlatformResources.PlatformCommandLineExitOnProcessExitOptionDescription, ArgumentArity.ExactlyOne, false, isBuiltIn: true),
         new(ConfigFileOptionKey, PlatformResources.PlatformCommandLineConfigFileOptionDescription, ArgumentArity.ExactlyOne, false, isBuiltIn: true),
+        new(MaxFailedTestsOptionKey, PlatformResources.PlatformCommandLineMaxFailedTestsOptionDescription, ArgumentArity.ExactlyOne, false, isBuiltIn: true),
 
         // Hidden options
         new(HelpOptionQuestionMark, PlatformResources.PlatformCommandLineHelpOptionDescription, ArgumentArity.Zero, true, isBuiltIn: true),
@@ -141,6 +143,15 @@ public Task<ValidationResult> ValidateOptionArgumentsAsync(CommandLineOption com
             }
         }
 
+        if (commandOption.Name == MaxFailedTestsOptionKey)
+        {
+            string arg = arguments[0];
+            if (!int.TryParse(arg, out int maxFailedTestsResult) || maxFailedTestsResult <= 0)
+            {
+                return ValidationResult.InvalidTask(string.Format(CultureInfo.InvariantCulture, PlatformResources.MaxFailedTestsMustBePositive, arg));
+            }
+        }
+
         // Now validate the minimum expected tests option
         return IsMinimumExpectedTestsOptionValidAsync(commandOption, arguments);
     }

src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs

@@ -0,0 +1,66 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using Microsoft.Testing.Platform.CommandLine;
+using Microsoft.Testing.Platform.Extensions.Messages;
+using Microsoft.Testing.Platform.Extensions.TestHost;
+using Microsoft.Testing.Platform.Helpers;
+using Microsoft.Testing.Platform.Resources;
+using Microsoft.Testing.Platform.Services;
+
+namespace Microsoft.Testing.Platform.Extensions;
+
+internal sealed class AbortForMaxFailedTestsExtension : IDataConsumer
+{
+    private readonly IServiceProvider _serviceProvider;
+    private readonly int? _maxFailedTests;
+
+    private int _failCount;
+
+    public AbortForMaxFailedTestsExtension(IServiceProvider serviceProvider)
+    {
+        _serviceProvider = serviceProvider;
+        if (serviceProvider.GetCommandLineOptions().TryGetOptionArgumentList(PlatformCommandLineProvider.MaxFailedTestsOptionKey, out string[]? args) &&
+            int.TryParse(args[0], out int maxFailedTests) &&
+            maxFailedTests > 0)
+        {
+            _maxFailedTests = maxFailedTests;
+        }
+    }
+
+    public Type[] DataTypesConsumed { get; } = [typeof(TestNodeUpdateMessage)];
+
+    /// <inheritdoc />
+    public string Uid { get; } = nameof(AbortForMaxFailedTestsExtension);
+
+    /// <inheritdoc />
+    public string Version { get; } = AppVersion.DefaultSemVer;
+
+    /// <inheritdoc />
+    public string DisplayName { get; } = nameof(AbortForMaxFailedTestsExtension);
+
+    /// <inheritdoc />
+    public string Description { get; } = PlatformResources.AbortForMaxFailedTestsDescription;
+
+    /// <inheritdoc />
+    public Task<bool> IsEnabledAsync() => Task.FromResult(_maxFailedTests.HasValue);
+
+    public Task ConsumeAsync(IDataProducer dataProducer, IData value, CancellationToken cancellationToken)
+    {
+        var node = (TestNodeUpdateMessage)value;
+
+        // If we are called, the extension is enabled, which means _maxFailedTests.HasValue was true. So null suppression is safe.
+        int maxFailed = _maxFailedTests!.Value;
+        if (node.TestNode.Properties.Single<TestNodeStateProperty>() is FailedTestNodeStateProperty)
+        {
+            Interlocked.Increment(ref _failCount);
+        }
+
+        if (_failCount > maxFailed)
+        {
+            _serviceProvider.GetTestApplicationCancellationTokenSource().Cancel();
+        }
+
+        return Task.CompletedTask;
+    }
+}

src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs

@@ -236,6 +236,8 @@ public async Task<ITestHost> BuildAsync(
         serviceProvider.TryAddService(proxyOutputDevice);
         serviceProvider.TryAddService(proxyOutputDevice.OriginalOutputDevice);
 
+        TestHost.AddDataConsumer(serviceProvider => new AbortForMaxFailedTestsExtension(serviceProvider));
+
         // Create the test framework capabilities
         ITestFrameworkCapabilities testFrameworkCapabilities = TestFramework.TestFrameworkCapabilitiesFactory(serviceProvider);
         if (testFrameworkCapabilities is IAsyncInitializableExtension testFrameworkCapabilitiesAsyncInitializable)

src/Platform/Microsoft.Testing.Platform/Resources/PlatformResources.resx

@@ -694,4 +694,13 @@ Takes one argument as string in the format <value>[h|m|s] where 'value' is
     <value>Exception during the cancellation of request id '{0}'</value>
     <comment>{0} is the request id</comment>
   </data>
+  <data name="PlatformCommandLineMaxFailedTestsOptionDescription" xml:space="preserve">
+    <value>Specifies a maximum number of test failures that, when exceeded, will abort the test run.</value>
+  </data>
+  <data name="MaxFailedTestsMustBePositive" xml:space="preserve">
+    <value>The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid.</value>
+  </data>
+  <data name="AbortForMaxFailedTestsDescription" xml:space="preserve">
+    <value>Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted.</value>
+  </data>
 </root>

src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.cs.xlf

@@ -2,6 +2,11 @@
 <xliff xmlns="urn:oasis:names:tc:xliff:document:1.2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" version="1.2" xsi:schemaLocation="urn:oasis:names:tc:xliff:document:1.2 xliff-core-1.2-transitional.xsd">
   <file datatype="xml" source-language="en" target-language="cs" original="../PlatformResources.resx">
     <body>
+      <trans-unit id="AbortForMaxFailedTestsDescription">
+        <source>Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted.</source>
+        <target state="new">Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted.</target>
+        <note />
+      </trans-unit>
       <trans-unit id="Aborted">
         <source>Aborted</source>
         <target state="translated">Přerušeno</target>
@@ -371,6 +376,11 @@
         <target state="translated">Rozhraní ILoggerFactory ještě nebylo sestaveno.</target>
         <note />
       </trans-unit>
+      <trans-unit id="MaxFailedTestsMustBePositive">
+        <source>The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid.</source>
+        <target state="new">The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid.</target>
+        <note />
+      </trans-unit>
       <trans-unit id="MessageBusNotReady">
         <source>The message bus has not been built yet or is no more usable at this stage.</source>
         <target state="translated">Sběrnice zpráv ještě nebyla sestavena nebo už není v této fázi použitelná.</target>
@@ -525,6 +535,11 @@ Dostupné hodnoty jsou Trace, Debug, Information, Warning, Error a Critical.</ta
         <target state="translated">Umožňuje zobrazit informace o testovací aplikaci .NET.</target>
         <note />
       </trans-unit>
+      <trans-unit id="PlatformCommandLineMaxFailedTestsOptionDescription">
+        <source>Specifies a maximum number of test failures that, when exceeded, will abort the test run.</source>
+        <target state="new">Specifies a maximum number of test failures that, when exceeded, will abort the test run.</target>
+        <note />
+      </trans-unit>
       <trans-unit id="PlatformCommandLineMinimumExpectedTestsIncompatibleDiscoverTests">
         <source>'--list-tests' and '--minimum-expected-tests' are incompatible options</source>
         <target state="translated">--list-tests a --minimum-expected-tests jsou nekompatibilní možnosti.</target>

src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.de.xlf

@@ -2,6 +2,11 @@
 <xliff xmlns="urn:oasis:names:tc:xliff:document:1.2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" version="1.2" xsi:schemaLocation="urn:oasis:names:tc:xliff:document:1.2 xliff-core-1.2-transitional.xsd">
   <file datatype="xml" source-language="en" target-language="de" original="../PlatformResources.resx">
     <body>
+      <trans-unit id="AbortForMaxFailedTestsDescription">
+        <source>Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted.</source>
+        <target state="new">Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted.</target>
+        <note />
+      </trans-unit>
       <trans-unit id="Aborted">
         <source>Aborted</source>
         <target state="translated">Abgebrochen</target>
@@ -371,6 +376,11 @@
         <target state="translated">Die ILoggerFactory wurde noch nicht erstellt.</target>
         <note />
       </trans-unit>
+      <trans-unit id="MaxFailedTestsMustBePositive">
+        <source>The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid.</source>
+        <target state="new">The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid.</target>
+        <note />
+      </trans-unit>
       <trans-unit id="MessageBusNotReady">
         <source>The message bus has not been built yet or is no more usable at this stage.</source>
         <target state="translated">Der Nachrichtenbus wurde noch nicht erstellt oder kann zu diesem Zeitpunkt nicht mehr verwendet werden.</target>
@@ -525,6 +535,11 @@ Die verfügbaren Werte sind "Trace", "Debug", "Information", "Warning", "Error"
         <target state="translated">Zeigen Sie .NET-Testanwendungsinformationen an.</target>
         <note />
       </trans-unit>
+      <trans-unit id="PlatformCommandLineMaxFailedTestsOptionDescription">
+        <source>Specifies a maximum number of test failures that, when exceeded, will abort the test run.</source>
+        <target state="new">Specifies a maximum number of test failures that, when exceeded, will abort the test run.</target>
+        <note />
+      </trans-unit>
       <trans-unit id="PlatformCommandLineMinimumExpectedTestsIncompatibleDiscoverTests">
         <source>'--list-tests' and '--minimum-expected-tests' are incompatible options</source>
         <target state="translated">"--list-tests" und "--minimum-expected-tests" sind inkompatible Optionen.</target>

src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.es.xlf

@@ -2,6 +2,11 @@
 <xliff xmlns="urn:oasis:names:tc:xliff:document:1.2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" version="1.2" xsi:schemaLocation="urn:oasis:names:tc:xliff:document:1.2 xliff-core-1.2-transitional.xsd">
   <file datatype="xml" source-language="en" target-language="es" original="../PlatformResources.resx">
     <body>
+      <trans-unit id="AbortForMaxFailedTestsDescription">
+        <source>Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted.</source>
+        <target state="new">Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted.</target>
+        <note />
+      </trans-unit>
       <trans-unit id="Aborted">
         <source>Aborted</source>
         <target state="translated">Anulado</target>
@@ -371,6 +376,11 @@
         <target state="translated">ILoggerFactory aún no se ha compilado.</target>
         <note />
       </trans-unit>
+      <trans-unit id="MaxFailedTestsMustBePositive">
+        <source>The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid.</source>
+        <target state="new">The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid.</target>
+        <note />
+      </trans-unit>
       <trans-unit id="MessageBusNotReady">
         <source>The message bus has not been built yet or is no more usable at this stage.</source>
         <target state="translated">El bus de mensajes aún no se ha compilado o ya no se puede usar en esta fase.</target>
@@ -525,6 +535,11 @@ Los valores disponibles son 'Seguimiento', 'Depurar', 'Información', 'Advertenc
         <target state="translated">Muestre la información de la aplicación de prueba de .NET.</target>
         <note />
       </trans-unit>
+      <trans-unit id="PlatformCommandLineMaxFailedTestsOptionDescription">
+        <source>Specifies a maximum number of test failures that, when exceeded, will abort the test run.</source>
+        <target state="new">Specifies a maximum number of test failures that, when exceeded, will abort the test run.</target>
+        <note />
+      </trans-unit>
       <trans-unit id="PlatformCommandLineMinimumExpectedTestsIncompatibleDiscoverTests">
         <source>'--list-tests' and '--minimum-expected-tests' are incompatible options</source>
         <target state="translated">“--list-tests” y “--minimum-expected-tests” son opciones incompatibles</target>

src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.fr.xlf

@@ -2,6 +2,11 @@
 <xliff xmlns="urn:oasis:names:tc:xliff:document:1.2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" version="1.2" xsi:schemaLocation="urn:oasis:names:tc:xliff:document:1.2 xliff-core-1.2-transitional.xsd">
   <file datatype="xml" source-language="en" target-language="fr" original="../PlatformResources.resx">
     <body>
+      <trans-unit id="AbortForMaxFailedTestsDescription">
+        <source>Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted.</source>
+        <target state="new">Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted.</target>
+        <note />
+      </trans-unit>
       <trans-unit id="Aborted">
         <source>Aborted</source>
         <target state="translated">Abandonné</target>
@@ -371,6 +376,11 @@
         <target state="translated">ILoggerFactory n’a pas encore été généré.</target>
         <note />
       </trans-unit>
+      <trans-unit id="MaxFailedTestsMustBePositive">
+        <source>The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid.</source>
+        <target state="new">The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid.</target>
+        <note />
+      </trans-unit>
       <trans-unit id="MessageBusNotReady">
         <source>The message bus has not been built yet or is no more usable at this stage.</source>
         <target state="translated">Le bus de messages n’a pas encore été généré ou n’est plus utilisable à ce stade.</target>
@@ -525,6 +535,11 @@ Les valeurs disponibles sont « Trace », « Debug », « Information »,
         <target state="translated">Afficher les informations de l’application de test .NET.</target>
         <note />
       </trans-unit>
+      <trans-unit id="PlatformCommandLineMaxFailedTestsOptionDescription">
+        <source>Specifies a maximum number of test failures that, when exceeded, will abort the test run.</source>
+        <target state="new">Specifies a maximum number of test failures that, when exceeded, will abort the test run.</target>
+        <note />
+      </trans-unit>
       <trans-unit id="PlatformCommandLineMinimumExpectedTestsIncompatibleDiscoverTests">
         <source>'--list-tests' and '--minimum-expected-tests' are incompatible options</source>
         <target state="translated">« --list-tests » et « --minimum-expected-tests » sont des options incompatibles</target>

src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.it.xlf

@@ -2,6 +2,11 @@
 <xliff xmlns="urn:oasis:names:tc:xliff:document:1.2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" version="1.2" xsi:schemaLocation="urn:oasis:names:tc:xliff:document:1.2 xliff-core-1.2-transitional.xsd">
   <file datatype="xml" source-language="en" target-language="it" original="../PlatformResources.resx">
     <body>
+      <trans-unit id="AbortForMaxFailedTestsDescription">
+        <source>Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted.</source>
+        <target state="new">Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted.</target>
+        <note />
+      </trans-unit>
       <trans-unit id="Aborted">
         <source>Aborted</source>
         <target state="translated">Operazione interrotta</target>
@@ -371,6 +376,11 @@
         <target state="translated">ILoggerFactory non è stato ancora compilato.</target>
         <note />
       </trans-unit>
+      <trans-unit id="MaxFailedTestsMustBePositive">
+        <source>The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid.</source>
+        <target state="new">The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid.</target>
+        <note />
+      </trans-unit>
       <trans-unit id="MessageBusNotReady">
         <source>The message bus has not been built yet or is no more usable at this stage.</source>
         <target state="translated">Il bus di messaggi non è stato ancora compilato o non è più utilizzabile in questa fase.</target>
@@ -525,6 +535,11 @@ I valori disponibili sono 'Trace', 'Debug', 'Information', 'Warning', 'Error' e
         <target state="translated">Visualizza le informazioni sull'applicazione di test .NET.</target>
         <note />
       </trans-unit>
+      <trans-unit id="PlatformCommandLineMaxFailedTestsOptionDescription">
+        <source>Specifies a maximum number of test failures that, when exceeded, will abort the test run.</source>
+        <target state="new">Specifies a maximum number of test failures that, when exceeded, will abort the test run.</target>
+        <note />
+      </trans-unit>
       <trans-unit id="PlatformCommandLineMinimumExpectedTestsIncompatibleDiscoverTests">
         <source>'--list-tests' and '--minimum-expected-tests' are incompatible options</source>
         <target state="translated">'--list-tests' e '--minimum-expected-tests' sono opzioni incompatibili</target>