From 12520ce52c8584dab482b28c5cbf4b089d585cbf Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Wed, 4 Dec 2024 14:51:27 +0100 Subject: [PATCH 01/64] Support '--max-failed-tests' to abort test run when failure threshold is reached --- .../PlatformCommandLineProvider.cs | 11 ++++ .../AbortForMaxFailedTestsExtension.cs | 66 +++++++++++++++++++ .../Hosts/TestHostBuilder.cs | 2 + .../Resources/PlatformResources.resx | 9 +++ .../Resources/xlf/PlatformResources.cs.xlf | 15 +++++ .../Resources/xlf/PlatformResources.de.xlf | 15 +++++ .../Resources/xlf/PlatformResources.es.xlf | 15 +++++ .../Resources/xlf/PlatformResources.fr.xlf | 15 +++++ .../Resources/xlf/PlatformResources.it.xlf | 15 +++++ .../Resources/xlf/PlatformResources.ja.xlf | 15 +++++ .../Resources/xlf/PlatformResources.ko.xlf | 15 +++++ .../Resources/xlf/PlatformResources.pl.xlf | 15 +++++ .../Resources/xlf/PlatformResources.pt-BR.xlf | 15 +++++ .../Resources/xlf/PlatformResources.ru.xlf | 15 +++++ .../Resources/xlf/PlatformResources.tr.xlf | 15 +++++ .../xlf/PlatformResources.zh-Hans.xlf | 15 +++++ .../xlf/PlatformResources.zh-Hant.xlf | 15 +++++ 17 files changed, 283 insertions(+) create mode 100644 src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs diff --git a/src/Platform/Microsoft.Testing.Platform/CommandLine/PlatformCommandLineProvider.cs b/src/Platform/Microsoft.Testing.Platform/CommandLine/PlatformCommandLineProvider.cs index ba26ab2f22..c5a53d473a 100644 --- a/src/Platform/Microsoft.Testing.Platform/CommandLine/PlatformCommandLineProvider.cs +++ b/src/Platform/Microsoft.Testing.Platform/CommandLine/PlatformCommandLineProvider.cs @@ -32,6 +32,7 @@ internal sealed class PlatformCommandLineProvider : ICommandLineOptionsProvider public const string TestHostControllerPIDOptionKey = "internal-testhostcontroller-pid"; public const string ExitOnProcessExitOptionKey = "exit-on-process-exit"; public const string ConfigFileOptionKey = "config-file"; + public const string MaxFailedTestsOptionKey = "max-failed-tests"; public const string ServerOptionKey = "server"; public const string ClientPortOptionKey = "client-port"; @@ -61,6 +62,7 @@ internal sealed class PlatformCommandLineProvider : ICommandLineOptionsProvider new(IgnoreExitCodeOptionKey, PlatformResources.PlatformCommandLineIgnoreExitCodeOptionDescription, ArgumentArity.ExactlyOne, false, isBuiltIn: true), new(ExitOnProcessExitOptionKey, PlatformResources.PlatformCommandLineExitOnProcessExitOptionDescription, ArgumentArity.ExactlyOne, false, isBuiltIn: true), new(ConfigFileOptionKey, PlatformResources.PlatformCommandLineConfigFileOptionDescription, ArgumentArity.ExactlyOne, false, isBuiltIn: true), + new(MaxFailedTestsOptionKey, PlatformResources.PlatformCommandLineMaxFailedTestsOptionDescription, ArgumentArity.ExactlyOne, false, isBuiltIn: true), // Hidden options new(HelpOptionQuestionMark, PlatformResources.PlatformCommandLineHelpOptionDescription, ArgumentArity.Zero, true, isBuiltIn: true), @@ -141,6 +143,15 @@ public Task ValidateOptionArgumentsAsync(CommandLineOption com } } + if (commandOption.Name == MaxFailedTestsOptionKey) + { + string arg = arguments[0]; + if (!int.TryParse(arg, out int maxFailedTestsResult) || maxFailedTestsResult <= 0) + { + return ValidationResult.InvalidTask(string.Format(CultureInfo.InvariantCulture, PlatformResources.MaxFailedTestsMustBePositive, arg)); + } + } + // Now validate the minimum expected tests option return IsMinimumExpectedTestsOptionValidAsync(commandOption, arguments); } diff --git a/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs b/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs new file mode 100644 index 0000000000..9603913cb3 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs @@ -0,0 +1,66 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Platform.CommandLine; +using Microsoft.Testing.Platform.Extensions.Messages; +using Microsoft.Testing.Platform.Extensions.TestHost; +using Microsoft.Testing.Platform.Helpers; +using Microsoft.Testing.Platform.Resources; +using Microsoft.Testing.Platform.Services; + +namespace Microsoft.Testing.Platform.Extensions; + +internal sealed class AbortForMaxFailedTestsExtension : IDataConsumer +{ + private readonly IServiceProvider _serviceProvider; + private readonly int? _maxFailedTests; + + private int _failCount; + + public AbortForMaxFailedTestsExtension(IServiceProvider serviceProvider) + { + _serviceProvider = serviceProvider; + if (serviceProvider.GetCommandLineOptions().TryGetOptionArgumentList(PlatformCommandLineProvider.MaxFailedTestsOptionKey, out string[]? args) && + int.TryParse(args[0], out int maxFailedTests) && + maxFailedTests > 0) + { + _maxFailedTests = maxFailedTests; + } + } + + public Type[] DataTypesConsumed { get; } = [typeof(TestNodeUpdateMessage)]; + + /// + public string Uid { get; } = nameof(AbortForMaxFailedTestsExtension); + + /// + public string Version { get; } = AppVersion.DefaultSemVer; + + /// + public string DisplayName { get; } = nameof(AbortForMaxFailedTestsExtension); + + /// + public string Description { get; } = PlatformResources.AbortForMaxFailedTestsDescription; + + /// + public Task IsEnabledAsync() => Task.FromResult(_maxFailedTests.HasValue); + + public Task ConsumeAsync(IDataProducer dataProducer, IData value, CancellationToken cancellationToken) + { + var node = (TestNodeUpdateMessage)value; + + // If we are called, the extension is enabled, which means _maxFailedTests.HasValue was true. So null suppression is safe. + int maxFailed = _maxFailedTests!.Value; + if (node.TestNode.Properties.Single() is FailedTestNodeStateProperty) + { + Interlocked.Increment(ref _failCount); + } + + if (_failCount > maxFailed) + { + _serviceProvider.GetTestApplicationCancellationTokenSource().Cancel(); + } + + return Task.CompletedTask; + } +} diff --git a/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs b/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs index bc0a2184df..7d3e8b5cfd 100644 --- a/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs +++ b/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs @@ -236,6 +236,8 @@ public async Task BuildAsync( serviceProvider.TryAddService(proxyOutputDevice); serviceProvider.TryAddService(proxyOutputDevice.OriginalOutputDevice); + TestHost.AddDataConsumer(serviceProvider => new AbortForMaxFailedTestsExtension(serviceProvider)); + // Create the test framework capabilities ITestFrameworkCapabilities testFrameworkCapabilities = TestFramework.TestFrameworkCapabilitiesFactory(serviceProvider); if (testFrameworkCapabilities is IAsyncInitializableExtension testFrameworkCapabilitiesAsyncInitializable) diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/PlatformResources.resx b/src/Platform/Microsoft.Testing.Platform/Resources/PlatformResources.resx index 3ae6c7d53f..0affd9d0a7 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/PlatformResources.resx +++ b/src/Platform/Microsoft.Testing.Platform/Resources/PlatformResources.resx @@ -694,4 +694,13 @@ Takes one argument as string in the format <value>[h|m|s] where 'value' is Exception during the cancellation of request id '{0}' {0} is the request id + + Specifies a maximum number of test failures that, when exceeded, will abort the test run. + + + The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid. + + + Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + \ No newline at end of file diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.cs.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.cs.xlf index 4526c31089..1959039899 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.cs.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.cs.xlf @@ -2,6 +2,11 @@ + + Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + + Aborted Přerušeno @@ -371,6 +376,11 @@ Rozhraní ILoggerFactory ještě nebylo sestaveno. + + The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid. + The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid. + + The message bus has not been built yet or is no more usable at this stage. Sběrnice zpráv ještě nebyla sestavena nebo už není v této fázi použitelná. @@ -525,6 +535,11 @@ Dostupné hodnoty jsou Trace, Debug, Information, Warning, Error a Critical.Umožňuje zobrazit informace o testovací aplikaci .NET. + + Specifies a maximum number of test failures that, when exceeded, will abort the test run. + Specifies a maximum number of test failures that, when exceeded, will abort the test run. + + '--list-tests' and '--minimum-expected-tests' are incompatible options --list-tests a --minimum-expected-tests jsou nekompatibilní možnosti. diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.de.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.de.xlf index ce0825be7a..a7906fbb04 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.de.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.de.xlf @@ -2,6 +2,11 @@ + + Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + + Aborted Abgebrochen @@ -371,6 +376,11 @@ Die ILoggerFactory wurde noch nicht erstellt. + + The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid. + The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid. + + The message bus has not been built yet or is no more usable at this stage. Der Nachrichtenbus wurde noch nicht erstellt oder kann zu diesem Zeitpunkt nicht mehr verwendet werden. @@ -525,6 +535,11 @@ Die verfügbaren Werte sind "Trace", "Debug", "Information", "Warning", "Error" Zeigen Sie .NET-Testanwendungsinformationen an. + + Specifies a maximum number of test failures that, when exceeded, will abort the test run. + Specifies a maximum number of test failures that, when exceeded, will abort the test run. + + '--list-tests' and '--minimum-expected-tests' are incompatible options "--list-tests" und "--minimum-expected-tests" sind inkompatible Optionen. diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.es.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.es.xlf index 095d36bf98..ef4adb82cf 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.es.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.es.xlf @@ -2,6 +2,11 @@ + + Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + + Aborted Anulado @@ -371,6 +376,11 @@ ILoggerFactory aún no se ha compilado. + + The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid. + The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid. + + The message bus has not been built yet or is no more usable at this stage. El bus de mensajes aún no se ha compilado o ya no se puede usar en esta fase. @@ -525,6 +535,11 @@ Los valores disponibles son 'Seguimiento', 'Depurar', 'Información', 'Advertenc Muestre la información de la aplicación de prueba de .NET. + + Specifies a maximum number of test failures that, when exceeded, will abort the test run. + Specifies a maximum number of test failures that, when exceeded, will abort the test run. + + '--list-tests' and '--minimum-expected-tests' are incompatible options “--list-tests” y “--minimum-expected-tests” son opciones incompatibles diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.fr.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.fr.xlf index 2b23f3d009..3c76f4bf51 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.fr.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.fr.xlf @@ -2,6 +2,11 @@ + + Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + + Aborted Abandonné @@ -371,6 +376,11 @@ ILoggerFactory n’a pas encore été généré. + + The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid. + The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid. + + The message bus has not been built yet or is no more usable at this stage. Le bus de messages n’a pas encore été généré ou n’est plus utilisable à ce stade. @@ -525,6 +535,11 @@ Les valeurs disponibles sont « Trace », « Debug », « Information », Afficher les informations de l’application de test .NET. + + Specifies a maximum number of test failures that, when exceeded, will abort the test run. + Specifies a maximum number of test failures that, when exceeded, will abort the test run. + + '--list-tests' and '--minimum-expected-tests' are incompatible options « --list-tests » et « --minimum-expected-tests » sont des options incompatibles diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.it.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.it.xlf index f8450b7534..cb5a023a1f 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.it.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.it.xlf @@ -2,6 +2,11 @@ + + Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + + Aborted Operazione interrotta @@ -371,6 +376,11 @@ ILoggerFactory non è stato ancora compilato. + + The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid. + The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid. + + The message bus has not been built yet or is no more usable at this stage. Il bus di messaggi non è stato ancora compilato o non è più utilizzabile in questa fase. @@ -525,6 +535,11 @@ I valori disponibili sono 'Trace', 'Debug', 'Information', 'Warning', 'Error' e Visualizza le informazioni sull'applicazione di test .NET. + + Specifies a maximum number of test failures that, when exceeded, will abort the test run. + Specifies a maximum number of test failures that, when exceeded, will abort the test run. + + '--list-tests' and '--minimum-expected-tests' are incompatible options '--list-tests' e '--minimum-expected-tests' sono opzioni incompatibili diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ja.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ja.xlf index 6411ac9405..5580e899b0 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ja.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ja.xlf @@ -2,6 +2,11 @@ + + Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + + Aborted 中止されました @@ -371,6 +376,11 @@ ILoggerFactory はまだ構築されていません。 + + The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid. + The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid. + + The message bus has not been built yet or is no more usable at this stage. メッセージ バスはまだ構築されていないか、この段階ではこれ以上使用できません。 @@ -526,6 +536,11 @@ The available values are 'Trace', 'Debug', 'Information', 'Warning', 'Error', an .NET テスト アプリケーション情報を表示します。 + + Specifies a maximum number of test failures that, when exceeded, will abort the test run. + Specifies a maximum number of test failures that, when exceeded, will abort the test run. + + '--list-tests' and '--minimum-expected-tests' are incompatible options '--list-tests' と '--minimum-expected-tests' は互換性のないオプションです diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ko.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ko.xlf index 73ab4a51df..9b78b8eb14 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ko.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ko.xlf @@ -2,6 +2,11 @@ + + Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + + Aborted 중단됨 @@ -371,6 +376,11 @@ ILoggerFactory가 아직 빌드되지 않았습니다. + + The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid. + The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid. + + The message bus has not been built yet or is no more usable at this stage. 메시지 버스가 아직 빌드되지 않았거나 이 단계에서 더 이상 사용할 수 없습니다. @@ -525,6 +535,11 @@ The available values are 'Trace', 'Debug', 'Information', 'Warning', 'Error', an .NET 테스트 애플리케이션 정보를 표시합니다. + + Specifies a maximum number of test failures that, when exceeded, will abort the test run. + Specifies a maximum number of test failures that, when exceeded, will abort the test run. + + '--list-tests' and '--minimum-expected-tests' are incompatible options '--list-tests' 및 '--minimum-expected-tests'는 호환되지 않는 옵션입니다. diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pl.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pl.xlf index 127548414d..65f16e7a06 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pl.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pl.xlf @@ -2,6 +2,11 @@ + + Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + + Aborted Przerwano @@ -371,6 +376,11 @@ Obiekt ILoggerFactory nie został jeszcze skompilowany. + + The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid. + The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid. + + The message bus has not been built yet or is no more usable at this stage. Magistrala komunikatów nie została jeszcze zbudowana lub nie można jej już na tym etapie użyteczna. @@ -525,6 +535,11 @@ Dostępne wartości to „Trace”, „Debug”, „Information”, „Warning Wyświetl informacje o aplikacji testowej platformy .NET. + + Specifies a maximum number of test failures that, when exceeded, will abort the test run. + Specifies a maximum number of test failures that, when exceeded, will abort the test run. + + '--list-tests' and '--minimum-expected-tests' are incompatible options Opcje „--list-tests” i „--minimum-expected-tests” są niezgodne diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pt-BR.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pt-BR.xlf index d303930104..648a1a36bd 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pt-BR.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pt-BR.xlf @@ -2,6 +2,11 @@ + + Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + + Aborted Anulado @@ -371,6 +376,11 @@ O ILoggerFactory ainda não foi criado. + + The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid. + The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid. + + The message bus has not been built yet or is no more usable at this stage. O barramento de mensagens ainda não foi criado ou não pode mais ser usado nesse estágio. @@ -525,6 +535,11 @@ Os valores disponíveis são 'Rastreamento', 'Depuração', 'Informação', 'Avi Exibir informações do aplicativo de teste do .NET. + + Specifies a maximum number of test failures that, when exceeded, will abort the test run. + Specifies a maximum number of test failures that, when exceeded, will abort the test run. + + '--list-tests' and '--minimum-expected-tests' are incompatible options '--list-tests' e '--minimum-expected-tests' são opções incompatíveis diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ru.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ru.xlf index 42539f5b67..40bdb84fa2 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ru.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ru.xlf @@ -2,6 +2,11 @@ + + Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + + Aborted Прервано @@ -371,6 +376,11 @@ Параметр ILoggerFactory еще не создан. + + The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid. + The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid. + + The message bus has not been built yet or is no more usable at this stage. Шина сообщений еще не построена или на данном этапе непригодна для использования. @@ -525,6 +535,11 @@ The available values are 'Trace', 'Debug', 'Information', 'Warning', 'Error', an Отображение сведений о тестовом приложении .NET. + + Specifies a maximum number of test failures that, when exceeded, will abort the test run. + Specifies a maximum number of test failures that, when exceeded, will abort the test run. + + '--list-tests' and '--minimum-expected-tests' are incompatible options Параметры "--list-tests" и "--minimum-expected-tests" несовместимы diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.tr.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.tr.xlf index 91326a0e25..88cad048df 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.tr.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.tr.xlf @@ -2,6 +2,11 @@ + + Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + + Aborted Durduruldu @@ -371,6 +376,11 @@ ILoggerFactory henüz derlenmedi. + + The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid. + The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid. + + The message bus has not been built yet or is no more usable at this stage. İleti veri yolu henüz derlenmedi veya bu aşamada artık kullanılamıyor. @@ -525,6 +535,11 @@ Kullanılabilir değerler: 'Trace', 'Debug', 'Information', 'Warning', 'Error' v .NET test uygulaması bilgilerini görüntüler. + + Specifies a maximum number of test failures that, when exceeded, will abort the test run. + Specifies a maximum number of test failures that, when exceeded, will abort the test run. + + '--list-tests' and '--minimum-expected-tests' are incompatible options '--list-tests' ve '--minimum-expected-tests' uyumsuz seçenekler diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hans.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hans.xlf index 3ee2239188..6adce85349 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hans.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hans.xlf @@ -2,6 +2,11 @@ + + Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + + Aborted 已中止 @@ -371,6 +376,11 @@ 尚未生成 ILoggerFactory。 + + The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid. + The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid. + + The message bus has not been built yet or is no more usable at this stage. 消息总线尚未生成或在此阶段不再可用。 @@ -525,6 +535,11 @@ The available values are 'Trace', 'Debug', 'Information', 'Warning', 'Error', an 显示 .NET 测试应用程序信息。 + + Specifies a maximum number of test failures that, when exceeded, will abort the test run. + Specifies a maximum number of test failures that, when exceeded, will abort the test run. + + '--list-tests' and '--minimum-expected-tests' are incompatible options “--list-tests”和“--minimum-expected-tests”是不兼容的选项 diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hant.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hant.xlf index 80a74a1d82..dd79c64866 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hant.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hant.xlf @@ -2,6 +2,11 @@ + + Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + + Aborted 已中止 @@ -371,6 +376,11 @@ 尚未建置 ILoggerFactory。 + + The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid. + The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid. + + The message bus has not been built yet or is no more usable at this stage. 訊息匯流排尚未建置或在此階段無法再使用。 @@ -525,6 +535,11 @@ The available values are 'Trace', 'Debug', 'Information', 'Warning', 'Error', an 顯示 .NET 測試應用程式資訊。 + + Specifies a maximum number of test failures that, when exceeded, will abort the test run. + Specifies a maximum number of test failures that, when exceeded, will abort the test run. + + '--list-tests' and '--minimum-expected-tests' are incompatible options '--list-tests' 和 '--minimum-expected-tests' 是不相容的選項 From dd480c2ddf0f8198735b6563ae3f15795adafb49 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Wed, 4 Dec 2024 15:05:50 +0100 Subject: [PATCH 02/64] Address review comment --- .../Extensions/AbortForMaxFailedTestsExtension.cs | 12 +++++++----- .../Hosts/TestHostBuilder.cs | 5 ++++- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs b/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs index 9603913cb3..7b670f64e9 100644 --- a/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs +++ b/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +using System.Threading; + using Microsoft.Testing.Platform.CommandLine; using Microsoft.Testing.Platform.Extensions.Messages; using Microsoft.Testing.Platform.Extensions.TestHost; @@ -12,15 +14,15 @@ namespace Microsoft.Testing.Platform.Extensions; internal sealed class AbortForMaxFailedTestsExtension : IDataConsumer { - private readonly IServiceProvider _serviceProvider; + private readonly ITestApplicationCancellationTokenSource _cancellationTokenSource; private readonly int? _maxFailedTests; private int _failCount; - public AbortForMaxFailedTestsExtension(IServiceProvider serviceProvider) + public AbortForMaxFailedTestsExtension(ICommandLineOptions commandLineOptions, ITestApplicationCancellationTokenSource cancellationTokenSource) { - _serviceProvider = serviceProvider; - if (serviceProvider.GetCommandLineOptions().TryGetOptionArgumentList(PlatformCommandLineProvider.MaxFailedTestsOptionKey, out string[]? args) && + _cancellationTokenSource = cancellationTokenSource; + if (commandLineOptions.TryGetOptionArgumentList(PlatformCommandLineProvider.MaxFailedTestsOptionKey, out string[]? args) && int.TryParse(args[0], out int maxFailedTests) && maxFailedTests > 0) { @@ -58,7 +60,7 @@ public Task ConsumeAsync(IDataProducer dataProducer, IData value, CancellationTo if (_failCount > maxFailed) { - _serviceProvider.GetTestApplicationCancellationTokenSource().Cancel(); + _cancellationTokenSource.Cancel(); } return Task.CompletedTask; diff --git a/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs b/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs index 7d3e8b5cfd..2f41656d06 100644 --- a/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs +++ b/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs @@ -236,7 +236,10 @@ public async Task BuildAsync( serviceProvider.TryAddService(proxyOutputDevice); serviceProvider.TryAddService(proxyOutputDevice.OriginalOutputDevice); - TestHost.AddDataConsumer(serviceProvider => new AbortForMaxFailedTestsExtension(serviceProvider)); + TestHost.AddDataConsumer( + serviceProvider => new AbortForMaxFailedTestsExtension( + serviceProvider.GetCommandLineOptions(), + serviceProvider.GetTestApplicationCancellationTokenSource())); // Create the test framework capabilities ITestFrameworkCapabilities testFrameworkCapabilities = TestFramework.TestFrameworkCapabilitiesFactory(serviceProvider); From e9a4dd8e519fde578877a6cceb44414f86575ea9 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Wed, 4 Dec 2024 15:13:36 +0100 Subject: [PATCH 03/64] Remove unused using --- .../Extensions/AbortForMaxFailedTestsExtension.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs b/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs index 7b670f64e9..bb6a7c4a30 100644 --- a/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs +++ b/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs @@ -1,8 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using System.Threading; - using Microsoft.Testing.Platform.CommandLine; using Microsoft.Testing.Platform.Extensions.Messages; using Microsoft.Testing.Platform.Extensions.TestHost; From a9e7d32acd898273483b34f3f7bdd2d2618661c6 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Wed, 4 Dec 2024 16:49:42 +0100 Subject: [PATCH 04/64] Progress --- .../Extensions/AbortForMaxFailedTestsExtension.cs | 8 ++------ .../HelpInfoTests.cs | 2 ++ .../HelpInfoTests.cs | 12 ++++++++++++ 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs b/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs index bb6a7c4a30..a6c4d369f9 100644 --- a/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs +++ b/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs @@ -51,12 +51,8 @@ public Task ConsumeAsync(IDataProducer dataProducer, IData value, CancellationTo // If we are called, the extension is enabled, which means _maxFailedTests.HasValue was true. So null suppression is safe. int maxFailed = _maxFailedTests!.Value; - if (node.TestNode.Properties.Single() is FailedTestNodeStateProperty) - { - Interlocked.Increment(ref _failCount); - } - - if (_failCount > maxFailed) + if (node.TestNode.Properties.Single() is FailedTestNodeStateProperty && + Interlocked.Increment(ref _failCount) > maxFailed) { _cancellationTokenSource.Cancel(); } diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/HelpInfoTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/HelpInfoTests.cs index 8894ffbc2d..e2aeac66b3 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/HelpInfoTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/HelpInfoTests.cs @@ -59,6 +59,8 @@ Show the command line help. Display .NET test application information. --list-tests List available tests. + --max-failed-tests + Specifies a maximum number of test failures that, when exceeded, will abort the test run. --minimum-expected-tests Specifies the minimum number of tests that are expected to run. --results-directory diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpInfoTests.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpInfoTests.cs index 1f66b97679..40c1c822ff 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpInfoTests.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpInfoTests.cs @@ -55,6 +55,8 @@ Show the command line help. Display .NET test application information. --list-tests List available tests. + --max-failed-tests + Specifies a maximum number of test failures that, when exceeded, will abort the test run. --minimum-expected-tests Specifies the minimum number of tests that are expected to run. --results-directory @@ -218,6 +220,10 @@ Note that this is slowing down the test execution\. Arity: 0 Hidden: False Description: List available tests\. + --max-failed-tests + Arity: 1 + Hidden: False + Description: Specifies a maximum number of test failures that, when exceeded, will abort the test run\. --minimum-expected-tests Arity: 0\.\.1 Hidden: False @@ -317,6 +323,8 @@ Show the command line help. Display .NET test application information. --list-tests List available tests. + --max-failed-tests + Specifies a maximum number of test failures that, when exceeded, will abort the test run. --minimum-expected-tests Specifies the minimum number of tests that are expected to run. --results-directory @@ -492,6 +500,10 @@ Note that this is slowing down the test execution. Arity: 0 Hidden: False Description: List available tests. + --max-failed-tests + Arity: 1 + Hidden: False + Description: Specifies a maximum number of test failures that, when exceeded, will abort the test run. --minimum-expected-tests Arity: 0..1 Hidden: False From f37ea380c9ba2617e7e1ad944e47554ab2fbb5ad Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Wed, 4 Dec 2024 16:50:29 +0100 Subject: [PATCH 05/64] Not ideal, and not working even. Keeping in history though --- .../Hosts/CommonTestHost.cs | 11 +++++++++++ .../Messages/AsynchronousMessageBus.cs | 4 ++-- .../Messages/BaseMessageBus.cs | 2 +- .../Messages/ChannelConsumerDataProcessor.cs | 11 ++++------- .../Messages/ListTestsMessageBus.cs | 2 +- .../Messages/MessageBusProxy.cs | 4 ++-- 6 files changed, 21 insertions(+), 13 deletions(-) diff --git a/src/Platform/Microsoft.Testing.Platform/Hosts/CommonTestHost.cs b/src/Platform/Microsoft.Testing.Platform/Hosts/CommonTestHost.cs index 0e146d7e9c..ce63c751bd 100644 --- a/src/Platform/Microsoft.Testing.Platform/Hosts/CommonTestHost.cs +++ b/src/Platform/Microsoft.Testing.Platform/Hosts/CommonTestHost.cs @@ -35,6 +35,11 @@ public async Task RunAsync() if (testApplicationCancellationToken.IsCancellationRequested) { + if (ServiceProvider.GetBaseMessageBus() is BaseMessageBus baseMessageBus) + { + await baseMessageBus.DrainDataAsync(forceIfCancelled: true); + } + exitCode = ExitCodes.TestSessionAborted; } @@ -76,6 +81,12 @@ public async Task RunAsync() // await DisposeHelper.DisposeAsync(ServiceProvider.GetTestApplicationCancellationTokenSource()); } + IPlatformOutputDevice outputDevice = ServiceProvider.GetPlatformOutputDevice(); + if (outputDevice is IDataConsumer outputDeviceConsumer) + { + // ServiceProvider.GetMessageBus(); + } + if (testApplicationCancellationToken.IsCancellationRequested) { exitCode = ExitCodes.TestSessionAborted; diff --git a/src/Platform/Microsoft.Testing.Platform/Messages/AsynchronousMessageBus.cs b/src/Platform/Microsoft.Testing.Platform/Messages/AsynchronousMessageBus.cs index baede6aec5..95bf77cb58 100644 --- a/src/Platform/Microsoft.Testing.Platform/Messages/AsynchronousMessageBus.cs +++ b/src/Platform/Microsoft.Testing.Platform/Messages/AsynchronousMessageBus.cs @@ -131,7 +131,7 @@ private async Task LogDataAsync(IDataProducer dataProducer, IData data) await _logger.LogTraceAsync(messageBuilder.ToString()); } - public override async Task DrainDataAsync() + public override async Task DrainDataAsync(bool forceIfCancelled = false) { Dictionary consumerToDrain = []; bool anotherRound = true; @@ -145,7 +145,7 @@ public override async Task DrainDataAsync() CancellationToken cancellationToken = _testApplicationCancellationTokenSource.CancellationToken; while (anotherRound) { - if (cancellationToken.IsCancellationRequested) + if (!forceIfCancelled && cancellationToken.IsCancellationRequested) { return; } diff --git a/src/Platform/Microsoft.Testing.Platform/Messages/BaseMessageBus.cs b/src/Platform/Microsoft.Testing.Platform/Messages/BaseMessageBus.cs index fedfdce19b..cf48707787 100644 --- a/src/Platform/Microsoft.Testing.Platform/Messages/BaseMessageBus.cs +++ b/src/Platform/Microsoft.Testing.Platform/Messages/BaseMessageBus.cs @@ -12,7 +12,7 @@ internal abstract class BaseMessageBus : IMessageBus, IDisposable public abstract Task InitAsync(); - public abstract Task DrainDataAsync(); + public abstract Task DrainDataAsync(bool forceIfCancelled = false); public abstract Task DisableAsync(); diff --git a/src/Platform/Microsoft.Testing.Platform/Messages/ChannelConsumerDataProcessor.cs b/src/Platform/Microsoft.Testing.Platform/Messages/ChannelConsumerDataProcessor.cs index 6ef8484b97..59a7f302d7 100644 --- a/src/Platform/Microsoft.Testing.Platform/Messages/ChannelConsumerDataProcessor.cs +++ b/src/Platform/Microsoft.Testing.Platform/Messages/ChannelConsumerDataProcessor.cs @@ -94,7 +94,10 @@ private async Task ConsumeAsync() } catch (OperationCanceledException oc) when (oc.CancellationToken == _cancellationToken) { - // Ignore we're shutting down + // Make sure to drain the data. + // This is important for --max-failed-tests where a cancellation is requested when reaching a specific number of failures. + // We want to drain data before returning so that the terminal is able to print everything. + await DrainDataAsync(); } catch (Exception ex) { @@ -131,12 +134,6 @@ public async Task DrainDataAsync() int currentDelayTimeMs = minDelayTimeMs; while (Interlocked.CompareExchange(ref _totalPayloadReceived, totalPayloadReceived, totalPayloadProcessed) != totalPayloadProcessed) { - // When we cancel we throw inside ConsumeAsync and we won't drain anymore any data - if (_cancellationToken.IsCancellationRequested) - { - break; - } - await _task.Delay(currentDelayTimeMs); currentDelayTimeMs = Math.Min(currentDelayTimeMs + minDelayTimeMs, 200); diff --git a/src/Platform/Microsoft.Testing.Platform/Messages/ListTestsMessageBus.cs b/src/Platform/Microsoft.Testing.Platform/Messages/ListTestsMessageBus.cs index a2b54245d8..4cb76ddaf0 100644 --- a/src/Platform/Microsoft.Testing.Platform/Messages/ListTestsMessageBus.cs +++ b/src/Platform/Microsoft.Testing.Platform/Messages/ListTestsMessageBus.cs @@ -53,7 +53,7 @@ public override void Dispose() { } - public override Task DrainDataAsync() => Task.CompletedTask; + public override Task DrainDataAsync(bool forceIfCancelled = false) => Task.CompletedTask; public override Task InitAsync() => Task.CompletedTask; diff --git a/src/Platform/Microsoft.Testing.Platform/Messages/MessageBusProxy.cs b/src/Platform/Microsoft.Testing.Platform/Messages/MessageBusProxy.cs index 889dcd77c7..f71bbe568f 100644 --- a/src/Platform/Microsoft.Testing.Platform/Messages/MessageBusProxy.cs +++ b/src/Platform/Microsoft.Testing.Platform/Messages/MessageBusProxy.cs @@ -33,10 +33,10 @@ public override async Task PublishAsync(IDataProducer dataProducer, IData data) await _messageBus.PublishAsync(dataProducer, data); } - public override async Task DrainDataAsync() + public override async Task DrainDataAsync(bool forceIfCancelled = false) { EnsureMessageBusAvailable(); - await _messageBus.DrainDataAsync(); + await _messageBus.DrainDataAsync(forceIfCancelled); } public override async Task DisableAsync() From ce36d36eb1202cd8cb8505d73e8f853809eaf5d7 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Wed, 4 Dec 2024 16:59:55 +0100 Subject: [PATCH 06/64] Fix build error --- .../Messages/AsynchronousMessageBusTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Messages/AsynchronousMessageBusTests.cs b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Messages/AsynchronousMessageBusTests.cs index 94af4c458e..2675102edf 100644 --- a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Messages/AsynchronousMessageBusTests.cs +++ b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Messages/AsynchronousMessageBusTests.cs @@ -34,7 +34,7 @@ public async Task UnexpectedTypePublished_ShouldFail() // Fire consume with a good message await proxy.PublishAsync(new DummyProducer("DummyProducer", typeof(InvalidTypePublished.ValidDataToProduce)), new InvalidTypePublished.ValidDataToProduce()); consumer.Published.WaitOne(TimeoutHelper.DefaultHangTimeoutMilliseconds); - await Assert.ThrowsAsync(proxy.DrainDataAsync); + await Assert.ThrowsAsync(() => proxy.DrainDataAsync()); } public async Task DrainDataAsync_Loop_ShouldFail() From cb1d50a6de0beff0579b47cbcb9bc7894d3532cd Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Thu, 5 Dec 2024 11:18:56 +0100 Subject: [PATCH 07/64] Revert --- .../Hosts/CommonTestHost.cs | 11 ----------- .../Messages/AsynchronousMessageBus.cs | 4 ++-- .../Messages/BaseMessageBus.cs | 2 +- .../Messages/ChannelConsumerDataProcessor.cs | 11 +++++++---- .../Messages/ListTestsMessageBus.cs | 2 +- .../Messages/MessageBusProxy.cs | 4 ++-- .../Messages/AsynchronousMessageBusTests.cs | 2 +- 7 files changed, 14 insertions(+), 22 deletions(-) diff --git a/src/Platform/Microsoft.Testing.Platform/Hosts/CommonTestHost.cs b/src/Platform/Microsoft.Testing.Platform/Hosts/CommonTestHost.cs index ce63c751bd..0e146d7e9c 100644 --- a/src/Platform/Microsoft.Testing.Platform/Hosts/CommonTestHost.cs +++ b/src/Platform/Microsoft.Testing.Platform/Hosts/CommonTestHost.cs @@ -35,11 +35,6 @@ public async Task RunAsync() if (testApplicationCancellationToken.IsCancellationRequested) { - if (ServiceProvider.GetBaseMessageBus() is BaseMessageBus baseMessageBus) - { - await baseMessageBus.DrainDataAsync(forceIfCancelled: true); - } - exitCode = ExitCodes.TestSessionAborted; } @@ -81,12 +76,6 @@ public async Task RunAsync() // await DisposeHelper.DisposeAsync(ServiceProvider.GetTestApplicationCancellationTokenSource()); } - IPlatformOutputDevice outputDevice = ServiceProvider.GetPlatformOutputDevice(); - if (outputDevice is IDataConsumer outputDeviceConsumer) - { - // ServiceProvider.GetMessageBus(); - } - if (testApplicationCancellationToken.IsCancellationRequested) { exitCode = ExitCodes.TestSessionAborted; diff --git a/src/Platform/Microsoft.Testing.Platform/Messages/AsynchronousMessageBus.cs b/src/Platform/Microsoft.Testing.Platform/Messages/AsynchronousMessageBus.cs index 95bf77cb58..baede6aec5 100644 --- a/src/Platform/Microsoft.Testing.Platform/Messages/AsynchronousMessageBus.cs +++ b/src/Platform/Microsoft.Testing.Platform/Messages/AsynchronousMessageBus.cs @@ -131,7 +131,7 @@ private async Task LogDataAsync(IDataProducer dataProducer, IData data) await _logger.LogTraceAsync(messageBuilder.ToString()); } - public override async Task DrainDataAsync(bool forceIfCancelled = false) + public override async Task DrainDataAsync() { Dictionary consumerToDrain = []; bool anotherRound = true; @@ -145,7 +145,7 @@ public override async Task DrainDataAsync(bool forceIfCancelled = false) CancellationToken cancellationToken = _testApplicationCancellationTokenSource.CancellationToken; while (anotherRound) { - if (!forceIfCancelled && cancellationToken.IsCancellationRequested) + if (cancellationToken.IsCancellationRequested) { return; } diff --git a/src/Platform/Microsoft.Testing.Platform/Messages/BaseMessageBus.cs b/src/Platform/Microsoft.Testing.Platform/Messages/BaseMessageBus.cs index cf48707787..fedfdce19b 100644 --- a/src/Platform/Microsoft.Testing.Platform/Messages/BaseMessageBus.cs +++ b/src/Platform/Microsoft.Testing.Platform/Messages/BaseMessageBus.cs @@ -12,7 +12,7 @@ internal abstract class BaseMessageBus : IMessageBus, IDisposable public abstract Task InitAsync(); - public abstract Task DrainDataAsync(bool forceIfCancelled = false); + public abstract Task DrainDataAsync(); public abstract Task DisableAsync(); diff --git a/src/Platform/Microsoft.Testing.Platform/Messages/ChannelConsumerDataProcessor.cs b/src/Platform/Microsoft.Testing.Platform/Messages/ChannelConsumerDataProcessor.cs index 59a7f302d7..6ef8484b97 100644 --- a/src/Platform/Microsoft.Testing.Platform/Messages/ChannelConsumerDataProcessor.cs +++ b/src/Platform/Microsoft.Testing.Platform/Messages/ChannelConsumerDataProcessor.cs @@ -94,10 +94,7 @@ private async Task ConsumeAsync() } catch (OperationCanceledException oc) when (oc.CancellationToken == _cancellationToken) { - // Make sure to drain the data. - // This is important for --max-failed-tests where a cancellation is requested when reaching a specific number of failures. - // We want to drain data before returning so that the terminal is able to print everything. - await DrainDataAsync(); + // Ignore we're shutting down } catch (Exception ex) { @@ -134,6 +131,12 @@ public async Task DrainDataAsync() int currentDelayTimeMs = minDelayTimeMs; while (Interlocked.CompareExchange(ref _totalPayloadReceived, totalPayloadReceived, totalPayloadProcessed) != totalPayloadProcessed) { + // When we cancel we throw inside ConsumeAsync and we won't drain anymore any data + if (_cancellationToken.IsCancellationRequested) + { + break; + } + await _task.Delay(currentDelayTimeMs); currentDelayTimeMs = Math.Min(currentDelayTimeMs + minDelayTimeMs, 200); diff --git a/src/Platform/Microsoft.Testing.Platform/Messages/ListTestsMessageBus.cs b/src/Platform/Microsoft.Testing.Platform/Messages/ListTestsMessageBus.cs index 4cb76ddaf0..a2b54245d8 100644 --- a/src/Platform/Microsoft.Testing.Platform/Messages/ListTestsMessageBus.cs +++ b/src/Platform/Microsoft.Testing.Platform/Messages/ListTestsMessageBus.cs @@ -53,7 +53,7 @@ public override void Dispose() { } - public override Task DrainDataAsync(bool forceIfCancelled = false) => Task.CompletedTask; + public override Task DrainDataAsync() => Task.CompletedTask; public override Task InitAsync() => Task.CompletedTask; diff --git a/src/Platform/Microsoft.Testing.Platform/Messages/MessageBusProxy.cs b/src/Platform/Microsoft.Testing.Platform/Messages/MessageBusProxy.cs index f71bbe568f..889dcd77c7 100644 --- a/src/Platform/Microsoft.Testing.Platform/Messages/MessageBusProxy.cs +++ b/src/Platform/Microsoft.Testing.Platform/Messages/MessageBusProxy.cs @@ -33,10 +33,10 @@ public override async Task PublishAsync(IDataProducer dataProducer, IData data) await _messageBus.PublishAsync(dataProducer, data); } - public override async Task DrainDataAsync(bool forceIfCancelled = false) + public override async Task DrainDataAsync() { EnsureMessageBusAvailable(); - await _messageBus.DrainDataAsync(forceIfCancelled); + await _messageBus.DrainDataAsync(); } public override async Task DisableAsync() diff --git a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Messages/AsynchronousMessageBusTests.cs b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Messages/AsynchronousMessageBusTests.cs index 2675102edf..94af4c458e 100644 --- a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Messages/AsynchronousMessageBusTests.cs +++ b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Messages/AsynchronousMessageBusTests.cs @@ -34,7 +34,7 @@ public async Task UnexpectedTypePublished_ShouldFail() // Fire consume with a good message await proxy.PublishAsync(new DummyProducer("DummyProducer", typeof(InvalidTypePublished.ValidDataToProduce)), new InvalidTypePublished.ValidDataToProduce()); consumer.Published.WaitOne(TimeoutHelper.DefaultHangTimeoutMilliseconds); - await Assert.ThrowsAsync(() => proxy.DrainDataAsync()); + await Assert.ThrowsAsync(proxy.DrainDataAsync); } public async Task DrainDataAsync_Loop_ShouldFail() From 0717a76eaaea8a7c97461af22f618f54554a5cae Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Thu, 5 Dec 2024 11:30:47 +0100 Subject: [PATCH 08/64] Use a capability --- .../IStopTestExecutionCapability.cs | 16 +++++++++++++++ .../AbortForMaxFailedTestsExtension.cs | 20 +++++++++---------- .../Hosts/TestHostBuilder.cs | 10 +++++----- .../PublicAPI/PublicAPI.Unshipped.txt | 2 ++ 4 files changed, 33 insertions(+), 15 deletions(-) create mode 100644 src/Platform/Microsoft.Testing.Platform/Capabilities/TestFramework/IStopTestExecutionCapability.cs diff --git a/src/Platform/Microsoft.Testing.Platform/Capabilities/TestFramework/IStopTestExecutionCapability.cs b/src/Platform/Microsoft.Testing.Platform/Capabilities/TestFramework/IStopTestExecutionCapability.cs new file mode 100644 index 0000000000..553f55d247 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Platform/Capabilities/TestFramework/IStopTestExecutionCapability.cs @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.Testing.Platform.Capabilities.TestFramework; + +/// +/// A capability to support stopping test execution gracefully, without cancelling/aborting everything. +/// This is used to support '--max-failed-tests'. +/// +/// +/// Test frameworks can choose to run any needed cleanup when cancellation is requested. +/// +public interface IStopTestExecutionCapability : ITestFrameworkCapability +{ + public CancellationTokenSource CancellationTokenSource { get; } +} diff --git a/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs b/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs index a6c4d369f9..51b5d47bb5 100644 --- a/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs +++ b/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs @@ -6,18 +6,17 @@ using Microsoft.Testing.Platform.Extensions.TestHost; using Microsoft.Testing.Platform.Helpers; using Microsoft.Testing.Platform.Resources; -using Microsoft.Testing.Platform.Services; namespace Microsoft.Testing.Platform.Extensions; internal sealed class AbortForMaxFailedTestsExtension : IDataConsumer { - private readonly ITestApplicationCancellationTokenSource _cancellationTokenSource; + private readonly CancellationTokenSource? _cancellationTokenSource; private readonly int? _maxFailedTests; private int _failCount; - public AbortForMaxFailedTestsExtension(ICommandLineOptions commandLineOptions, ITestApplicationCancellationTokenSource cancellationTokenSource) + public AbortForMaxFailedTestsExtension(ICommandLineOptions commandLineOptions, CancellationTokenSource? cancellationTokenSource) { _cancellationTokenSource = cancellationTokenSource; if (commandLineOptions.TryGetOptionArgumentList(PlatformCommandLineProvider.MaxFailedTestsOptionKey, out string[]? args) && @@ -43,20 +42,21 @@ public AbortForMaxFailedTestsExtension(ICommandLineOptions commandLineOptions, I public string Description { get; } = PlatformResources.AbortForMaxFailedTestsDescription; /// - public Task IsEnabledAsync() => Task.FromResult(_maxFailedTests.HasValue); + public Task IsEnabledAsync() => Task.FromResult(_maxFailedTests.HasValue && _cancellationTokenSource is not null); - public Task ConsumeAsync(IDataProducer dataProducer, IData value, CancellationToken cancellationToken) + public async Task ConsumeAsync(IDataProducer dataProducer, IData value, CancellationToken cancellationToken) { var node = (TestNodeUpdateMessage)value; - // If we are called, the extension is enabled, which means _maxFailedTests.HasValue was true. So null suppression is safe. - int maxFailed = _maxFailedTests!.Value; + // If we are called, the extension is enabled, which means both _maxFailedTests and _cancellationTokenSource are not null. + RoslynDebug.Assert(_maxFailedTests is not null); + RoslynDebug.Assert(_cancellationTokenSource is not null); + + int maxFailed = _maxFailedTests.Value; if (node.TestNode.Properties.Single() is FailedTestNodeStateProperty && Interlocked.Increment(ref _failCount) > maxFailed) { - _cancellationTokenSource.Cancel(); + await _cancellationTokenSource.CancelAsync(); } - - return Task.CompletedTask; } } diff --git a/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs b/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs index 2f41656d06..d45e6a9d30 100644 --- a/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs +++ b/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs @@ -236,11 +236,6 @@ public async Task BuildAsync( serviceProvider.TryAddService(proxyOutputDevice); serviceProvider.TryAddService(proxyOutputDevice.OriginalOutputDevice); - TestHost.AddDataConsumer( - serviceProvider => new AbortForMaxFailedTestsExtension( - serviceProvider.GetCommandLineOptions(), - serviceProvider.GetTestApplicationCancellationTokenSource())); - // Create the test framework capabilities ITestFrameworkCapabilities testFrameworkCapabilities = TestFramework.TestFrameworkCapabilitiesFactory(serviceProvider); if (testFrameworkCapabilities is IAsyncInitializableExtension testFrameworkCapabilitiesAsyncInitializable) @@ -248,6 +243,11 @@ public async Task BuildAsync( await testFrameworkCapabilitiesAsyncInitializable.InitializeAsync(); } + TestHost.AddDataConsumer( + serviceProvider => new AbortForMaxFailedTestsExtension( + serviceProvider.GetCommandLineOptions(), + serviceProvider.GetTestFrameworkCapabilities().GetCapability()?.CancellationTokenSource)); + // If command line is not valid we return immediately. ValidationResult commandLineValidationResult = await CommandLineOptionsValidator.ValidateAsync( loggingState.CommandLineParseResult, diff --git a/src/Platform/Microsoft.Testing.Platform/PublicAPI/PublicAPI.Unshipped.txt b/src/Platform/Microsoft.Testing.Platform/PublicAPI/PublicAPI.Unshipped.txt index 3976e688ef..147bda8aba 100644 --- a/src/Platform/Microsoft.Testing.Platform/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/Platform/Microsoft.Testing.Platform/PublicAPI/PublicAPI.Unshipped.txt @@ -1,4 +1,6 @@ #nullable enable +Microsoft.Testing.Platform.Capabilities.TestFramework.IStopTestExecutionCapability +Microsoft.Testing.Platform.Capabilities.TestFramework.IStopTestExecutionCapability.CancellationTokenSource.get -> System.Threading.CancellationTokenSource! Microsoft.Testing.Platform.Extensions.Messages.TestMetadataProperty.TestMetadataProperty(string! key) -> void Microsoft.Testing.Platform.OutputDevice.ErrorMessageOutputDeviceData Microsoft.Testing.Platform.OutputDevice.ErrorMessageOutputDeviceData.ErrorMessageOutputDeviceData(string! message) -> void From bd638031b1897a7e1c9990c988864e47c0c03f36 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Thu, 5 Dec 2024 13:14:03 +0100 Subject: [PATCH 09/64] Take a different approach --- .../IStopTestExecutionCapability.cs | 2 +- .../AbortForMaxFailedTestsExtension.cs | 17 ++++++++++------- .../Hosts/TestHostBuilder.cs | 3 ++- .../PublicAPI/PublicAPI.Unshipped.txt | 2 +- 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/Platform/Microsoft.Testing.Platform/Capabilities/TestFramework/IStopTestExecutionCapability.cs b/src/Platform/Microsoft.Testing.Platform/Capabilities/TestFramework/IStopTestExecutionCapability.cs index 553f55d247..7eae7e4d27 100644 --- a/src/Platform/Microsoft.Testing.Platform/Capabilities/TestFramework/IStopTestExecutionCapability.cs +++ b/src/Platform/Microsoft.Testing.Platform/Capabilities/TestFramework/IStopTestExecutionCapability.cs @@ -12,5 +12,5 @@ namespace Microsoft.Testing.Platform.Capabilities.TestFramework; /// public interface IStopTestExecutionCapability : ITestFrameworkCapability { - public CancellationTokenSource CancellationTokenSource { get; } + Task StopTestExecutionAsync(CancellationToken cancellationToken); } diff --git a/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs b/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs index 51b5d47bb5..b8cde4e982 100644 --- a/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs +++ b/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +using Microsoft.Testing.Platform.Capabilities.TestFramework; using Microsoft.Testing.Platform.CommandLine; using Microsoft.Testing.Platform.Extensions.Messages; using Microsoft.Testing.Platform.Extensions.TestHost; @@ -11,20 +12,22 @@ namespace Microsoft.Testing.Platform.Extensions; internal sealed class AbortForMaxFailedTestsExtension : IDataConsumer { - private readonly CancellationTokenSource? _cancellationTokenSource; private readonly int? _maxFailedTests; - + private readonly IStopTestExecutionCapability? _capability; + private readonly CancellationToken _cancellationToken; private int _failCount; - public AbortForMaxFailedTestsExtension(ICommandLineOptions commandLineOptions, CancellationTokenSource? cancellationTokenSource) + public AbortForMaxFailedTestsExtension(ICommandLineOptions commandLineOptions, IStopTestExecutionCapability? capability, CancellationToken cancellationToken) { - _cancellationTokenSource = cancellationTokenSource; if (commandLineOptions.TryGetOptionArgumentList(PlatformCommandLineProvider.MaxFailedTestsOptionKey, out string[]? args) && int.TryParse(args[0], out int maxFailedTests) && maxFailedTests > 0) { _maxFailedTests = maxFailedTests; } + + _capability = capability; + _cancellationToken = cancellationToken; } public Type[] DataTypesConsumed { get; } = [typeof(TestNodeUpdateMessage)]; @@ -42,7 +45,7 @@ public AbortForMaxFailedTestsExtension(ICommandLineOptions commandLineOptions, C public string Description { get; } = PlatformResources.AbortForMaxFailedTestsDescription; /// - public Task IsEnabledAsync() => Task.FromResult(_maxFailedTests.HasValue && _cancellationTokenSource is not null); + public Task IsEnabledAsync() => Task.FromResult(_maxFailedTests.HasValue && _capability is not null); public async Task ConsumeAsync(IDataProducer dataProducer, IData value, CancellationToken cancellationToken) { @@ -50,13 +53,13 @@ public async Task ConsumeAsync(IDataProducer dataProducer, IData value, Cancella // If we are called, the extension is enabled, which means both _maxFailedTests and _cancellationTokenSource are not null. RoslynDebug.Assert(_maxFailedTests is not null); - RoslynDebug.Assert(_cancellationTokenSource is not null); + RoslynDebug.Assert(_capability is not null); int maxFailed = _maxFailedTests.Value; if (node.TestNode.Properties.Single() is FailedTestNodeStateProperty && Interlocked.Increment(ref _failCount) > maxFailed) { - await _cancellationTokenSource.CancelAsync(); + await _capability.StopTestExecutionAsync(_cancellationToken); } } } diff --git a/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs b/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs index d45e6a9d30..bd4a9aa0a1 100644 --- a/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs +++ b/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs @@ -246,7 +246,8 @@ public async Task BuildAsync( TestHost.AddDataConsumer( serviceProvider => new AbortForMaxFailedTestsExtension( serviceProvider.GetCommandLineOptions(), - serviceProvider.GetTestFrameworkCapabilities().GetCapability()?.CancellationTokenSource)); + serviceProvider.GetTestFrameworkCapabilities().GetCapability(), + serviceProvider.GetTestApplicationCancellationTokenSource().CancellationToken)); // If command line is not valid we return immediately. ValidationResult commandLineValidationResult = await CommandLineOptionsValidator.ValidateAsync( diff --git a/src/Platform/Microsoft.Testing.Platform/PublicAPI/PublicAPI.Unshipped.txt b/src/Platform/Microsoft.Testing.Platform/PublicAPI/PublicAPI.Unshipped.txt index 147bda8aba..569dabbc86 100644 --- a/src/Platform/Microsoft.Testing.Platform/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/Platform/Microsoft.Testing.Platform/PublicAPI/PublicAPI.Unshipped.txt @@ -1,6 +1,6 @@ #nullable enable Microsoft.Testing.Platform.Capabilities.TestFramework.IStopTestExecutionCapability -Microsoft.Testing.Platform.Capabilities.TestFramework.IStopTestExecutionCapability.CancellationTokenSource.get -> System.Threading.CancellationTokenSource! +Microsoft.Testing.Platform.Capabilities.TestFramework.IStopTestExecutionCapability.StopTestExecutionAsync(System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task! Microsoft.Testing.Platform.Extensions.Messages.TestMetadataProperty.TestMetadataProperty(string! key) -> void Microsoft.Testing.Platform.OutputDevice.ErrorMessageOutputDeviceData Microsoft.Testing.Platform.OutputDevice.ErrorMessageOutputDeviceData.ErrorMessageOutputDeviceData(string! message) -> void From 46b7577056b4fbc92dae6b71c61a646a200a262f Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Thu, 5 Dec 2024 16:56:38 +0100 Subject: [PATCH 10/64] Progress for policies service --- .../AbortForMaxFailedTestsExtension.cs | 10 +++++-- .../Hosts/TestHostBuilder.cs | 5 ++++ .../OutputDevice/OutputDeviceManager.cs | 7 ++++- .../OutputDevice/ProxyOutputDevice.cs | 7 ++++- .../Services/IPoliciesService.cs | 9 +++++++ .../Services/PoliciesService.cs | 27 +++++++++++++++++++ 6 files changed, 61 insertions(+), 4 deletions(-) create mode 100644 src/Platform/Microsoft.Testing.Platform/Services/IPoliciesService.cs create mode 100644 src/Platform/Microsoft.Testing.Platform/Services/PoliciesService.cs diff --git a/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs b/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs index b8cde4e982..5e7bafc6dd 100644 --- a/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs +++ b/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs @@ -6,7 +6,9 @@ using Microsoft.Testing.Platform.Extensions.Messages; using Microsoft.Testing.Platform.Extensions.TestHost; using Microsoft.Testing.Platform.Helpers; +using Microsoft.Testing.Platform.Messages; using Microsoft.Testing.Platform.Resources; +using Microsoft.Testing.Platform.Services; namespace Microsoft.Testing.Platform.Extensions; @@ -14,10 +16,11 @@ internal sealed class AbortForMaxFailedTestsExtension : IDataConsumer { private readonly int? _maxFailedTests; private readonly IStopTestExecutionCapability? _capability; + private readonly PoliciesService _policiesService; private readonly CancellationToken _cancellationToken; private int _failCount; - public AbortForMaxFailedTestsExtension(ICommandLineOptions commandLineOptions, IStopTestExecutionCapability? capability, CancellationToken cancellationToken) + public AbortForMaxFailedTestsExtension(ICommandLineOptions commandLineOptions, IStopTestExecutionCapability? capability, PoliciesService policiesService, CancellationToken cancellationToken) { if (commandLineOptions.TryGetOptionArgumentList(PlatformCommandLineProvider.MaxFailedTestsOptionKey, out string[]? args) && int.TryParse(args[0], out int maxFailedTests) && @@ -27,6 +30,7 @@ public AbortForMaxFailedTestsExtension(ICommandLineOptions commandLineOptions, I } _capability = capability; + _policiesService = policiesService; _cancellationToken = cancellationToken; } @@ -56,10 +60,12 @@ public async Task ConsumeAsync(IDataProducer dataProducer, IData value, Cancella RoslynDebug.Assert(_capability is not null); int maxFailed = _maxFailedTests.Value; - if (node.TestNode.Properties.Single() is FailedTestNodeStateProperty && + TestNodeStateProperty testNodeStateProperty = node.TestNode.Properties.Single(); + if (TestNodePropertiesCategories.WellKnownTestNodeTestRunOutcomeFailedProperties.Any(t => t == testNodeStateProperty.GetType()) && Interlocked.Increment(ref _failCount) > maxFailed) { await _capability.StopTestExecutionAsync(_cancellationToken); + await _policiesService.ExecuteOnStopTestExecutionCallbacks(_cancellationToken); } } } diff --git a/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs b/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs index bd4a9aa0a1..b3bf75d9f4 100644 --- a/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs +++ b/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs @@ -206,6 +206,10 @@ public async Task BuildAsync( // Set the concrete command line options to the proxy. commandLineOptionsProxy.SetCommandLineOptions(commandLineHandler); + // This needs to be before output device because output device will use it. + var policiesService = new PoliciesService(); + serviceProvider.AddService(policiesService); + bool hasServerFlag = commandLineHandler.TryGetOptionArgumentList(PlatformCommandLineProvider.ServerOptionKey, out string[]? protocolName); bool isJsonRpcProtocol = protocolName is null || protocolName.Length == 0 || protocolName[0].Equals(PlatformCommandLineProvider.JsonRpcProtocolName, StringComparison.OrdinalIgnoreCase); @@ -247,6 +251,7 @@ public async Task BuildAsync( serviceProvider => new AbortForMaxFailedTestsExtension( serviceProvider.GetCommandLineOptions(), serviceProvider.GetTestFrameworkCapabilities().GetCapability(), + policiesService, serviceProvider.GetTestApplicationCancellationTokenSource().CancellationToken)); // If command line is not valid we return immediately. diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/OutputDeviceManager.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/OutputDeviceManager.cs index d3e09c4e8b..978495b2a2 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/OutputDeviceManager.cs +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/OutputDeviceManager.cs @@ -30,7 +30,12 @@ internal async Task BuildAsync(ServiceProvider serviceProvide nonServerOutputDevice = GetDefaultTerminalOutputDevice(serviceProvider); } - return new ProxyOutputDevice(nonServerOutputDevice, useServerModeOutputDevice ? new ServerModePerCallOutputDevice(serviceProvider) : null); + return new ProxyOutputDevice( + nonServerOutputDevice, + useServerModeOutputDevice + ? new ServerModePerCallOutputDevice(serviceProvider) + : null, + serviceProvider.GetService()); } public static TerminalOutputDevice GetDefaultTerminalOutputDevice(ServiceProvider serviceProvider) diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/ProxyOutputDevice.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/ProxyOutputDevice.cs index 6896debd84..861505c0d9 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/ProxyOutputDevice.cs +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/ProxyOutputDevice.cs @@ -4,6 +4,7 @@ using Microsoft.Testing.Platform.Extensions.OutputDevice; using Microsoft.Testing.Platform.Hosts; using Microsoft.Testing.Platform.ServerMode; +using Microsoft.Testing.Platform.Services; namespace Microsoft.Testing.Platform.OutputDevice; @@ -11,10 +12,14 @@ internal sealed class ProxyOutputDevice : IOutputDevice { private readonly ServerModePerCallOutputDevice? _serverModeOutputDevice; - public ProxyOutputDevice(IPlatformOutputDevice originalOutputDevice, ServerModePerCallOutputDevice? serverModeOutputDevice) + public ProxyOutputDevice(IPlatformOutputDevice originalOutputDevice, ServerModePerCallOutputDevice? serverModeOutputDevice, IPoliciesService policiesService) { OriginalOutputDevice = originalOutputDevice; _serverModeOutputDevice = serverModeOutputDevice; + policiesService.RegisterOnStopTestExecution( + async ct => await DisplayAsync( + /*TODO: pass 'this' and implement IOutputDeviceDataProducer*/null!, + new TextOutputDeviceData(/*TODO: Localize*/"Test session is being stopped."))); } internal IPlatformOutputDevice OriginalOutputDevice { get; } diff --git a/src/Platform/Microsoft.Testing.Platform/Services/IPoliciesService.cs b/src/Platform/Microsoft.Testing.Platform/Services/IPoliciesService.cs new file mode 100644 index 0000000000..811ec922d5 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Platform/Services/IPoliciesService.cs @@ -0,0 +1,9 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.Testing.Platform.Services; + +internal interface IPoliciesService +{ + void RegisterOnStopTestExecution(Func callback); +} diff --git a/src/Platform/Microsoft.Testing.Platform/Services/PoliciesService.cs b/src/Platform/Microsoft.Testing.Platform/Services/PoliciesService.cs new file mode 100644 index 0000000000..6c9e788100 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Platform/Services/PoliciesService.cs @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Collections.Concurrent; + +namespace Microsoft.Testing.Platform.Services; + +internal sealed class PoliciesService : IPoliciesService +{ + private BlockingCollection>? _onStopTestExecutionCallbacks; + + public void RegisterOnStopTestExecution(Func callback) + => (_onStopTestExecutionCallbacks ??= new()).Add(callback); + + internal async Task ExecuteOnStopTestExecutionCallbacks(CancellationToken cancellationToken) + { + if (_onStopTestExecutionCallbacks is null) + { + return; + } + + foreach (Func callback in _onStopTestExecutionCallbacks) + { + await callback.Invoke(cancellationToken); + } + } +} From be7c6cc92f1a1416f4279a12c2680e20b13c1ff9 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Thu, 5 Dec 2024 16:57:53 +0100 Subject: [PATCH 11/64] Use GetRequiredService --- .../OutputDevice/OutputDeviceManager.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/OutputDeviceManager.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/OutputDeviceManager.cs index 978495b2a2..68d336f2c7 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/OutputDeviceManager.cs +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/OutputDeviceManager.cs @@ -35,7 +35,7 @@ internal async Task BuildAsync(ServiceProvider serviceProvide useServerModeOutputDevice ? new ServerModePerCallOutputDevice(serviceProvider) : null, - serviceProvider.GetService()); + serviceProvider.GetRequiredService()); } public static TerminalOutputDevice GetDefaultTerminalOutputDevice(ServiceProvider serviceProvider) From db432efa0b9180968210be5597ebc5c24337ae05 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Thu, 5 Dec 2024 17:16:24 +0100 Subject: [PATCH 12/64] Progress --- .../Helpers/ExitCodes.cs | 1 + .../Hosts/TestHostBuilder.cs | 4 +- .../OutputDevice/OutputDeviceManager.cs | 3 +- .../OutputDevice/ProxyOutputDevice.cs | 7 +--- .../Services/TestApplicationResult.cs | 41 +++++++++++++++---- 5 files changed, 37 insertions(+), 19 deletions(-) diff --git a/src/Platform/Microsoft.Testing.Platform/Helpers/ExitCodes.cs b/src/Platform/Microsoft.Testing.Platform/Helpers/ExitCodes.cs index 28514ae2c0..246717620e 100644 --- a/src/Platform/Microsoft.Testing.Platform/Helpers/ExitCodes.cs +++ b/src/Platform/Microsoft.Testing.Platform/Helpers/ExitCodes.cs @@ -22,4 +22,5 @@ internal static class ExitCodes public const int TestAdapterTestSessionFailure = 10; public const int DependentProcessExited = 11; public const int IncompatibleProtocolVersion = 12; + public const int TestExecutionStopped = 13; } diff --git a/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs b/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs index b3bf75d9f4..bf5a92a0da 100644 --- a/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs +++ b/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs @@ -206,7 +206,6 @@ public async Task BuildAsync( // Set the concrete command line options to the proxy. commandLineOptionsProxy.SetCommandLineOptions(commandLineHandler); - // This needs to be before output device because output device will use it. var policiesService = new PoliciesService(); serviceProvider.AddService(policiesService); @@ -326,7 +325,8 @@ public async Task BuildAsync( proxyOutputDevice, serviceProvider.GetTestApplicationCancellationTokenSource(), serviceProvider.GetCommandLineOptions(), - serviceProvider.GetEnvironment()); + serviceProvider.GetEnvironment(), + policiesService); serviceProvider.AddService(testApplicationResult); // ============= SETUP COMMON SERVICE USED IN ALL MODES END ===============// diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/OutputDeviceManager.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/OutputDeviceManager.cs index 68d336f2c7..e8e7ac88b0 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/OutputDeviceManager.cs +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/OutputDeviceManager.cs @@ -34,8 +34,7 @@ internal async Task BuildAsync(ServiceProvider serviceProvide nonServerOutputDevice, useServerModeOutputDevice ? new ServerModePerCallOutputDevice(serviceProvider) - : null, - serviceProvider.GetRequiredService()); + : null); } public static TerminalOutputDevice GetDefaultTerminalOutputDevice(ServiceProvider serviceProvider) diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/ProxyOutputDevice.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/ProxyOutputDevice.cs index 861505c0d9..6896debd84 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/ProxyOutputDevice.cs +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/ProxyOutputDevice.cs @@ -4,7 +4,6 @@ using Microsoft.Testing.Platform.Extensions.OutputDevice; using Microsoft.Testing.Platform.Hosts; using Microsoft.Testing.Platform.ServerMode; -using Microsoft.Testing.Platform.Services; namespace Microsoft.Testing.Platform.OutputDevice; @@ -12,14 +11,10 @@ internal sealed class ProxyOutputDevice : IOutputDevice { private readonly ServerModePerCallOutputDevice? _serverModeOutputDevice; - public ProxyOutputDevice(IPlatformOutputDevice originalOutputDevice, ServerModePerCallOutputDevice? serverModeOutputDevice, IPoliciesService policiesService) + public ProxyOutputDevice(IPlatformOutputDevice originalOutputDevice, ServerModePerCallOutputDevice? serverModeOutputDevice) { OriginalOutputDevice = originalOutputDevice; _serverModeOutputDevice = serverModeOutputDevice; - policiesService.RegisterOnStopTestExecution( - async ct => await DisplayAsync( - /*TODO: pass 'this' and implement IOutputDeviceDataProducer*/null!, - new TextOutputDeviceData(/*TODO: Localize*/"Test session is being stopped."))); } internal IPlatformOutputDevice OriginalOutputDevice { get; } diff --git a/src/Platform/Microsoft.Testing.Platform/Services/TestApplicationResult.cs b/src/Platform/Microsoft.Testing.Platform/Services/TestApplicationResult.cs index 3304d317de..2acbdd95f2 100644 --- a/src/Platform/Microsoft.Testing.Platform/Services/TestApplicationResult.cs +++ b/src/Platform/Microsoft.Testing.Platform/Services/TestApplicationResult.cs @@ -13,19 +13,41 @@ namespace Microsoft.Testing.Platform.Services; -internal sealed class TestApplicationResult( - IOutputDevice outputService, - ITestApplicationCancellationTokenSource testApplicationCancellationTokenSource, - ICommandLineOptions commandLineOptions, - IEnvironment environment) : ITestApplicationProcessExitCode, IOutputDeviceDataProducer +internal sealed class TestApplicationResult : ITestApplicationProcessExitCode, IOutputDeviceDataProducer { - private readonly IOutputDevice _outputService = outputService; - private readonly ITestApplicationCancellationTokenSource _testApplicationCancellationTokenSource = testApplicationCancellationTokenSource; - private readonly ICommandLineOptions _commandLineOptions = commandLineOptions; - private readonly IEnvironment _environment = environment; + private readonly IOutputDevice _outputService; + private readonly ITestApplicationCancellationTokenSource _testApplicationCancellationTokenSource; + private readonly ICommandLineOptions _commandLineOptions; + private readonly IEnvironment _environment; + private readonly PoliciesService _policiesService; private readonly List _failedTests = []; private int _totalRanTests; private bool _testAdapterTestSessionFailure; + private bool _testExecutionStopped; + + public TestApplicationResult( + IOutputDevice outputService, + ITestApplicationCancellationTokenSource testApplicationCancellationTokenSource, + ICommandLineOptions commandLineOptions, + IEnvironment environment, + PoliciesService policiesService) + { + _outputService = outputService; + _testApplicationCancellationTokenSource = testApplicationCancellationTokenSource; + _commandLineOptions = commandLineOptions; + _environment = environment; + policiesService.RegisterOnStopTestExecution( + async cancellationToken => + { + // How to make the message more clear that it's due to --max-failed-tests? + // The way we are currently abstracting is too generic that any one could use IStopTestExecutionCapability for whatever other reason. + // So, we can't here "hardcode" the message to be specific to --max-failed-tests. + // Do we want to introduce a "reason" parameter in IStopTestExecutionCapability.StopTestExecutionAsync? + // TODO: Localize + await _outputService.DisplayAsync(this, new TextOutputDeviceData("Test session was stopped.")); + _testExecutionStopped = true; + }); + } /// public string Uid { get; } = nameof(TestApplicationResult); @@ -81,6 +103,7 @@ public Task ConsumeAsync(IDataProducer dataProducer, IData value, CancellationTo public int GetProcessExitCode() { int exitCode = ExitCodes.Success; + exitCode = exitCode == ExitCodes.Success && _testExecutionStopped ? ExitCodes.TestExecutionStopped : exitCode; exitCode = exitCode == ExitCodes.Success && _testAdapterTestSessionFailure ? ExitCodes.TestAdapterTestSessionFailure : exitCode; exitCode = exitCode == ExitCodes.Success && _failedTests.Count > 0 ? ExitCodes.AtLeastOneTestFailed : exitCode; exitCode = exitCode == ExitCodes.Success && _testApplicationCancellationTokenSource.CancellationToken.IsCancellationRequested ? ExitCodes.TestSessionAborted : exitCode; From c7b13925539fb4c886077a111c7d0bc496881ebb Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Thu, 5 Dec 2024 17:18:55 +0100 Subject: [PATCH 13/64] Fix warning --- .../Microsoft.Testing.Platform/Services/TestApplicationResult.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Platform/Microsoft.Testing.Platform/Services/TestApplicationResult.cs b/src/Platform/Microsoft.Testing.Platform/Services/TestApplicationResult.cs index 2acbdd95f2..c788db5cfd 100644 --- a/src/Platform/Microsoft.Testing.Platform/Services/TestApplicationResult.cs +++ b/src/Platform/Microsoft.Testing.Platform/Services/TestApplicationResult.cs @@ -19,7 +19,6 @@ internal sealed class TestApplicationResult : ITestApplicationProcessExitCode, I private readonly ITestApplicationCancellationTokenSource _testApplicationCancellationTokenSource; private readonly ICommandLineOptions _commandLineOptions; private readonly IEnvironment _environment; - private readonly PoliciesService _policiesService; private readonly List _failedTests = []; private int _totalRanTests; private bool _testAdapterTestSessionFailure; From 624f86f683e52ac65e34f43d2cb34a1128629495 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Fri, 6 Dec 2024 11:50:19 +0100 Subject: [PATCH 14/64] Progress --- ...IStopGracefullyTestExecutionCapability.cs} | 2 +- .../AbortForMaxFailedTestsExtension.cs | 9 +++-- .../Hosts/TestHostBuilder.cs | 15 ++++---- .../OutputDevice/OutputDeviceManager.cs | 8 +++-- .../OutputDevice/TerminalOutputDevice.cs | 6 +++- .../PublicAPI/PublicAPI.Unshipped.txt | 4 +-- .../Resources/PlatformResources.resx | 5 ++- .../Resources/xlf/PlatformResources.cs.xlf | 5 +++ .../Resources/xlf/PlatformResources.de.xlf | 5 +++ .../Resources/xlf/PlatformResources.es.xlf | 5 +++ .../Resources/xlf/PlatformResources.fr.xlf | 5 +++ .../Resources/xlf/PlatformResources.it.xlf | 5 +++ .../Resources/xlf/PlatformResources.ja.xlf | 5 +++ .../Resources/xlf/PlatformResources.ko.xlf | 5 +++ .../Resources/xlf/PlatformResources.pl.xlf | 5 +++ .../Resources/xlf/PlatformResources.pt-BR.xlf | 5 +++ .../Resources/xlf/PlatformResources.ru.xlf | 5 +++ .../Resources/xlf/PlatformResources.tr.xlf | 5 +++ .../xlf/PlatformResources.zh-Hans.xlf | 5 +++ .../xlf/PlatformResources.zh-Hant.xlf | 5 +++ .../JsonRpc/ServerModePerCallOutputDevice.cs | 14 +++++--- .../Services/IPoliciesService.cs | 4 ++- .../Services/PoliciesService.cs | 35 +++++++++++++------ .../Services/TestApplicationResult.cs | 11 ++---- .../Services/TestApplicationResultTests.cs | 15 ++++---- 25 files changed, 142 insertions(+), 51 deletions(-) rename src/Platform/Microsoft.Testing.Platform/Capabilities/TestFramework/{IStopTestExecutionCapability.cs => IStopGracefullyTestExecutionCapability.cs} (87%) diff --git a/src/Platform/Microsoft.Testing.Platform/Capabilities/TestFramework/IStopTestExecutionCapability.cs b/src/Platform/Microsoft.Testing.Platform/Capabilities/TestFramework/IStopGracefullyTestExecutionCapability.cs similarity index 87% rename from src/Platform/Microsoft.Testing.Platform/Capabilities/TestFramework/IStopTestExecutionCapability.cs rename to src/Platform/Microsoft.Testing.Platform/Capabilities/TestFramework/IStopGracefullyTestExecutionCapability.cs index 7eae7e4d27..6bde8a6ec5 100644 --- a/src/Platform/Microsoft.Testing.Platform/Capabilities/TestFramework/IStopTestExecutionCapability.cs +++ b/src/Platform/Microsoft.Testing.Platform/Capabilities/TestFramework/IStopGracefullyTestExecutionCapability.cs @@ -10,7 +10,7 @@ namespace Microsoft.Testing.Platform.Capabilities.TestFramework; /// /// Test frameworks can choose to run any needed cleanup when cancellation is requested. /// -public interface IStopTestExecutionCapability : ITestFrameworkCapability +public interface IStopGracefullyTestExecutionCapability : ITestFrameworkCapability { Task StopTestExecutionAsync(CancellationToken cancellationToken); } diff --git a/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs b/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs index 5e7bafc6dd..6c3581b1ed 100644 --- a/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs +++ b/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs @@ -15,12 +15,12 @@ namespace Microsoft.Testing.Platform.Extensions; internal sealed class AbortForMaxFailedTestsExtension : IDataConsumer { private readonly int? _maxFailedTests; - private readonly IStopTestExecutionCapability? _capability; + private readonly IStopGracefullyTestExecutionCapability? _capability; private readonly PoliciesService _policiesService; private readonly CancellationToken _cancellationToken; private int _failCount; - public AbortForMaxFailedTestsExtension(ICommandLineOptions commandLineOptions, IStopTestExecutionCapability? capability, PoliciesService policiesService, CancellationToken cancellationToken) + public AbortForMaxFailedTestsExtension(ICommandLineOptions commandLineOptions, IStopGracefullyTestExecutionCapability? capability, PoliciesService policiesService, CancellationToken cancellationToken) { if (commandLineOptions.TryGetOptionArgumentList(PlatformCommandLineProvider.MaxFailedTestsOptionKey, out string[]? args) && int.TryParse(args[0], out int maxFailedTests) && @@ -59,13 +59,12 @@ public async Task ConsumeAsync(IDataProducer dataProducer, IData value, Cancella RoslynDebug.Assert(_maxFailedTests is not null); RoslynDebug.Assert(_capability is not null); - int maxFailed = _maxFailedTests.Value; TestNodeStateProperty testNodeStateProperty = node.TestNode.Properties.Single(); if (TestNodePropertiesCategories.WellKnownTestNodeTestRunOutcomeFailedProperties.Any(t => t == testNodeStateProperty.GetType()) && - Interlocked.Increment(ref _failCount) > maxFailed) + ++_failCount > _maxFailedTests.Value) { await _capability.StopTestExecutionAsync(_cancellationToken); - await _policiesService.ExecuteOnStopTestExecutionCallbacks(_cancellationToken); + await _policiesService.MaxFailedTestsPolicy.ExecuteCallbacksAsync(_cancellationToken); } } } diff --git a/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs b/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs index bf5a92a0da..abf35397a1 100644 --- a/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs +++ b/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs @@ -206,6 +206,7 @@ public async Task BuildAsync( // Set the concrete command line options to the proxy. commandLineOptionsProxy.SetCommandLineOptions(commandLineHandler); + // This is needed by output device. var policiesService = new PoliciesService(); serviceProvider.AddService(policiesService); @@ -246,13 +247,6 @@ public async Task BuildAsync( await testFrameworkCapabilitiesAsyncInitializable.InitializeAsync(); } - TestHost.AddDataConsumer( - serviceProvider => new AbortForMaxFailedTestsExtension( - serviceProvider.GetCommandLineOptions(), - serviceProvider.GetTestFrameworkCapabilities().GetCapability(), - policiesService, - serviceProvider.GetTestApplicationCancellationTokenSource().CancellationToken)); - // If command line is not valid we return immediately. ValidationResult commandLineValidationResult = await CommandLineOptionsValidator.ValidateAsync( loggingState.CommandLineParseResult, @@ -739,6 +733,13 @@ private async Task BuildTestFrameworkAsync(TestFrameworkBuilderD dataConsumersBuilder.Add(pushOnlyProtocolDataConsumer); } + dataConsumersBuilder.Add( + new AbortForMaxFailedTestsExtension( + serviceProvider.GetCommandLineOptions(), + serviceProvider.GetTestFrameworkCapabilities().GetCapability(), + serviceProvider.GetRequiredService(), + serviceProvider.GetTestApplicationCancellationTokenSource().CancellationToken)); + IDataConsumer[] dataConsumerServices = dataConsumersBuilder.ToArray(); // Build the message bus diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/OutputDeviceManager.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/OutputDeviceManager.cs index e8e7ac88b0..38ab778132 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/OutputDeviceManager.cs +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/OutputDeviceManager.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +using Microsoft.Testing.Platform.Logging; using Microsoft.Testing.Platform.ServerMode; using Microsoft.Testing.Platform.Services; @@ -33,7 +34,9 @@ internal async Task BuildAsync(ServiceProvider serviceProvide return new ProxyOutputDevice( nonServerOutputDevice, useServerModeOutputDevice - ? new ServerModePerCallOutputDevice(serviceProvider) + ? new ServerModePerCallOutputDevice( + serviceProvider.GetService(), + serviceProvider.GetRequiredService()) : null); } @@ -51,5 +54,6 @@ public static TerminalOutputDevice GetDefaultTerminalOutputDevice(ServiceProvide serviceProvider.GetCommandLineOptions(), serviceProvider.GetFileLoggerInformation(), serviceProvider.GetLoggerFactory(), - serviceProvider.GetClock()); + serviceProvider.GetClock(), + serviceProvider.GetRequiredService()); } diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.cs index 05b5a39fc3..9b2882ef8c 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.cs +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.cs @@ -72,7 +72,8 @@ internal partial class TerminalOutputDevice : IHotReloadPlatformOutputDevice, public TerminalOutputDevice(ITestApplicationCancellationTokenSource testApplicationCancellationTokenSource, IConsole console, ITestApplicationModuleInfo testApplicationModuleInfo, ITestHostControllerInfo testHostControllerInfo, IAsyncMonitor asyncMonitor, IRuntimeFeature runtimeFeature, IEnvironment environment, IProcessHandler process, IPlatformInformation platformInformation, - ICommandLineOptions commandLineOptions, IFileLoggerInformation? fileLoggerInformation, ILoggerFactory loggerFactory, IClock clock) + ICommandLineOptions commandLineOptions, IFileLoggerInformation? fileLoggerInformation, ILoggerFactory loggerFactory, IClock clock, + IPoliciesService policiesService) { _testApplicationCancellationTokenSource = testApplicationCancellationTokenSource; _console = console; @@ -88,6 +89,9 @@ public TerminalOutputDevice(ITestApplicationCancellationTokenSource testApplicat _loggerFactory = loggerFactory; _clock = clock; + policiesService.RegisterOnMaxFailedTestsCallback( + async _ => await DisplayAsync(this, new TextOutputDeviceData(PlatformResources.ReachedMaxFailedTestsMessage))); + if (_runtimeFeature.IsDynamicCodeSupported) { #if !NETCOREAPP diff --git a/src/Platform/Microsoft.Testing.Platform/PublicAPI/PublicAPI.Unshipped.txt b/src/Platform/Microsoft.Testing.Platform/PublicAPI/PublicAPI.Unshipped.txt index 569dabbc86..89d7a5d813 100644 --- a/src/Platform/Microsoft.Testing.Platform/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/Platform/Microsoft.Testing.Platform/PublicAPI/PublicAPI.Unshipped.txt @@ -1,6 +1,6 @@ #nullable enable -Microsoft.Testing.Platform.Capabilities.TestFramework.IStopTestExecutionCapability -Microsoft.Testing.Platform.Capabilities.TestFramework.IStopTestExecutionCapability.StopTestExecutionAsync(System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task! +Microsoft.Testing.Platform.Capabilities.TestFramework.IStopGracefullyTestExecutionCapability +Microsoft.Testing.Platform.Capabilities.TestFramework.IStopGracefullyTestExecutionCapability.StopTestExecutionAsync(System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task! Microsoft.Testing.Platform.Extensions.Messages.TestMetadataProperty.TestMetadataProperty(string! key) -> void Microsoft.Testing.Platform.OutputDevice.ErrorMessageOutputDeviceData Microsoft.Testing.Platform.OutputDevice.ErrorMessageOutputDeviceData.ErrorMessageOutputDeviceData(string! message) -> void diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/PlatformResources.resx b/src/Platform/Microsoft.Testing.Platform/Resources/PlatformResources.resx index 0affd9d0a7..60e537cc96 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/PlatformResources.resx +++ b/src/Platform/Microsoft.Testing.Platform/Resources/PlatformResources.resx @@ -703,4 +703,7 @@ Takes one argument as string in the format <value>[h|m|s] where 'value' is Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted. - \ No newline at end of file + + Test session is aborting due to reaching failures specified by the '--max-failed-tests' option. + + diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.cs.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.cs.xlf index 1959039899..3f8522f7cb 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.cs.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.cs.xlf @@ -628,6 +628,11 @@ Může mít jenom jeden argument jako řetězec ve formátu <value>[h|m|s] Proces měl být ukončen před tím, než jsme mohli určit tuto hodnotu. + + Test session is aborting due to reaching failures specified by the '--max-failed-tests' option. + Test session is aborting due to reaching failures specified by the '--max-failed-tests' option. + + Retry failed after {0} times Opakování se po {0} pokusech nezdařilo. diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.de.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.de.xlf index a7906fbb04..0809f66116 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.de.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.de.xlf @@ -628,6 +628,11 @@ Nimmt ein Argument als Zeichenfolge im Format <value>[h|m|s], wobei "value Der Prozess hätte beendet werden müssen, bevor dieser Wert ermittelt werden kann + + Test session is aborting due to reaching failures specified by the '--max-failed-tests' option. + Test session is aborting due to reaching failures specified by the '--max-failed-tests' option. + + Retry failed after {0} times Wiederholungsfehler nach {0} Mal diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.es.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.es.xlf index ef4adb82cf..2711aafb0d 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.es.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.es.xlf @@ -628,6 +628,11 @@ Toma un argumento como cadena con el formato <value>[h|m|s] donde 'value' El proceso debería haberse terminado para poder determinar este valor + + Test session is aborting due to reaching failures specified by the '--max-failed-tests' option. + Test session is aborting due to reaching failures specified by the '--max-failed-tests' option. + + Retry failed after {0} times Error al reintentar después de {0} veces diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.fr.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.fr.xlf index 3c76f4bf51..35879307c3 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.fr.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.fr.xlf @@ -628,6 +628,11 @@ Prend un argument sous forme de chaîne au format <value>[h|m|s] où « v Le processus aurait dû s’arrêter avant que nous puissions déterminer cette valeur + + Test session is aborting due to reaching failures specified by the '--max-failed-tests' option. + Test session is aborting due to reaching failures specified by the '--max-failed-tests' option. + + Retry failed after {0} times Échec de la nouvelle tentative après {0} fois diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.it.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.it.xlf index cb5a023a1f..be692f5b08 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.it.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.it.xlf @@ -628,6 +628,11 @@ Acquisisce un argomento come stringa nel formato <value>[h|m|s] dove 'valu Il processo dovrebbe essere terminato prima di poter determinare questo valore + + Test session is aborting due to reaching failures specified by the '--max-failed-tests' option. + Test session is aborting due to reaching failures specified by the '--max-failed-tests' option. + + Retry failed after {0} times Tentativi non riusciti dopo {0} tentativi diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ja.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ja.xlf index 5580e899b0..fa60bacb12 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ja.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ja.xlf @@ -629,6 +629,11 @@ Takes one argument as string in the format <value>[h|m|s] where 'value' is この値を決定する前にプロセスを終了する必要があります + + Test session is aborting due to reaching failures specified by the '--max-failed-tests' option. + Test session is aborting due to reaching failures specified by the '--max-failed-tests' option. + + Retry failed after {0} times 再試行が {0} 回後に失敗しました diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ko.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ko.xlf index 9b78b8eb14..97c7d28b76 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ko.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ko.xlf @@ -628,6 +628,11 @@ Takes one argument as string in the format <value>[h|m|s] where 'value' is 이 값을 결정하려면 프로세스가 종료되어야 합니다. + + Test session is aborting due to reaching failures specified by the '--max-failed-tests' option. + Test session is aborting due to reaching failures specified by the '--max-failed-tests' option. + + Retry failed after {0} times {0}회 후 다시 시도 실패 diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pl.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pl.xlf index 65f16e7a06..073a6be265 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pl.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pl.xlf @@ -628,6 +628,11 @@ Pobiera jeden argument jako ciąg w formacie <value>[h|m|s], gdzie element Proces powinien zakończyć się przed ustaleniem tej wartości + + Test session is aborting due to reaching failures specified by the '--max-failed-tests' option. + Test session is aborting due to reaching failures specified by the '--max-failed-tests' option. + + Retry failed after {0} times Ponowna próba nie powiodła się po {0} razach diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pt-BR.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pt-BR.xlf index 648a1a36bd..6bb4f5739f 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pt-BR.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pt-BR.xlf @@ -628,6 +628,11 @@ Recebe um argumento como cadeia de caracteres no formato <valor>[h|m|s] em O processo deve ter sido encerrado antes que possamos determinar esse valor + + Test session is aborting due to reaching failures specified by the '--max-failed-tests' option. + Test session is aborting due to reaching failures specified by the '--max-failed-tests' option. + + Retry failed after {0} times Falha na repetição após o {0} tempo diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ru.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ru.xlf index 40bdb84fa2..ad42932e14 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ru.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ru.xlf @@ -628,6 +628,11 @@ Takes one argument as string in the format <value>[h|m|s] where 'value' is Процесс должен быть завершен, прежде чем мы сможем определить это значение + + Test session is aborting due to reaching failures specified by the '--max-failed-tests' option. + Test session is aborting due to reaching failures specified by the '--max-failed-tests' option. + + Retry failed after {0} times Повторные попытки ({0}) завершились неудачно diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.tr.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.tr.xlf index 88cad048df..ce820d699a 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.tr.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.tr.xlf @@ -628,6 +628,11 @@ Bir bağımsız değişkeni, 'value' değerinin kayan olduğu <value>[h|m| Bu değeri belirleyebilmemiz için süreçten çıkılmış olması gerekir + + Test session is aborting due to reaching failures specified by the '--max-failed-tests' option. + Test session is aborting due to reaching failures specified by the '--max-failed-tests' option. + + Retry failed after {0} times Yeniden deneme {0} deneme sonrasında başarısız oldu diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hans.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hans.xlf index 6adce85349..c512b5f49c 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hans.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hans.xlf @@ -628,6 +628,11 @@ Takes one argument as string in the format <value>[h|m|s] where 'value' is 在我们确定此值之前，流程应该已退出 + + Test session is aborting due to reaching failures specified by the '--max-failed-tests' option. + Test session is aborting due to reaching failures specified by the '--max-failed-tests' option. + + Retry failed after {0} times {0} 次之后重试失败 diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hant.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hant.xlf index dd79c64866..ad367169f7 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hant.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hant.xlf @@ -628,6 +628,11 @@ Takes one argument as string in the format <value>[h|m|s] where 'value' is 在我們確定此值之前，流程應已結束 + + Test session is aborting due to reaching failures specified by the '--max-failed-tests' option. + Test session is aborting due to reaching failures specified by the '--max-failed-tests' option. + + Retry failed after {0} times 在 {0} 次重試之後失敗 diff --git a/src/Platform/Microsoft.Testing.Platform/ServerMode/JsonRpc/ServerModePerCallOutputDevice.cs b/src/Platform/Microsoft.Testing.Platform/ServerMode/JsonRpc/ServerModePerCallOutputDevice.cs index 091f782ad3..8593575ffc 100644 --- a/src/Platform/Microsoft.Testing.Platform/ServerMode/JsonRpc/ServerModePerCallOutputDevice.cs +++ b/src/Platform/Microsoft.Testing.Platform/ServerMode/JsonRpc/ServerModePerCallOutputDevice.cs @@ -15,17 +15,21 @@ namespace Microsoft.Testing.Platform.ServerMode; -internal sealed class ServerModePerCallOutputDevice : IPlatformOutputDevice +internal sealed class ServerModePerCallOutputDevice : IPlatformOutputDevice, IOutputDeviceDataProducer { - private readonly IServiceProvider _serviceProvider; + private readonly FileLoggerProvider? _fileLoggerProvider; private readonly ConcurrentBag _messages = new(); private IServerTestHost? _serverTestHost; private static readonly string[] NewLineStrings = { "\r\n", "\n" }; - public ServerModePerCallOutputDevice(IServiceProvider serviceProvider) - => _serviceProvider = serviceProvider; + public ServerModePerCallOutputDevice(FileLoggerProvider? fileLoggerProvider, IPoliciesService policiesService) + { + _fileLoggerProvider = fileLoggerProvider; + policiesService.RegisterOnMaxFailedTestsCallback( + async _ => await DisplayAsync(this, new TextOutputDeviceData(PlatformResources.ReachedMaxFailedTestsMessage))); + } internal async Task InitializeAsync(IServerTestHost serverTestHost) { @@ -94,7 +98,7 @@ public async Task DisplayBannerAsync(string? bannerMessage) public async Task DisplayBeforeSessionStartAsync() { - if (_serviceProvider.GetService() is { FileLogger.FileName: { } logFileName }) + if (_fileLoggerProvider is { FileLogger.FileName: { } logFileName }) { await LogAsync(LogLevel.Trace, string.Format(CultureInfo.InvariantCulture, PlatformResources.StartingTestSessionWithLogFilePath, logFileName), padding: null); } diff --git a/src/Platform/Microsoft.Testing.Platform/Services/IPoliciesService.cs b/src/Platform/Microsoft.Testing.Platform/Services/IPoliciesService.cs index 811ec922d5..57a762a76d 100644 --- a/src/Platform/Microsoft.Testing.Platform/Services/IPoliciesService.cs +++ b/src/Platform/Microsoft.Testing.Platform/Services/IPoliciesService.cs @@ -5,5 +5,7 @@ namespace Microsoft.Testing.Platform.Services; internal interface IPoliciesService { - void RegisterOnStopTestExecution(Func callback); + void RegisterOnMaxFailedTestsCallback(Func callback); + + void RegisterOnAbortCallback(Func callback); } diff --git a/src/Platform/Microsoft.Testing.Platform/Services/PoliciesService.cs b/src/Platform/Microsoft.Testing.Platform/Services/PoliciesService.cs index 6c9e788100..e23b9e7553 100644 --- a/src/Platform/Microsoft.Testing.Platform/Services/PoliciesService.cs +++ b/src/Platform/Microsoft.Testing.Platform/Services/PoliciesService.cs @@ -7,21 +7,34 @@ namespace Microsoft.Testing.Platform.Services; internal sealed class PoliciesService : IPoliciesService { - private BlockingCollection>? _onStopTestExecutionCallbacks; + internal sealed class Policy + { + private BlockingCollection>? _callbacks; - public void RegisterOnStopTestExecution(Func callback) - => (_onStopTestExecutionCallbacks ??= new()).Add(callback); + public void RegisterCallback(Func callback) + => (_callbacks ??= new()).Add(callback); - internal async Task ExecuteOnStopTestExecutionCallbacks(CancellationToken cancellationToken) - { - if (_onStopTestExecutionCallbacks is null) + public async Task ExecuteCallbacksAsync(CancellationToken cancellationToken) { - return; - } + if (_callbacks is null) + { + return; + } - foreach (Func callback in _onStopTestExecutionCallbacks) - { - await callback.Invoke(cancellationToken); + foreach (Func callback in _callbacks) + { + await callback.Invoke(cancellationToken); + } } } + + internal Policy MaxFailedTestsPolicy { get; } = new(); + + internal Policy AbortPolicy { get; } = new(); + + public void RegisterOnMaxFailedTestsCallback(Func callback) + => MaxFailedTestsPolicy.RegisterCallback(callback); + + public void RegisterOnAbortCallback(Func callback) + => AbortPolicy.RegisterCallback(callback); } diff --git a/src/Platform/Microsoft.Testing.Platform/Services/TestApplicationResult.cs b/src/Platform/Microsoft.Testing.Platform/Services/TestApplicationResult.cs index c788db5cfd..e2d4e1c75d 100644 --- a/src/Platform/Microsoft.Testing.Platform/Services/TestApplicationResult.cs +++ b/src/Platform/Microsoft.Testing.Platform/Services/TestApplicationResult.cs @@ -35,16 +35,11 @@ public TestApplicationResult( _testApplicationCancellationTokenSource = testApplicationCancellationTokenSource; _commandLineOptions = commandLineOptions; _environment = environment; - policiesService.RegisterOnStopTestExecution( - async cancellationToken => + policiesService.RegisterOnMaxFailedTestsCallback( + _ => { - // How to make the message more clear that it's due to --max-failed-tests? - // The way we are currently abstracting is too generic that any one could use IStopTestExecutionCapability for whatever other reason. - // So, we can't here "hardcode" the message to be specific to --max-failed-tests. - // Do we want to introduce a "reason" parameter in IStopTestExecutionCapability.StopTestExecutionAsync? - // TODO: Localize - await _outputService.DisplayAsync(this, new TextOutputDeviceData("Test session was stopped.")); _testExecutionStopped = true; + return Task.CompletedTask; }); } diff --git a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Services/TestApplicationResultTests.cs b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Services/TestApplicationResultTests.cs index 6964e6aa45..9b95d93cf1 100644 --- a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Services/TestApplicationResultTests.cs +++ b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Services/TestApplicationResultTests.cs @@ -15,7 +15,7 @@ namespace Microsoft.Testing.Platform.UnitTests; public sealed class TestApplicationResultTests : TestBase { private readonly TestApplicationResult _testApplicationResult - = new(new Mock().Object, new Mock().Object, new Mock().Object, new Mock().Object); + = new(new Mock().Object, new Mock().Object, new Mock().Object, new Mock().Object, new Mock().Object); public TestApplicationResultTests(ITestExecutionContext testExecutionContext) : base(testExecutionContext) @@ -76,7 +76,7 @@ public async Task GetProcessExitCodeAsync_If_Canceled_Returns_TestSessionAborted }); TestApplicationResult testApplicationResult - = new(new Mock().Object, testApplicationCancellationTokenSource.Object, new Mock().Object, new Mock().Object); + = new(new Mock().Object, testApplicationCancellationTokenSource.Object, new Mock().Object, new Mock().Object, new Mock().Object); await testApplicationResult.ConsumeAsync(new DummyProducer(), new TestNodeUpdateMessage( default, @@ -110,7 +110,7 @@ public async Task GetProcessExitCodeAsync_If_MinimumExpectedTests_Violated_Retur TestApplicationResult testApplicationResult = new(new Mock().Object, new Mock().Object, new CommandLineOption(PlatformCommandLineProvider.MinimumExpectedTestsOptionKey, ["2"]), - new Mock().Object); + new Mock().Object, new Mock().Object); await testApplicationResult.ConsumeAsync(new DummyProducer(), new TestNodeUpdateMessage( default, @@ -138,7 +138,7 @@ public async Task GetProcessExitCodeAsync_OnDiscovery_No_Tests_Discovered_Return TestApplicationResult testApplicationResult = new(new Mock().Object, new Mock().Object, new CommandLineOption(PlatformCommandLineProvider.DiscoverTestsOptionKey, []), - new Mock().Object); + new Mock().Object, new Mock().Object); await testApplicationResult.ConsumeAsync(new DummyProducer(), new TestNodeUpdateMessage( default, @@ -156,7 +156,7 @@ public async Task GetProcessExitCodeAsync_OnDiscovery_Some_Tests_Discovered_Retu TestApplicationResult testApplicationResult = new(new Mock().Object, new Mock().Object, new CommandLineOption(PlatformCommandLineProvider.DiscoverTestsOptionKey, []), - new Mock().Object); + new Mock().Object, new Mock().Object); await testApplicationResult.ConsumeAsync(new DummyProducer(), new TestNodeUpdateMessage( default, @@ -190,10 +190,11 @@ public void GetProcessExitCodeAsync_IgnoreExitCodes(string argument, int expecte { new(new Mock().Object, new Mock().Object, new CommandLineOption(PlatformCommandLineProvider.IgnoreExitCodeOptionKey, argument is null ? [] : [argument]), - new Mock().Object), + new Mock().Object, new Mock().Object), new(new Mock().Object, new Mock().Object, new Mock().Object, - environment.Object), + environment.Object, + new Mock().Object), }) { Assert.AreEqual(expectedExitCode, testApplicationResult.GetProcessExitCode()); From 5ef6a172211165571c2cfe8132333ff134687049 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Fri, 6 Dec 2024 12:37:47 +0100 Subject: [PATCH 15/64] Progress --- .../Microsoft.Testing.Platform/Helpers/ExitCodes.cs | 3 ++- .../Hosts/TestHostBuilder.cs | 1 - .../OutputDevice/TerminalOutputDevice.cs | 11 ++++++++--- .../Services/CTRLPlusCCancellationTokenSource.cs | 7 ++++++- .../Services/IPoliciesService.cs | 2 +- .../Services/PoliciesService.cs | 6 ++++-- .../Services/TestApplicationResult.cs | 9 +++++---- 7 files changed, 26 insertions(+), 13 deletions(-) diff --git a/src/Platform/Microsoft.Testing.Platform/Helpers/ExitCodes.cs b/src/Platform/Microsoft.Testing.Platform/Helpers/ExitCodes.cs index 246717620e..23544d0f59 100644 --- a/src/Platform/Microsoft.Testing.Platform/Helpers/ExitCodes.cs +++ b/src/Platform/Microsoft.Testing.Platform/Helpers/ExitCodes.cs @@ -22,5 +22,6 @@ internal static class ExitCodes public const int TestAdapterTestSessionFailure = 10; public const int DependentProcessExited = 11; public const int IncompatibleProtocolVersion = 12; - public const int TestExecutionStopped = 13; + public const int TestExecutionStoppedForMaxFailedTests = 13; + public const int TestExecutionStopped = 14; } diff --git a/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs b/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs index abf35397a1..b3e8c5daf4 100644 --- a/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs +++ b/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs @@ -317,7 +317,6 @@ public async Task BuildAsync( // Register the ITestApplicationResult TestApplicationResult testApplicationResult = new( proxyOutputDevice, - serviceProvider.GetTestApplicationCancellationTokenSource(), serviceProvider.GetCommandLineOptions(), serviceProvider.GetEnvironment(), policiesService); diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.cs index 9b2882ef8c..0df439efe3 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.cs +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.cs @@ -90,7 +90,14 @@ public TerminalOutputDevice(ITestApplicationCancellationTokenSource testApplicat _clock = clock; policiesService.RegisterOnMaxFailedTestsCallback( - async _ => await DisplayAsync(this, new TextOutputDeviceData(PlatformResources.ReachedMaxFailedTestsMessage))); + async _ => await DisplayAsync(this, new TextOutputDeviceData(PlatformResources.ReachedMaxFailedTestsMessage))); + + policiesService.RegisterOnAbortCallback( + () => + { + _terminalTestReporter?.StartCancelling(); + return Task.CompletedTask; + }); if (_runtimeFeature.IsDynamicCodeSupported) { @@ -167,8 +174,6 @@ public Task InitializeAsync() ShowProgress = shouldShowProgress, }); - _testApplicationCancellationTokenSource.CancellationToken.Register(() => _terminalTestReporter.StartCancelling()); - return Task.CompletedTask; } diff --git a/src/Platform/Microsoft.Testing.Platform/Services/CTRLPlusCCancellationTokenSource.cs b/src/Platform/Microsoft.Testing.Platform/Services/CTRLPlusCCancellationTokenSource.cs index 62ce3e3186..f00952d93a 100644 --- a/src/Platform/Microsoft.Testing.Platform/Services/CTRLPlusCCancellationTokenSource.cs +++ b/src/Platform/Microsoft.Testing.Platform/Services/CTRLPlusCCancellationTokenSource.cs @@ -11,7 +11,7 @@ internal class CTRLPlusCCancellationTokenSource : ITestApplicationCancellationTo private readonly CancellationTokenSource _cancellationTokenSource = new(); private readonly ILogger? _logger; - public CTRLPlusCCancellationTokenSource(IConsole? console = null, ILogger? logger = null) + public CTRLPlusCCancellationTokenSource(IConsole? console = null, ILogger? logger = null, PoliciesService? policiesService = null) { if (console is not null) { @@ -19,6 +19,11 @@ public CTRLPlusCCancellationTokenSource(IConsole? console = null, ILogger? logge } _logger = logger; + if (policiesService is not null) + { + // TODO: Discuss about what we pass here as cancellation token. + _cancellationTokenSource.Token.Register(() => policiesService.AbortPolicy.ExecuteCallbacksAsync(cancellationToken: default).GetAwaiter().GetResult()); + } } public void CancelAfter(TimeSpan timeout) => _cancellationTokenSource.CancelAfter(timeout); diff --git a/src/Platform/Microsoft.Testing.Platform/Services/IPoliciesService.cs b/src/Platform/Microsoft.Testing.Platform/Services/IPoliciesService.cs index 57a762a76d..e212f54004 100644 --- a/src/Platform/Microsoft.Testing.Platform/Services/IPoliciesService.cs +++ b/src/Platform/Microsoft.Testing.Platform/Services/IPoliciesService.cs @@ -7,5 +7,5 @@ internal interface IPoliciesService { void RegisterOnMaxFailedTestsCallback(Func callback); - void RegisterOnAbortCallback(Func callback); + void RegisterOnAbortCallback(Func callback); } diff --git a/src/Platform/Microsoft.Testing.Platform/Services/PoliciesService.cs b/src/Platform/Microsoft.Testing.Platform/Services/PoliciesService.cs index e23b9e7553..0ec071ff6d 100644 --- a/src/Platform/Microsoft.Testing.Platform/Services/PoliciesService.cs +++ b/src/Platform/Microsoft.Testing.Platform/Services/PoliciesService.cs @@ -11,6 +11,8 @@ internal sealed class Policy { private BlockingCollection>? _callbacks; + public bool IsPolicyTriggered { get; private set; } + public void RegisterCallback(Func callback) => (_callbacks ??= new()).Add(callback); @@ -35,6 +37,6 @@ public async Task ExecuteCallbacksAsync(CancellationToken cancellationToken) public void RegisterOnMaxFailedTestsCallback(Func callback) => MaxFailedTestsPolicy.RegisterCallback(callback); - public void RegisterOnAbortCallback(Func callback) - => AbortPolicy.RegisterCallback(callback); + public void RegisterOnAbortCallback(Func callback) + => AbortPolicy.RegisterCallback(_ => callback()); } diff --git a/src/Platform/Microsoft.Testing.Platform/Services/TestApplicationResult.cs b/src/Platform/Microsoft.Testing.Platform/Services/TestApplicationResult.cs index e2d4e1c75d..71125c9a67 100644 --- a/src/Platform/Microsoft.Testing.Platform/Services/TestApplicationResult.cs +++ b/src/Platform/Microsoft.Testing.Platform/Services/TestApplicationResult.cs @@ -16,9 +16,9 @@ namespace Microsoft.Testing.Platform.Services; internal sealed class TestApplicationResult : ITestApplicationProcessExitCode, IOutputDeviceDataProducer { private readonly IOutputDevice _outputService; - private readonly ITestApplicationCancellationTokenSource _testApplicationCancellationTokenSource; private readonly ICommandLineOptions _commandLineOptions; private readonly IEnvironment _environment; + private readonly PoliciesService _policiesService; private readonly List _failedTests = []; private int _totalRanTests; private bool _testAdapterTestSessionFailure; @@ -26,15 +26,15 @@ internal sealed class TestApplicationResult : ITestApplicationProcessExitCode, I public TestApplicationResult( IOutputDevice outputService, - ITestApplicationCancellationTokenSource testApplicationCancellationTokenSource, ICommandLineOptions commandLineOptions, IEnvironment environment, PoliciesService policiesService) { _outputService = outputService; - _testApplicationCancellationTokenSource = testApplicationCancellationTokenSource; _commandLineOptions = commandLineOptions; _environment = environment; + _policiesService = policiesService; + policiesService.RegisterOnMaxFailedTestsCallback( _ => { @@ -97,10 +97,11 @@ public Task ConsumeAsync(IDataProducer dataProducer, IData value, CancellationTo public int GetProcessExitCode() { int exitCode = ExitCodes.Success; + exitCode = exitCode == ExitCodes.Success && _policiesService.MaxFailedTestsPolicy.IsPolicyTriggered ? ExitCodes.TestExecutionStoppedForMaxFailedTests : exitCode; exitCode = exitCode == ExitCodes.Success && _testExecutionStopped ? ExitCodes.TestExecutionStopped : exitCode; exitCode = exitCode == ExitCodes.Success && _testAdapterTestSessionFailure ? ExitCodes.TestAdapterTestSessionFailure : exitCode; exitCode = exitCode == ExitCodes.Success && _failedTests.Count > 0 ? ExitCodes.AtLeastOneTestFailed : exitCode; - exitCode = exitCode == ExitCodes.Success && _testApplicationCancellationTokenSource.CancellationToken.IsCancellationRequested ? ExitCodes.TestSessionAborted : exitCode; + exitCode = exitCode == ExitCodes.Success && _policiesService.AbortPolicy.IsPolicyTriggered ? ExitCodes.TestSessionAborted : exitCode; // If the user has specified the VSTestAdapterMode option, then we don't want to return a non-zero exit code if no tests ran. if (!_commandLineOptions.IsOptionSet(PlatformCommandLineProvider.VSTestAdapterModeOptionKey)) From c53c7d59ad4e958a6a3552dd5cc68219f35649ba Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Fri, 6 Dec 2024 12:42:58 +0100 Subject: [PATCH 16/64] Remove done TODO --- .../Services/CTRLPlusCCancellationTokenSource.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Platform/Microsoft.Testing.Platform/Services/CTRLPlusCCancellationTokenSource.cs b/src/Platform/Microsoft.Testing.Platform/Services/CTRLPlusCCancellationTokenSource.cs index 2b2f499ffc..ee64a12169 100644 --- a/src/Platform/Microsoft.Testing.Platform/Services/CTRLPlusCCancellationTokenSource.cs +++ b/src/Platform/Microsoft.Testing.Platform/Services/CTRLPlusCCancellationTokenSource.cs @@ -21,7 +21,6 @@ public CTRLPlusCCancellationTokenSource(IConsole? console = null, ILogger? logge _logger = logger; if (policiesService is not null) { - // TODO: Discuss about what we pass here as cancellation token. _cancellationTokenSource.Token.Register(() => policiesService.AbortPolicy.ExecuteCallbacksAsync(cancellationToken: default).GetAwaiter().GetResult()); } } From 757153171e5d2f80746f06ea181230f1c41c5981 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Fri, 6 Dec 2024 12:45:43 +0100 Subject: [PATCH 17/64] Cleanup --- .../Microsoft.Testing.Platform/Helpers/ExitCodes.cs | 1 - .../Services/TestApplicationResult.cs | 9 --------- 2 files changed, 10 deletions(-) diff --git a/src/Platform/Microsoft.Testing.Platform/Helpers/ExitCodes.cs b/src/Platform/Microsoft.Testing.Platform/Helpers/ExitCodes.cs index 23544d0f59..79e8fd0864 100644 --- a/src/Platform/Microsoft.Testing.Platform/Helpers/ExitCodes.cs +++ b/src/Platform/Microsoft.Testing.Platform/Helpers/ExitCodes.cs @@ -23,5 +23,4 @@ internal static class ExitCodes public const int DependentProcessExited = 11; public const int IncompatibleProtocolVersion = 12; public const int TestExecutionStoppedForMaxFailedTests = 13; - public const int TestExecutionStopped = 14; } diff --git a/src/Platform/Microsoft.Testing.Platform/Services/TestApplicationResult.cs b/src/Platform/Microsoft.Testing.Platform/Services/TestApplicationResult.cs index 71125c9a67..33a245456f 100644 --- a/src/Platform/Microsoft.Testing.Platform/Services/TestApplicationResult.cs +++ b/src/Platform/Microsoft.Testing.Platform/Services/TestApplicationResult.cs @@ -22,7 +22,6 @@ internal sealed class TestApplicationResult : ITestApplicationProcessExitCode, I private readonly List _failedTests = []; private int _totalRanTests; private bool _testAdapterTestSessionFailure; - private bool _testExecutionStopped; public TestApplicationResult( IOutputDevice outputService, @@ -34,13 +33,6 @@ public TestApplicationResult( _commandLineOptions = commandLineOptions; _environment = environment; _policiesService = policiesService; - - policiesService.RegisterOnMaxFailedTestsCallback( - _ => - { - _testExecutionStopped = true; - return Task.CompletedTask; - }); } /// @@ -98,7 +90,6 @@ public int GetProcessExitCode() { int exitCode = ExitCodes.Success; exitCode = exitCode == ExitCodes.Success && _policiesService.MaxFailedTestsPolicy.IsPolicyTriggered ? ExitCodes.TestExecutionStoppedForMaxFailedTests : exitCode; - exitCode = exitCode == ExitCodes.Success && _testExecutionStopped ? ExitCodes.TestExecutionStopped : exitCode; exitCode = exitCode == ExitCodes.Success && _testAdapterTestSessionFailure ? ExitCodes.TestAdapterTestSessionFailure : exitCode; exitCode = exitCode == ExitCodes.Success && _failedTests.Count > 0 ? ExitCodes.AtLeastOneTestFailed : exitCode; exitCode = exitCode == ExitCodes.Success && _policiesService.AbortPolicy.IsPolicyTriggered ? ExitCodes.TestSessionAborted : exitCode; From 1d245ecf5b0554699892a39600d1b92d88f6cb04 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Fri, 6 Dec 2024 13:15:17 +0100 Subject: [PATCH 18/64] Small fixes --- .../Services/TestApplicationResultTests.cs | 38 ++++++++++++------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Services/TestApplicationResultTests.cs b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Services/TestApplicationResultTests.cs index 9b95d93cf1..23b2b18d50 100644 --- a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Services/TestApplicationResultTests.cs +++ b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Services/TestApplicationResultTests.cs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +using System.Threading; + using Microsoft.Testing.Platform.CommandLine; using Microsoft.Testing.Platform.Extensions.Messages; using Microsoft.Testing.Platform.Helpers; @@ -15,7 +17,7 @@ namespace Microsoft.Testing.Platform.UnitTests; public sealed class TestApplicationResultTests : TestBase { private readonly TestApplicationResult _testApplicationResult - = new(new Mock().Object, new Mock().Object, new Mock().Object, new Mock().Object, new Mock().Object); + = new(new Mock().Object, new Mock().Object, new Mock().Object, new Mock().Object); public TestApplicationResultTests(ITestExecutionContext testExecutionContext) : base(testExecutionContext) @@ -68,15 +70,20 @@ public async Task GetProcessExitCodeAsync_If_Failed_Tests_Returns_AtLeastOneTest public async Task GetProcessExitCodeAsync_If_Canceled_Returns_TestSessionAborted() { Mock testApplicationCancellationTokenSource = new(); + var policiesService = new PoliciesService(); testApplicationCancellationTokenSource.SetupGet(x => x.CancellationToken).Returns(() => { CancellationTokenSource cancellationTokenSource = new(); + + // In reality, this is done by CTRLPlusCCancellationTokenSource (the real production impl of ITestApplicationCancellationTokenSource). + cancellationTokenSource.Token.Register(() => policiesService.AbortPolicy.ExecuteCallbacksAsync(cancellationToken: default).GetAwaiter().GetResult()); + cancellationTokenSource.Cancel(); return cancellationTokenSource.Token; }); TestApplicationResult testApplicationResult - = new(new Mock().Object, testApplicationCancellationTokenSource.Object, new Mock().Object, new Mock().Object, new Mock().Object); + = new(new Mock().Object, new Mock().Object, new Mock().Object, policiesService); await testApplicationResult.ConsumeAsync(new DummyProducer(), new TestNodeUpdateMessage( default, @@ -108,9 +115,10 @@ public async Task GetProcessExitCodeAsync_If_TestAdapter_Returns_TestAdapterTest public async Task GetProcessExitCodeAsync_If_MinimumExpectedTests_Violated_Returns_MinimumExpectedTestsPolicyViolation() { TestApplicationResult testApplicationResult - = new(new Mock().Object, new Mock().Object, - new CommandLineOption(PlatformCommandLineProvider.MinimumExpectedTestsOptionKey, ["2"]), - new Mock().Object, new Mock().Object); + = new( + new Mock().Object, + new CommandLineOption(PlatformCommandLineProvider.MinimumExpectedTestsOptionKey, ["2"]), + new Mock().Object, new Mock().Object); await testApplicationResult.ConsumeAsync(new DummyProducer(), new TestNodeUpdateMessage( default, @@ -136,9 +144,10 @@ TestApplicationResult testApplicationResult public async Task GetProcessExitCodeAsync_OnDiscovery_No_Tests_Discovered_Returns_ZeroTests() { TestApplicationResult testApplicationResult - = new(new Mock().Object, new Mock().Object, - new CommandLineOption(PlatformCommandLineProvider.DiscoverTestsOptionKey, []), - new Mock().Object, new Mock().Object); + = new( + new Mock().Object, + new CommandLineOption(PlatformCommandLineProvider.DiscoverTestsOptionKey, []), + new Mock().Object, new Mock().Object); await testApplicationResult.ConsumeAsync(new DummyProducer(), new TestNodeUpdateMessage( default, @@ -154,9 +163,10 @@ TestApplicationResult testApplicationResult public async Task GetProcessExitCodeAsync_OnDiscovery_Some_Tests_Discovered_Returns_Success() { TestApplicationResult testApplicationResult - = new(new Mock().Object, new Mock().Object, - new CommandLineOption(PlatformCommandLineProvider.DiscoverTestsOptionKey, []), - new Mock().Object, new Mock().Object); + = new( + new Mock().Object, + new CommandLineOption(PlatformCommandLineProvider.DiscoverTestsOptionKey, []), + new Mock().Object, new Mock().Object); await testApplicationResult.ConsumeAsync(new DummyProducer(), new TestNodeUpdateMessage( default, @@ -188,10 +198,12 @@ public void GetProcessExitCodeAsync_IgnoreExitCodes(string argument, int expecte foreach (TestApplicationResult testApplicationResult in new TestApplicationResult[] { - new(new Mock().Object, new Mock().Object, + new( + new Mock().Object, new CommandLineOption(PlatformCommandLineProvider.IgnoreExitCodeOptionKey, argument is null ? [] : [argument]), new Mock().Object, new Mock().Object), - new(new Mock().Object, new Mock().Object, + new( + new Mock().Object, new Mock().Object, environment.Object, new Mock().Object), From 993f94977ac7974084a512721913b1c75c81843f Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Fri, 6 Dec 2024 13:20:10 +0100 Subject: [PATCH 19/64] Experimental --- ...MSTestStopGracefullyTestExecutionCapability.cs | 15 +++++++++++++++ .../TestApplicationBuilderExtensions.cs | 3 ++- .../IStopGracefullyTestExecutionCapability.cs | 3 +++ .../PublicAPI/PublicAPI.Unshipped.txt | 4 ++-- 4 files changed, 22 insertions(+), 3 deletions(-) create mode 100644 src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/MSTestStopGracefullyTestExecutionCapability.cs diff --git a/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/MSTestStopGracefullyTestExecutionCapability.cs b/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/MSTestStopGracefullyTestExecutionCapability.cs new file mode 100644 index 0000000000..1da5bc3583 --- /dev/null +++ b/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/MSTestStopGracefullyTestExecutionCapability.cs @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Platform.Capabilities.TestFramework; + +namespace Microsoft.VisualStudio.TestTools.UnitTesting; + +#pragma warning disable TPEXP // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. +internal sealed class MSTestStopGracefullyTestExecutionCapability : IStopGracefullyTestExecutionCapability +#pragma warning restore TPEXP // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. +{ + // TODO: + public Task StopTestExecutionAsync(CancellationToken cancellationToken) + => Task.CompletedTask; +} diff --git a/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/TestApplicationBuilderExtensions.cs b/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/TestApplicationBuilderExtensions.cs index 2a86adaf09..65bc6446db 100644 --- a/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/TestApplicationBuilderExtensions.cs +++ b/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/TestApplicationBuilderExtensions.cs @@ -25,7 +25,8 @@ public static void AddMSTest(this ITestApplicationBuilder testApplicationBuilder serviceProvider => new TestFrameworkCapabilities( new VSTestBridgeExtensionBaseCapabilities(), #pragma warning disable TPEXP // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. - new MSTestBannerCapability(serviceProvider.GetRequiredService())), + new MSTestBannerCapability(serviceProvider.GetRequiredService()), + new MSTestStopGracefullyTestExecutionCapability()), #pragma warning restore TPEXP // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. (capabilities, serviceProvider) => new MSTestBridgedTestFramework(extension, getTestAssemblies, serviceProvider, capabilities)); } diff --git a/src/Platform/Microsoft.Testing.Platform/Capabilities/TestFramework/IStopGracefullyTestExecutionCapability.cs b/src/Platform/Microsoft.Testing.Platform/Capabilities/TestFramework/IStopGracefullyTestExecutionCapability.cs index 6bde8a6ec5..8b4d0b69e2 100644 --- a/src/Platform/Microsoft.Testing.Platform/Capabilities/TestFramework/IStopGracefullyTestExecutionCapability.cs +++ b/src/Platform/Microsoft.Testing.Platform/Capabilities/TestFramework/IStopGracefullyTestExecutionCapability.cs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +using System.Diagnostics.CodeAnalysis; + namespace Microsoft.Testing.Platform.Capabilities.TestFramework; /// @@ -10,6 +12,7 @@ namespace Microsoft.Testing.Platform.Capabilities.TestFramework; /// /// Test frameworks can choose to run any needed cleanup when cancellation is requested. /// +[Experimental("TPEXP", UrlFormat = "https://aka.ms/testingplatform/diagnostics#{0}")] public interface IStopGracefullyTestExecutionCapability : ITestFrameworkCapability { Task StopTestExecutionAsync(CancellationToken cancellationToken); diff --git a/src/Platform/Microsoft.Testing.Platform/PublicAPI/PublicAPI.Unshipped.txt b/src/Platform/Microsoft.Testing.Platform/PublicAPI/PublicAPI.Unshipped.txt index 89d7a5d813..18566ee9e9 100644 --- a/src/Platform/Microsoft.Testing.Platform/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/Platform/Microsoft.Testing.Platform/PublicAPI/PublicAPI.Unshipped.txt @@ -1,6 +1,4 @@ #nullable enable -Microsoft.Testing.Platform.Capabilities.TestFramework.IStopGracefullyTestExecutionCapability -Microsoft.Testing.Platform.Capabilities.TestFramework.IStopGracefullyTestExecutionCapability.StopTestExecutionAsync(System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task! Microsoft.Testing.Platform.Extensions.Messages.TestMetadataProperty.TestMetadataProperty(string! key) -> void Microsoft.Testing.Platform.OutputDevice.ErrorMessageOutputDeviceData Microsoft.Testing.Platform.OutputDevice.ErrorMessageOutputDeviceData.ErrorMessageOutputDeviceData(string! message) -> void @@ -8,6 +6,8 @@ Microsoft.Testing.Platform.OutputDevice.ErrorMessageOutputDeviceData.Message.get Microsoft.Testing.Platform.OutputDevice.WarningMessageOutputDeviceData Microsoft.Testing.Platform.OutputDevice.WarningMessageOutputDeviceData.Message.get -> string! Microsoft.Testing.Platform.OutputDevice.WarningMessageOutputDeviceData.WarningMessageOutputDeviceData(string! message) -> void +[TPEXP]Microsoft.Testing.Platform.Capabilities.TestFramework.IStopGracefullyTestExecutionCapability +[TPEXP]Microsoft.Testing.Platform.Capabilities.TestFramework.IStopGracefullyTestExecutionCapability.StopTestExecutionAsync(System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task! [TPEXP]Microsoft.Testing.Platform.Extensions.Messages.StandardOutputProperty [TPEXP]Microsoft.Testing.Platform.Extensions.Messages.StandardOutputProperty.StandardOutput.get -> string! [TPEXP]Microsoft.Testing.Platform.Extensions.Messages.StandardOutputProperty.StandardOutput.init -> void From 5496617dd3ee663e7fb914050c03d3e64c46a18c Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Fri, 6 Dec 2024 13:31:52 +0100 Subject: [PATCH 20/64] Register only when enabled --- .../Hosts/TestHostBuilder.cs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs b/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs index 4d39d34490..53ff51e411 100644 --- a/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs +++ b/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs @@ -728,12 +728,16 @@ private async Task BuildTestFrameworkAsync(TestFrameworkBuilderD dataConsumersBuilder.Add(pushOnlyProtocolDataConsumer); } - dataConsumersBuilder.Add( - new AbortForMaxFailedTestsExtension( - serviceProvider.GetCommandLineOptions(), - serviceProvider.GetTestFrameworkCapabilities().GetCapability(), - serviceProvider.GetRequiredService(), - serviceProvider.GetTestApplicationCancellationTokenSource().CancellationToken)); + var abortForMaxFailedTestsExtension = new AbortForMaxFailedTestsExtension( + serviceProvider.GetCommandLineOptions(), + serviceProvider.GetTestFrameworkCapabilities().GetCapability(), + serviceProvider.GetRequiredService(), + serviceProvider.GetTestApplicationCancellationTokenSource().CancellationToken); + + if (await abortForMaxFailedTestsExtension.IsEnabledAsync()) + { + dataConsumersBuilder.Add(abortForMaxFailedTestsExtension); + } IDataConsumer[] dataConsumerServices = dataConsumersBuilder.ToArray(); From db426ed32df9d9daef07e7a4254686c8f1f4ded3 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Fri, 6 Dec 2024 13:54:59 +0100 Subject: [PATCH 21/64] Fix build --- .../Services/TestApplicationResultTests.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Services/TestApplicationResultTests.cs b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Services/TestApplicationResultTests.cs index 23b2b18d50..d33133fa1b 100644 --- a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Services/TestApplicationResultTests.cs +++ b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Services/TestApplicationResultTests.cs @@ -1,8 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using System.Threading; - using Microsoft.Testing.Platform.CommandLine; using Microsoft.Testing.Platform.Extensions.Messages; using Microsoft.Testing.Platform.Helpers; From f66be1d0af6562f095b07ff05b9306a6ecec00e3 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Fri, 6 Dec 2024 14:14:30 +0100 Subject: [PATCH 22/64] Small progress --- ...MSTestStopGracefullyTestExecutionCapability.cs | 15 +++++++++++++-- .../TestApplicationBuilderExtensions.cs | 2 +- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/MSTestStopGracefullyTestExecutionCapability.cs b/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/MSTestStopGracefullyTestExecutionCapability.cs index 1da5bc3583..527c38dc13 100644 --- a/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/MSTestStopGracefullyTestExecutionCapability.cs +++ b/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/MSTestStopGracefullyTestExecutionCapability.cs @@ -9,7 +9,18 @@ namespace Microsoft.VisualStudio.TestTools.UnitTesting; internal sealed class MSTestStopGracefullyTestExecutionCapability : IStopGracefullyTestExecutionCapability #pragma warning restore TPEXP // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. { - // TODO: + private MSTestStopGracefullyTestExecutionCapability() + { + } + + public static MSTestStopGracefullyTestExecutionCapability Instance { get; } = new(); + + // TODO: Respect this properly to ensure cleanups are run. + public bool IsStopRequested { get; private set; } + public Task StopTestExecutionAsync(CancellationToken cancellationToken) - => Task.CompletedTask; + { + IsStopRequested = true; + return Task.CompletedTask; + } } diff --git a/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/TestApplicationBuilderExtensions.cs b/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/TestApplicationBuilderExtensions.cs index 65bc6446db..05d2fa2481 100644 --- a/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/TestApplicationBuilderExtensions.cs +++ b/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/TestApplicationBuilderExtensions.cs @@ -26,7 +26,7 @@ public static void AddMSTest(this ITestApplicationBuilder testApplicationBuilder new VSTestBridgeExtensionBaseCapabilities(), #pragma warning disable TPEXP // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. new MSTestBannerCapability(serviceProvider.GetRequiredService()), - new MSTestStopGracefullyTestExecutionCapability()), + MSTestStopGracefullyTestExecutionCapability.Instance), #pragma warning restore TPEXP // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. (capabilities, serviceProvider) => new MSTestBridgedTestFramework(extension, getTestAssemblies, serviceProvider, capabilities)); } From 3240e68617f4a80bd83d57d0dee7ba112a34d894 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Fri, 6 Dec 2024 14:33:10 +0100 Subject: [PATCH 23/64] Renames --- ...stStopGracefullyTestExecutionCapability.cs | 2 +- ...> IGracefulStopTestExecutionCapability.cs} | 2 +- .../AbortForMaxFailedTestsExtension.cs | 20 +++++++++++-------- .../Hosts/TestHostBuilder.cs | 6 +++--- .../OutputDevice/OutputDeviceManager.cs | 4 ++-- .../OutputDevice/TerminalOutputDevice.cs | 2 +- .../PublicAPI/PublicAPI.Unshipped.txt | 2 ++ .../JsonRpc/ServerModePerCallOutputDevice.cs | 2 +- .../CTRLPlusCCancellationTokenSource.cs | 2 +- ...ciesService.cs => IStopPoliciesService.cs} | 2 +- ...iciesService.cs => StopPoliciesService.cs} | 4 ++-- .../Services/TestApplicationResult.cs | 8 ++++---- .../Services/TestApplicationResultTests.cs | 14 ++++++------- 13 files changed, 38 insertions(+), 32 deletions(-) rename src/Platform/Microsoft.Testing.Platform/Capabilities/TestFramework/{IStopGracefullyTestExecutionCapability.cs => IGracefulStopTestExecutionCapability.cs} (89%) rename src/Platform/Microsoft.Testing.Platform/Services/{IPoliciesService.cs => IStopPoliciesService.cs} (89%) rename src/Platform/Microsoft.Testing.Platform/Services/{PoliciesService.cs => StopPoliciesService.cs} (91%) diff --git a/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/MSTestStopGracefullyTestExecutionCapability.cs b/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/MSTestStopGracefullyTestExecutionCapability.cs index 527c38dc13..8aa631b204 100644 --- a/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/MSTestStopGracefullyTestExecutionCapability.cs +++ b/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/MSTestStopGracefullyTestExecutionCapability.cs @@ -6,7 +6,7 @@ namespace Microsoft.VisualStudio.TestTools.UnitTesting; #pragma warning disable TPEXP // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. -internal sealed class MSTestStopGracefullyTestExecutionCapability : IStopGracefullyTestExecutionCapability +internal sealed class MSTestStopGracefullyTestExecutionCapability : IGracefulStopTestExecutionCapability #pragma warning restore TPEXP // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. { private MSTestStopGracefullyTestExecutionCapability() diff --git a/src/Platform/Microsoft.Testing.Platform/Capabilities/TestFramework/IStopGracefullyTestExecutionCapability.cs b/src/Platform/Microsoft.Testing.Platform/Capabilities/TestFramework/IGracefulStopTestExecutionCapability.cs similarity index 89% rename from src/Platform/Microsoft.Testing.Platform/Capabilities/TestFramework/IStopGracefullyTestExecutionCapability.cs rename to src/Platform/Microsoft.Testing.Platform/Capabilities/TestFramework/IGracefulStopTestExecutionCapability.cs index 8b4d0b69e2..6f0815e440 100644 --- a/src/Platform/Microsoft.Testing.Platform/Capabilities/TestFramework/IStopGracefullyTestExecutionCapability.cs +++ b/src/Platform/Microsoft.Testing.Platform/Capabilities/TestFramework/IGracefulStopTestExecutionCapability.cs @@ -13,7 +13,7 @@ namespace Microsoft.Testing.Platform.Capabilities.TestFramework; /// Test frameworks can choose to run any needed cleanup when cancellation is requested. /// [Experimental("TPEXP", UrlFormat = "https://aka.ms/testingplatform/diagnostics#{0}")] -public interface IStopGracefullyTestExecutionCapability : ITestFrameworkCapability +public interface IGracefulStopTestExecutionCapability : ITestFrameworkCapability { Task StopTestExecutionAsync(CancellationToken cancellationToken); } diff --git a/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs b/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs index 6c3581b1ed..3ae2516561 100644 --- a/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs +++ b/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs @@ -15,12 +15,16 @@ namespace Microsoft.Testing.Platform.Extensions; internal sealed class AbortForMaxFailedTestsExtension : IDataConsumer { private readonly int? _maxFailedTests; - private readonly IStopGracefullyTestExecutionCapability? _capability; - private readonly PoliciesService _policiesService; - private readonly CancellationToken _cancellationToken; + private readonly IGracefulStopTestExecutionCapability? _capability; + private readonly StopPoliciesService _policiesService; + private readonly ITestApplicationCancellationTokenSource _testApplicationCancellationTokenSource; private int _failCount; - public AbortForMaxFailedTestsExtension(ICommandLineOptions commandLineOptions, IStopGracefullyTestExecutionCapability? capability, PoliciesService policiesService, CancellationToken cancellationToken) + public AbortForMaxFailedTestsExtension( + ICommandLineOptions commandLineOptions, + IGracefulStopTestExecutionCapability? capability, + StopPoliciesService policiesService, + ITestApplicationCancellationTokenSource testApplicationCancellationTokenSource) { if (commandLineOptions.TryGetOptionArgumentList(PlatformCommandLineProvider.MaxFailedTestsOptionKey, out string[]? args) && int.TryParse(args[0], out int maxFailedTests) && @@ -31,7 +35,7 @@ public AbortForMaxFailedTestsExtension(ICommandLineOptions commandLineOptions, I _capability = capability; _policiesService = policiesService; - _cancellationToken = cancellationToken; + _testApplicationCancellationTokenSource = testApplicationCancellationTokenSource; } public Type[] DataTypesConsumed { get; } = [typeof(TestNodeUpdateMessage)]; @@ -55,7 +59,7 @@ public async Task ConsumeAsync(IDataProducer dataProducer, IData value, Cancella { var node = (TestNodeUpdateMessage)value; - // If we are called, the extension is enabled, which means both _maxFailedTests and _cancellationTokenSource are not null. + // If we are called, the extension is enabled, which means both _maxFailedTests and capability are not null. RoslynDebug.Assert(_maxFailedTests is not null); RoslynDebug.Assert(_capability is not null); @@ -63,8 +67,8 @@ public async Task ConsumeAsync(IDataProducer dataProducer, IData value, Cancella if (TestNodePropertiesCategories.WellKnownTestNodeTestRunOutcomeFailedProperties.Any(t => t == testNodeStateProperty.GetType()) && ++_failCount > _maxFailedTests.Value) { - await _capability.StopTestExecutionAsync(_cancellationToken); - await _policiesService.MaxFailedTestsPolicy.ExecuteCallbacksAsync(_cancellationToken); + await _capability.StopTestExecutionAsync(_testApplicationCancellationTokenSource.CancellationToken); + await _policiesService.MaxFailedTestsPolicy.ExecuteCallbacksAsync(_testApplicationCancellationTokenSource.CancellationToken); } } } diff --git a/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs b/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs index 53ff51e411..106b49a532 100644 --- a/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs +++ b/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs @@ -207,7 +207,7 @@ public async Task BuildAsync( commandLineOptionsProxy.SetCommandLineOptions(commandLineHandler); // This is needed by output device. - var policiesService = new PoliciesService(); + var policiesService = new StopPoliciesService(); serviceProvider.AddService(policiesService); bool hasServerFlag = commandLineHandler.TryGetOptionArgumentList(PlatformCommandLineProvider.ServerOptionKey, out string[]? protocolName); @@ -730,8 +730,8 @@ private async Task BuildTestFrameworkAsync(TestFrameworkBuilderD var abortForMaxFailedTestsExtension = new AbortForMaxFailedTestsExtension( serviceProvider.GetCommandLineOptions(), - serviceProvider.GetTestFrameworkCapabilities().GetCapability(), - serviceProvider.GetRequiredService(), + serviceProvider.GetTestFrameworkCapabilities().GetCapability(), + serviceProvider.GetRequiredService(), serviceProvider.GetTestApplicationCancellationTokenSource().CancellationToken); if (await abortForMaxFailedTestsExtension.IsEnabledAsync()) diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/OutputDeviceManager.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/OutputDeviceManager.cs index 38ab778132..78d4dafb0f 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/OutputDeviceManager.cs +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/OutputDeviceManager.cs @@ -36,7 +36,7 @@ internal async Task BuildAsync(ServiceProvider serviceProvide useServerModeOutputDevice ? new ServerModePerCallOutputDevice( serviceProvider.GetService(), - serviceProvider.GetRequiredService()) + serviceProvider.GetRequiredService()) : null); } @@ -55,5 +55,5 @@ public static TerminalOutputDevice GetDefaultTerminalOutputDevice(ServiceProvide serviceProvider.GetFileLoggerInformation(), serviceProvider.GetLoggerFactory(), serviceProvider.GetClock(), - serviceProvider.GetRequiredService()); + serviceProvider.GetRequiredService()); } diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.cs index 0b1f4bda1c..823ddb5068 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.cs +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.cs @@ -73,7 +73,7 @@ public TerminalOutputDevice(ITestApplicationCancellationTokenSource testApplicat ITestApplicationModuleInfo testApplicationModuleInfo, ITestHostControllerInfo testHostControllerInfo, IAsyncMonitor asyncMonitor, IRuntimeFeature runtimeFeature, IEnvironment environment, IProcessHandler process, IPlatformInformation platformInformation, ICommandLineOptions commandLineOptions, IFileLoggerInformation? fileLoggerInformation, ILoggerFactory loggerFactory, IClock clock, - IPoliciesService policiesService) + IStopPoliciesService policiesService) { _testApplicationCancellationTokenSource = testApplicationCancellationTokenSource; _console = console; diff --git a/src/Platform/Microsoft.Testing.Platform/PublicAPI/PublicAPI.Unshipped.txt b/src/Platform/Microsoft.Testing.Platform/PublicAPI/PublicAPI.Unshipped.txt index 18566ee9e9..1df3bd9c6c 100644 --- a/src/Platform/Microsoft.Testing.Platform/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/Platform/Microsoft.Testing.Platform/PublicAPI/PublicAPI.Unshipped.txt @@ -6,6 +6,8 @@ Microsoft.Testing.Platform.OutputDevice.ErrorMessageOutputDeviceData.Message.get Microsoft.Testing.Platform.OutputDevice.WarningMessageOutputDeviceData Microsoft.Testing.Platform.OutputDevice.WarningMessageOutputDeviceData.Message.get -> string! Microsoft.Testing.Platform.OutputDevice.WarningMessageOutputDeviceData.WarningMessageOutputDeviceData(string! message) -> void +[TPEXP]Microsoft.Testing.Platform.Capabilities.TestFramework.IGracefulStopTestExecutionCapability +[TPEXP]Microsoft.Testing.Platform.Capabilities.TestFramework.IGracefulStopTestExecutionCapability.StopTestExecutionAsync(System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task! [TPEXP]Microsoft.Testing.Platform.Capabilities.TestFramework.IStopGracefullyTestExecutionCapability [TPEXP]Microsoft.Testing.Platform.Capabilities.TestFramework.IStopGracefullyTestExecutionCapability.StopTestExecutionAsync(System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task! [TPEXP]Microsoft.Testing.Platform.Extensions.Messages.StandardOutputProperty diff --git a/src/Platform/Microsoft.Testing.Platform/ServerMode/JsonRpc/ServerModePerCallOutputDevice.cs b/src/Platform/Microsoft.Testing.Platform/ServerMode/JsonRpc/ServerModePerCallOutputDevice.cs index 8593575ffc..3c01a65dee 100644 --- a/src/Platform/Microsoft.Testing.Platform/ServerMode/JsonRpc/ServerModePerCallOutputDevice.cs +++ b/src/Platform/Microsoft.Testing.Platform/ServerMode/JsonRpc/ServerModePerCallOutputDevice.cs @@ -24,7 +24,7 @@ internal sealed class ServerModePerCallOutputDevice : IPlatformOutputDevice, IOu private static readonly string[] NewLineStrings = { "\r\n", "\n" }; - public ServerModePerCallOutputDevice(FileLoggerProvider? fileLoggerProvider, IPoliciesService policiesService) + public ServerModePerCallOutputDevice(FileLoggerProvider? fileLoggerProvider, IStopPoliciesService policiesService) { _fileLoggerProvider = fileLoggerProvider; policiesService.RegisterOnMaxFailedTestsCallback( diff --git a/src/Platform/Microsoft.Testing.Platform/Services/CTRLPlusCCancellationTokenSource.cs b/src/Platform/Microsoft.Testing.Platform/Services/CTRLPlusCCancellationTokenSource.cs index ee64a12169..2a94619a68 100644 --- a/src/Platform/Microsoft.Testing.Platform/Services/CTRLPlusCCancellationTokenSource.cs +++ b/src/Platform/Microsoft.Testing.Platform/Services/CTRLPlusCCancellationTokenSource.cs @@ -11,7 +11,7 @@ internal sealed class CTRLPlusCCancellationTokenSource : ITestApplicationCancell private readonly CancellationTokenSource _cancellationTokenSource = new(); private readonly ILogger? _logger; - public CTRLPlusCCancellationTokenSource(IConsole? console = null, ILogger? logger = null, PoliciesService? policiesService = null) + public CTRLPlusCCancellationTokenSource(IConsole? console = null, ILogger? logger = null, StopPoliciesService? policiesService = null) { if (console is not null) { diff --git a/src/Platform/Microsoft.Testing.Platform/Services/IPoliciesService.cs b/src/Platform/Microsoft.Testing.Platform/Services/IStopPoliciesService.cs similarity index 89% rename from src/Platform/Microsoft.Testing.Platform/Services/IPoliciesService.cs rename to src/Platform/Microsoft.Testing.Platform/Services/IStopPoliciesService.cs index e212f54004..e92b744b6f 100644 --- a/src/Platform/Microsoft.Testing.Platform/Services/IPoliciesService.cs +++ b/src/Platform/Microsoft.Testing.Platform/Services/IStopPoliciesService.cs @@ -3,7 +3,7 @@ namespace Microsoft.Testing.Platform.Services; -internal interface IPoliciesService +internal interface IStopPoliciesService { void RegisterOnMaxFailedTestsCallback(Func callback); diff --git a/src/Platform/Microsoft.Testing.Platform/Services/PoliciesService.cs b/src/Platform/Microsoft.Testing.Platform/Services/StopPoliciesService.cs similarity index 91% rename from src/Platform/Microsoft.Testing.Platform/Services/PoliciesService.cs rename to src/Platform/Microsoft.Testing.Platform/Services/StopPoliciesService.cs index 0ec071ff6d..0c168ca007 100644 --- a/src/Platform/Microsoft.Testing.Platform/Services/PoliciesService.cs +++ b/src/Platform/Microsoft.Testing.Platform/Services/StopPoliciesService.cs @@ -5,13 +5,13 @@ namespace Microsoft.Testing.Platform.Services; -internal sealed class PoliciesService : IPoliciesService +internal sealed class StopPoliciesService : IStopPoliciesService { internal sealed class Policy { private BlockingCollection>? _callbacks; - public bool IsPolicyTriggered { get; private set; } + public bool IsTriggered { get; private set; } public void RegisterCallback(Func callback) => (_callbacks ??= new()).Add(callback); diff --git a/src/Platform/Microsoft.Testing.Platform/Services/TestApplicationResult.cs b/src/Platform/Microsoft.Testing.Platform/Services/TestApplicationResult.cs index 33a245456f..e5fb21d0f9 100644 --- a/src/Platform/Microsoft.Testing.Platform/Services/TestApplicationResult.cs +++ b/src/Platform/Microsoft.Testing.Platform/Services/TestApplicationResult.cs @@ -18,7 +18,7 @@ internal sealed class TestApplicationResult : ITestApplicationProcessExitCode, I private readonly IOutputDevice _outputService; private readonly ICommandLineOptions _commandLineOptions; private readonly IEnvironment _environment; - private readonly PoliciesService _policiesService; + private readonly StopPoliciesService _policiesService; private readonly List _failedTests = []; private int _totalRanTests; private bool _testAdapterTestSessionFailure; @@ -27,7 +27,7 @@ public TestApplicationResult( IOutputDevice outputService, ICommandLineOptions commandLineOptions, IEnvironment environment, - PoliciesService policiesService) + StopPoliciesService policiesService) { _outputService = outputService; _commandLineOptions = commandLineOptions; @@ -89,10 +89,10 @@ public Task ConsumeAsync(IDataProducer dataProducer, IData value, CancellationTo public int GetProcessExitCode() { int exitCode = ExitCodes.Success; - exitCode = exitCode == ExitCodes.Success && _policiesService.MaxFailedTestsPolicy.IsPolicyTriggered ? ExitCodes.TestExecutionStoppedForMaxFailedTests : exitCode; + exitCode = exitCode == ExitCodes.Success && _policiesService.MaxFailedTestsPolicy.IsTriggered ? ExitCodes.TestExecutionStoppedForMaxFailedTests : exitCode; exitCode = exitCode == ExitCodes.Success && _testAdapterTestSessionFailure ? ExitCodes.TestAdapterTestSessionFailure : exitCode; exitCode = exitCode == ExitCodes.Success && _failedTests.Count > 0 ? ExitCodes.AtLeastOneTestFailed : exitCode; - exitCode = exitCode == ExitCodes.Success && _policiesService.AbortPolicy.IsPolicyTriggered ? ExitCodes.TestSessionAborted : exitCode; + exitCode = exitCode == ExitCodes.Success && _policiesService.AbortPolicy.IsTriggered ? ExitCodes.TestSessionAborted : exitCode; // If the user has specified the VSTestAdapterMode option, then we don't want to return a non-zero exit code if no tests ran. if (!_commandLineOptions.IsOptionSet(PlatformCommandLineProvider.VSTestAdapterModeOptionKey)) diff --git a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Services/TestApplicationResultTests.cs b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Services/TestApplicationResultTests.cs index d33133fa1b..64c3a6ebcf 100644 --- a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Services/TestApplicationResultTests.cs +++ b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Services/TestApplicationResultTests.cs @@ -15,7 +15,7 @@ namespace Microsoft.Testing.Platform.UnitTests; public sealed class TestApplicationResultTests : TestBase { private readonly TestApplicationResult _testApplicationResult - = new(new Mock().Object, new Mock().Object, new Mock().Object, new Mock().Object); + = new(new Mock().Object, new Mock().Object, new Mock().Object, new Mock().Object); public TestApplicationResultTests(ITestExecutionContext testExecutionContext) : base(testExecutionContext) @@ -68,7 +68,7 @@ public async Task GetProcessExitCodeAsync_If_Failed_Tests_Returns_AtLeastOneTest public async Task GetProcessExitCodeAsync_If_Canceled_Returns_TestSessionAborted() { Mock testApplicationCancellationTokenSource = new(); - var policiesService = new PoliciesService(); + var policiesService = new StopPoliciesService(); testApplicationCancellationTokenSource.SetupGet(x => x.CancellationToken).Returns(() => { CancellationTokenSource cancellationTokenSource = new(); @@ -116,7 +116,7 @@ TestApplicationResult testApplicationResult = new( new Mock().Object, new CommandLineOption(PlatformCommandLineProvider.MinimumExpectedTestsOptionKey, ["2"]), - new Mock().Object, new Mock().Object); + new Mock().Object, new Mock().Object); await testApplicationResult.ConsumeAsync(new DummyProducer(), new TestNodeUpdateMessage( default, @@ -145,7 +145,7 @@ TestApplicationResult testApplicationResult = new( new Mock().Object, new CommandLineOption(PlatformCommandLineProvider.DiscoverTestsOptionKey, []), - new Mock().Object, new Mock().Object); + new Mock().Object, new Mock().Object); await testApplicationResult.ConsumeAsync(new DummyProducer(), new TestNodeUpdateMessage( default, @@ -164,7 +164,7 @@ TestApplicationResult testApplicationResult = new( new Mock().Object, new CommandLineOption(PlatformCommandLineProvider.DiscoverTestsOptionKey, []), - new Mock().Object, new Mock().Object); + new Mock().Object, new Mock().Object); await testApplicationResult.ConsumeAsync(new DummyProducer(), new TestNodeUpdateMessage( default, @@ -199,12 +199,12 @@ public void GetProcessExitCodeAsync_IgnoreExitCodes(string argument, int expecte new( new Mock().Object, new CommandLineOption(PlatformCommandLineProvider.IgnoreExitCodeOptionKey, argument is null ? [] : [argument]), - new Mock().Object, new Mock().Object), + new Mock().Object, new Mock().Object), new( new Mock().Object, new Mock().Object, environment.Object, - new Mock().Object), + new Mock().Object), }) { Assert.AreEqual(expectedExitCode, testApplicationResult.GetProcessExitCode()); From 13ea139f8248671dd43b9f3a9892502ca136cfc6 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Fri, 6 Dec 2024 16:05:51 +0100 Subject: [PATCH 24/64] Cleanup --- .../Execution/TestExecutionManager.cs | 4 ++ .../AbortForMaxFailedTestsExtension.cs | 6 +-- .../Hosts/TestHostBuilder.cs | 4 +- .../CTRLPlusCCancellationTokenSource.cs | 6 +-- .../Services/IStopPoliciesService.cs | 8 +++ .../Services/StopPoliciesService.cs | 54 ++++++++++++------- .../Services/TestApplicationResult.cs | 8 +-- 7 files changed, 56 insertions(+), 34 deletions(-) diff --git a/src/Adapter/MSTest.TestAdapter/Execution/TestExecutionManager.cs b/src/Adapter/MSTest.TestAdapter/Execution/TestExecutionManager.cs index f3a8913b9d..99461b05bc 100644 --- a/src/Adapter/MSTest.TestAdapter/Execution/TestExecutionManager.cs +++ b/src/Adapter/MSTest.TestAdapter/Execution/TestExecutionManager.cs @@ -419,6 +419,10 @@ private void ExecuteTestsWithTestRunner( foreach (TestCase currentTest in orderedTests) { _testRunCancellationToken?.ThrowIfCancellationRequested(); + if (MSTestStopGracefullyTestExecutionCapability.Instance.IsStopRequested) + { + break; + } // If it is a fixture test, add it to the list of fixture tests and do not execute it. // It is executed by test itself. diff --git a/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs b/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs index 3ae2516561..6e854da8a5 100644 --- a/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs +++ b/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs @@ -16,14 +16,14 @@ internal sealed class AbortForMaxFailedTestsExtension : IDataConsumer { private readonly int? _maxFailedTests; private readonly IGracefulStopTestExecutionCapability? _capability; - private readonly StopPoliciesService _policiesService; + private readonly IStopPoliciesService _policiesService; private readonly ITestApplicationCancellationTokenSource _testApplicationCancellationTokenSource; private int _failCount; public AbortForMaxFailedTestsExtension( ICommandLineOptions commandLineOptions, IGracefulStopTestExecutionCapability? capability, - StopPoliciesService policiesService, + IStopPoliciesService policiesService, ITestApplicationCancellationTokenSource testApplicationCancellationTokenSource) { if (commandLineOptions.TryGetOptionArgumentList(PlatformCommandLineProvider.MaxFailedTestsOptionKey, out string[]? args) && @@ -68,7 +68,7 @@ public async Task ConsumeAsync(IDataProducer dataProducer, IData value, Cancella ++_failCount > _maxFailedTests.Value) { await _capability.StopTestExecutionAsync(_testApplicationCancellationTokenSource.CancellationToken); - await _policiesService.MaxFailedTestsPolicy.ExecuteCallbacksAsync(_testApplicationCancellationTokenSource.CancellationToken); + await _policiesService.ExecuteMaxFailedTestsCallbacksAsync(_testApplicationCancellationTokenSource.CancellationToken); } } } diff --git a/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs b/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs index 106b49a532..4d0bf8c5d3 100644 --- a/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs +++ b/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs @@ -731,8 +731,8 @@ private async Task BuildTestFrameworkAsync(TestFrameworkBuilderD var abortForMaxFailedTestsExtension = new AbortForMaxFailedTestsExtension( serviceProvider.GetCommandLineOptions(), serviceProvider.GetTestFrameworkCapabilities().GetCapability(), - serviceProvider.GetRequiredService(), - serviceProvider.GetTestApplicationCancellationTokenSource().CancellationToken); + serviceProvider.GetRequiredService(), + serviceProvider.GetTestApplicationCancellationTokenSource()); if (await abortForMaxFailedTestsExtension.IsEnabledAsync()) { diff --git a/src/Platform/Microsoft.Testing.Platform/Services/CTRLPlusCCancellationTokenSource.cs b/src/Platform/Microsoft.Testing.Platform/Services/CTRLPlusCCancellationTokenSource.cs index 2a94619a68..9028714672 100644 --- a/src/Platform/Microsoft.Testing.Platform/Services/CTRLPlusCCancellationTokenSource.cs +++ b/src/Platform/Microsoft.Testing.Platform/Services/CTRLPlusCCancellationTokenSource.cs @@ -11,7 +11,7 @@ internal sealed class CTRLPlusCCancellationTokenSource : ITestApplicationCancell private readonly CancellationTokenSource _cancellationTokenSource = new(); private readonly ILogger? _logger; - public CTRLPlusCCancellationTokenSource(IConsole? console = null, ILogger? logger = null, StopPoliciesService? policiesService = null) + public CTRLPlusCCancellationTokenSource(IConsole? console = null, ILogger? logger = null) { if (console is not null) { @@ -19,10 +19,6 @@ public CTRLPlusCCancellationTokenSource(IConsole? console = null, ILogger? logge } _logger = logger; - if (policiesService is not null) - { - _cancellationTokenSource.Token.Register(() => policiesService.AbortPolicy.ExecuteCallbacksAsync(cancellationToken: default).GetAwaiter().GetResult()); - } } public void CancelAfter(TimeSpan timeout) => _cancellationTokenSource.CancelAfter(timeout); diff --git a/src/Platform/Microsoft.Testing.Platform/Services/IStopPoliciesService.cs b/src/Platform/Microsoft.Testing.Platform/Services/IStopPoliciesService.cs index e92b744b6f..911d9eba5a 100644 --- a/src/Platform/Microsoft.Testing.Platform/Services/IStopPoliciesService.cs +++ b/src/Platform/Microsoft.Testing.Platform/Services/IStopPoliciesService.cs @@ -5,7 +5,15 @@ namespace Microsoft.Testing.Platform.Services; internal interface IStopPoliciesService { + bool IsMaxFailedTestsTriggered { get; } + + bool IsAbortTriggered { get; } + void RegisterOnMaxFailedTestsCallback(Func callback); void RegisterOnAbortCallback(Func callback); + + Task ExecuteMaxFailedTestsCallbacksAsync(CancellationToken cancellationToken); + + Task ExecuteAbortCallbacksAsync(); } diff --git a/src/Platform/Microsoft.Testing.Platform/Services/StopPoliciesService.cs b/src/Platform/Microsoft.Testing.Platform/Services/StopPoliciesService.cs index 0c168ca007..cf5d36f20b 100644 --- a/src/Platform/Microsoft.Testing.Platform/Services/StopPoliciesService.cs +++ b/src/Platform/Microsoft.Testing.Platform/Services/StopPoliciesService.cs @@ -7,36 +7,50 @@ namespace Microsoft.Testing.Platform.Services; internal sealed class StopPoliciesService : IStopPoliciesService { - internal sealed class Policy - { - private BlockingCollection>? _callbacks; + public StopPoliciesService(ITestApplicationCancellationTokenSource testApplicationCancellationTokenSource) => +#pragma warning disable VSTHRD101 // Avoid unsupported async delegates + testApplicationCancellationTokenSource.CancellationToken.Register(async () => await ExecuteAbortCallbacksAsync()); +#pragma warning restore VSTHRD101 // Avoid unsupported async delegates + + private BlockingCollection>? _maxFailedTestsCallbacks; + private BlockingCollection>? _abortCallbacks; + + public bool IsMaxFailedTestsTriggered { get; private set; } + + public bool IsAbortTriggered { get; private set; } - public bool IsTriggered { get; private set; } + private static void RegisterCallback(ref BlockingCollection? callbacks, T callback) + => (callbacks ??= new()).Add(callback); - public void RegisterCallback(Func callback) - => (_callbacks ??= new()).Add(callback); + public async Task ExecuteMaxFailedTestsCallbacksAsync(CancellationToken cancellationToken) + { + if (_maxFailedTestsCallbacks is null) + { + return; + } - public async Task ExecuteCallbacksAsync(CancellationToken cancellationToken) + foreach (Func callback in _maxFailedTestsCallbacks) { - if (_callbacks is null) - { - return; - } - - foreach (Func callback in _callbacks) - { - await callback.Invoke(cancellationToken); - } + await callback.Invoke(cancellationToken); } } - internal Policy MaxFailedTestsPolicy { get; } = new(); + public async Task ExecuteAbortCallbacksAsync() + { + if (_abortCallbacks is null) + { + return; + } - internal Policy AbortPolicy { get; } = new(); + foreach (Func callback in _abortCallbacks) + { + await callback.Invoke(); + } + } public void RegisterOnMaxFailedTestsCallback(Func callback) - => MaxFailedTestsPolicy.RegisterCallback(callback); + => RegisterCallback(ref _maxFailedTestsCallbacks, callback); public void RegisterOnAbortCallback(Func callback) - => AbortPolicy.RegisterCallback(_ => callback()); + => RegisterCallback(ref _abortCallbacks, callback); } diff --git a/src/Platform/Microsoft.Testing.Platform/Services/TestApplicationResult.cs b/src/Platform/Microsoft.Testing.Platform/Services/TestApplicationResult.cs index e5fb21d0f9..0b19b09d89 100644 --- a/src/Platform/Microsoft.Testing.Platform/Services/TestApplicationResult.cs +++ b/src/Platform/Microsoft.Testing.Platform/Services/TestApplicationResult.cs @@ -18,7 +18,7 @@ internal sealed class TestApplicationResult : ITestApplicationProcessExitCode, I private readonly IOutputDevice _outputService; private readonly ICommandLineOptions _commandLineOptions; private readonly IEnvironment _environment; - private readonly StopPoliciesService _policiesService; + private readonly IStopPoliciesService _policiesService; private readonly List _failedTests = []; private int _totalRanTests; private bool _testAdapterTestSessionFailure; @@ -27,7 +27,7 @@ public TestApplicationResult( IOutputDevice outputService, ICommandLineOptions commandLineOptions, IEnvironment environment, - StopPoliciesService policiesService) + IStopPoliciesService policiesService) { _outputService = outputService; _commandLineOptions = commandLineOptions; @@ -89,10 +89,10 @@ public Task ConsumeAsync(IDataProducer dataProducer, IData value, CancellationTo public int GetProcessExitCode() { int exitCode = ExitCodes.Success; - exitCode = exitCode == ExitCodes.Success && _policiesService.MaxFailedTestsPolicy.IsTriggered ? ExitCodes.TestExecutionStoppedForMaxFailedTests : exitCode; + exitCode = exitCode == ExitCodes.Success && _policiesService.IsMaxFailedTestsTriggered ? ExitCodes.TestExecutionStoppedForMaxFailedTests : exitCode; exitCode = exitCode == ExitCodes.Success && _testAdapterTestSessionFailure ? ExitCodes.TestAdapterTestSessionFailure : exitCode; exitCode = exitCode == ExitCodes.Success && _failedTests.Count > 0 ? ExitCodes.AtLeastOneTestFailed : exitCode; - exitCode = exitCode == ExitCodes.Success && _policiesService.AbortPolicy.IsTriggered ? ExitCodes.TestSessionAborted : exitCode; + exitCode = exitCode == ExitCodes.Success && _policiesService.IsAbortTriggered ? ExitCodes.TestSessionAborted : exitCode; // If the user has specified the VSTestAdapterMode option, then we don't want to return a non-zero exit code if no tests ran. if (!_commandLineOptions.IsOptionSet(PlatformCommandLineProvider.VSTestAdapterModeOptionKey)) From db0fded7109db64a0d05de43f0cfab5ce0566d32 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Fri, 6 Dec 2024 16:06:03 +0100 Subject: [PATCH 25/64] Update tests --- .../Services/TestApplicationResultTests.cs | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Services/TestApplicationResultTests.cs b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Services/TestApplicationResultTests.cs index 64c3a6ebcf..6da2696496 100644 --- a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Services/TestApplicationResultTests.cs +++ b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Services/TestApplicationResultTests.cs @@ -15,7 +15,7 @@ namespace Microsoft.Testing.Platform.UnitTests; public sealed class TestApplicationResultTests : TestBase { private readonly TestApplicationResult _testApplicationResult - = new(new Mock().Object, new Mock().Object, new Mock().Object, new Mock().Object); + = new(new Mock().Object, new Mock().Object, new Mock().Object, new Mock().Object); public TestApplicationResultTests(ITestExecutionContext testExecutionContext) : base(testExecutionContext) @@ -68,20 +68,16 @@ public async Task GetProcessExitCodeAsync_If_Failed_Tests_Returns_AtLeastOneTest public async Task GetProcessExitCodeAsync_If_Canceled_Returns_TestSessionAborted() { Mock testApplicationCancellationTokenSource = new(); - var policiesService = new StopPoliciesService(); testApplicationCancellationTokenSource.SetupGet(x => x.CancellationToken).Returns(() => { CancellationTokenSource cancellationTokenSource = new(); - // In reality, this is done by CTRLPlusCCancellationTokenSource (the real production impl of ITestApplicationCancellationTokenSource). - cancellationTokenSource.Token.Register(() => policiesService.AbortPolicy.ExecuteCallbacksAsync(cancellationToken: default).GetAwaiter().GetResult()); - cancellationTokenSource.Cancel(); return cancellationTokenSource.Token; }); TestApplicationResult testApplicationResult - = new(new Mock().Object, new Mock().Object, new Mock().Object, policiesService); + = new(new Mock().Object, new Mock().Object, new Mock().Object, new Mock().Object); await testApplicationResult.ConsumeAsync(new DummyProducer(), new TestNodeUpdateMessage( default, @@ -116,7 +112,7 @@ TestApplicationResult testApplicationResult = new( new Mock().Object, new CommandLineOption(PlatformCommandLineProvider.MinimumExpectedTestsOptionKey, ["2"]), - new Mock().Object, new Mock().Object); + new Mock().Object, new Mock().Object); await testApplicationResult.ConsumeAsync(new DummyProducer(), new TestNodeUpdateMessage( default, @@ -145,7 +141,7 @@ TestApplicationResult testApplicationResult = new( new Mock().Object, new CommandLineOption(PlatformCommandLineProvider.DiscoverTestsOptionKey, []), - new Mock().Object, new Mock().Object); + new Mock().Object, new Mock().Object); await testApplicationResult.ConsumeAsync(new DummyProducer(), new TestNodeUpdateMessage( default, @@ -164,7 +160,7 @@ TestApplicationResult testApplicationResult = new( new Mock().Object, new CommandLineOption(PlatformCommandLineProvider.DiscoverTestsOptionKey, []), - new Mock().Object, new Mock().Object); + new Mock().Object, new Mock().Object); await testApplicationResult.ConsumeAsync(new DummyProducer(), new TestNodeUpdateMessage( default, @@ -199,12 +195,12 @@ public void GetProcessExitCodeAsync_IgnoreExitCodes(string argument, int expecte new( new Mock().Object, new CommandLineOption(PlatformCommandLineProvider.IgnoreExitCodeOptionKey, argument is null ? [] : [argument]), - new Mock().Object, new Mock().Object), + new Mock().Object, new Mock().Object), new( new Mock().Object, new Mock().Object, environment.Object, - new Mock().Object), + new Mock().Object), }) { Assert.AreEqual(expectedExitCode, testApplicationResult.GetProcessExitCode()); From efa34f13980566c043c4269f27008cb319551fa6 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Fri, 6 Dec 2024 16:30:48 +0100 Subject: [PATCH 26/64] Bit of progress --- .../PlatformCommandLineProvider.cs | 11 ---- .../AbortForMaxFailedTestsExtension.cs | 3 +- .../PublicAPI/PublicAPI.Unshipped.txt | 10 ++++ ...axFailedTestsCommandLineOptionsProvider.cs | 52 +++++++++++++++++++ 4 files changed, 64 insertions(+), 12 deletions(-) create mode 100644 src/Platform/Microsoft.Testing.Platform/TestFramework/MaxFailedTestsCommandLineOptionsProvider.cs diff --git a/src/Platform/Microsoft.Testing.Platform/CommandLine/PlatformCommandLineProvider.cs b/src/Platform/Microsoft.Testing.Platform/CommandLine/PlatformCommandLineProvider.cs index c5a53d473a..ba26ab2f22 100644 --- a/src/Platform/Microsoft.Testing.Platform/CommandLine/PlatformCommandLineProvider.cs +++ b/src/Platform/Microsoft.Testing.Platform/CommandLine/PlatformCommandLineProvider.cs @@ -32,7 +32,6 @@ internal sealed class PlatformCommandLineProvider : ICommandLineOptionsProvider public const string TestHostControllerPIDOptionKey = "internal-testhostcontroller-pid"; public const string ExitOnProcessExitOptionKey = "exit-on-process-exit"; public const string ConfigFileOptionKey = "config-file"; - public const string MaxFailedTestsOptionKey = "max-failed-tests"; public const string ServerOptionKey = "server"; public const string ClientPortOptionKey = "client-port"; @@ -62,7 +61,6 @@ internal sealed class PlatformCommandLineProvider : ICommandLineOptionsProvider new(IgnoreExitCodeOptionKey, PlatformResources.PlatformCommandLineIgnoreExitCodeOptionDescription, ArgumentArity.ExactlyOne, false, isBuiltIn: true), new(ExitOnProcessExitOptionKey, PlatformResources.PlatformCommandLineExitOnProcessExitOptionDescription, ArgumentArity.ExactlyOne, false, isBuiltIn: true), new(ConfigFileOptionKey, PlatformResources.PlatformCommandLineConfigFileOptionDescription, ArgumentArity.ExactlyOne, false, isBuiltIn: true), - new(MaxFailedTestsOptionKey, PlatformResources.PlatformCommandLineMaxFailedTestsOptionDescription, ArgumentArity.ExactlyOne, false, isBuiltIn: true), // Hidden options new(HelpOptionQuestionMark, PlatformResources.PlatformCommandLineHelpOptionDescription, ArgumentArity.Zero, true, isBuiltIn: true), @@ -143,15 +141,6 @@ public Task ValidateOptionArgumentsAsync(CommandLineOption com } } - if (commandOption.Name == MaxFailedTestsOptionKey) - { - string arg = arguments[0]; - if (!int.TryParse(arg, out int maxFailedTestsResult) || maxFailedTestsResult <= 0) - { - return ValidationResult.InvalidTask(string.Format(CultureInfo.InvariantCulture, PlatformResources.MaxFailedTestsMustBePositive, arg)); - } - } - // Now validate the minimum expected tests option return IsMinimumExpectedTestsOptionValidAsync(commandOption, arguments); } diff --git a/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs b/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs index 6e854da8a5..f8949c5f20 100644 --- a/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs +++ b/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs @@ -9,6 +9,7 @@ using Microsoft.Testing.Platform.Messages; using Microsoft.Testing.Platform.Resources; using Microsoft.Testing.Platform.Services; +using Microsoft.Testing.Platform.TestFramework; namespace Microsoft.Testing.Platform.Extensions; @@ -26,7 +27,7 @@ public AbortForMaxFailedTestsExtension( IStopPoliciesService policiesService, ITestApplicationCancellationTokenSource testApplicationCancellationTokenSource) { - if (commandLineOptions.TryGetOptionArgumentList(PlatformCommandLineProvider.MaxFailedTestsOptionKey, out string[]? args) && + if (commandLineOptions.TryGetOptionArgumentList(MaxFailedTestsCommandLineOptionsProvider.MaxFailedTestsOptionKey, out string[]? args) && int.TryParse(args[0], out int maxFailedTests) && maxFailedTests > 0) { diff --git a/src/Platform/Microsoft.Testing.Platform/PublicAPI/PublicAPI.Unshipped.txt b/src/Platform/Microsoft.Testing.Platform/PublicAPI/PublicAPI.Unshipped.txt index 1df3bd9c6c..44fd6896ef 100644 --- a/src/Platform/Microsoft.Testing.Platform/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/Platform/Microsoft.Testing.Platform/PublicAPI/PublicAPI.Unshipped.txt @@ -6,6 +6,16 @@ Microsoft.Testing.Platform.OutputDevice.ErrorMessageOutputDeviceData.Message.get Microsoft.Testing.Platform.OutputDevice.WarningMessageOutputDeviceData Microsoft.Testing.Platform.OutputDevice.WarningMessageOutputDeviceData.Message.get -> string! Microsoft.Testing.Platform.OutputDevice.WarningMessageOutputDeviceData.WarningMessageOutputDeviceData(string! message) -> void +Microsoft.Testing.Platform.TestFramework.MaxFailedTestsCommandLineOptionsProvider +Microsoft.Testing.Platform.TestFramework.MaxFailedTestsCommandLineOptionsProvider.Description.get -> string! +Microsoft.Testing.Platform.TestFramework.MaxFailedTestsCommandLineOptionsProvider.DisplayName.get -> string! +Microsoft.Testing.Platform.TestFramework.MaxFailedTestsCommandLineOptionsProvider.GetCommandLineOptions() -> System.Collections.Generic.IReadOnlyCollection! +Microsoft.Testing.Platform.TestFramework.MaxFailedTestsCommandLineOptionsProvider.IsEnabledAsync() -> System.Threading.Tasks.Task! +Microsoft.Testing.Platform.TestFramework.MaxFailedTestsCommandLineOptionsProvider.MaxFailedTestsCommandLineOptionsProvider() -> void +Microsoft.Testing.Platform.TestFramework.MaxFailedTestsCommandLineOptionsProvider.Uid.get -> string! +Microsoft.Testing.Platform.TestFramework.MaxFailedTestsCommandLineOptionsProvider.ValidateCommandLineOptionsAsync(Microsoft.Testing.Platform.CommandLine.ICommandLineOptions! commandLineOptions) -> System.Threading.Tasks.Task! +Microsoft.Testing.Platform.TestFramework.MaxFailedTestsCommandLineOptionsProvider.ValidateOptionArgumentsAsync(Microsoft.Testing.Platform.Extensions.CommandLine.CommandLineOption! commandOption, string![]! arguments) -> System.Threading.Tasks.Task! +Microsoft.Testing.Platform.TestFramework.MaxFailedTestsCommandLineOptionsProvider.Version.get -> string! [TPEXP]Microsoft.Testing.Platform.Capabilities.TestFramework.IGracefulStopTestExecutionCapability [TPEXP]Microsoft.Testing.Platform.Capabilities.TestFramework.IGracefulStopTestExecutionCapability.StopTestExecutionAsync(System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task! [TPEXP]Microsoft.Testing.Platform.Capabilities.TestFramework.IStopGracefullyTestExecutionCapability diff --git a/src/Platform/Microsoft.Testing.Platform/TestFramework/MaxFailedTestsCommandLineOptionsProvider.cs b/src/Platform/Microsoft.Testing.Platform/TestFramework/MaxFailedTestsCommandLineOptionsProvider.cs new file mode 100644 index 0000000000..71390c27e7 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Platform/TestFramework/MaxFailedTestsCommandLineOptionsProvider.cs @@ -0,0 +1,52 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Globalization; + +using Microsoft.Testing.Platform.CommandLine; +using Microsoft.Testing.Platform.Extensions; +using Microsoft.Testing.Platform.Extensions.CommandLine; +using Microsoft.Testing.Platform.Helpers; +using Microsoft.Testing.Platform.Resources; + +namespace Microsoft.Testing.Platform.TestFramework; + +public sealed class MaxFailedTestsCommandLineOptionsProvider : ICommandLineOptionsProvider +{ + // TODO: We have 'minimum-expected-tests', so should we use "maximum" instead of "max" here as well for consistency? + internal const string MaxFailedTestsOptionKey = "max-failed-tests"; + + private static readonly IReadOnlyCollection OptionsCache = + [ + new(MaxFailedTestsOptionKey, PlatformResources.PlatformCommandLineMaxFailedTestsOptionDescription, ArgumentArity.ExactlyOne, false, isBuiltIn: true), + ]; + + public string Uid => nameof(MaxFailedTestsCommandLineOptionsProvider); + + public string Version => AppVersion.DefaultSemVer; + + public string DisplayName => PlatformResources.PlatformCommandLineProviderDisplayName; /*TODO: New display name?*/ + + public string Description => PlatformResources.PlatformCommandLineProviderDescription; /*TODO: New description?*/ + + public IReadOnlyCollection GetCommandLineOptions() + => OptionsCache; + + public Task IsEnabledAsync() => Task.FromResult(true); + + public Task ValidateCommandLineOptionsAsync(ICommandLineOptions commandLineOptions) => throw new NotImplementedException(); + + public Task ValidateOptionArgumentsAsync(CommandLineOption commandOption, string[] arguments) + { + if (commandOption.Name == MaxFailedTestsOptionKey) + { + string arg = arguments[0]; + if (!int.TryParse(arg, out int maxFailedTestsResult) || maxFailedTestsResult <= 0) + { + return ValidationResult.InvalidTask(string.Format(CultureInfo.InvariantCulture, PlatformResources.MaxFailedTestsMustBePositive, arg)); + } + } + + throw ApplicationStateGuard.Unreachable(); + } +} From e3ecc851fabe80338e477e6b9bc46cec5c5007f9 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Fri, 6 Dec 2024 16:40:12 +0100 Subject: [PATCH 27/64] Adjust messaging --- .../Extensions/AbortForMaxFailedTestsExtension.cs | 2 +- .../OutputDevice/TerminalOutputDevice.cs | 4 +++- .../Resources/PlatformResources.resx | 3 ++- .../Resources/xlf/PlatformResources.cs.xlf | 6 +++--- .../Resources/xlf/PlatformResources.de.xlf | 6 +++--- .../Resources/xlf/PlatformResources.es.xlf | 6 +++--- .../Resources/xlf/PlatformResources.fr.xlf | 6 +++--- .../Resources/xlf/PlatformResources.it.xlf | 6 +++--- .../Resources/xlf/PlatformResources.ja.xlf | 6 +++--- .../Resources/xlf/PlatformResources.ko.xlf | 6 +++--- .../Resources/xlf/PlatformResources.pl.xlf | 6 +++--- .../Resources/xlf/PlatformResources.pt-BR.xlf | 6 +++--- .../Resources/xlf/PlatformResources.ru.xlf | 6 +++--- .../Resources/xlf/PlatformResources.tr.xlf | 6 +++--- .../Resources/xlf/PlatformResources.zh-Hans.xlf | 6 +++--- .../Resources/xlf/PlatformResources.zh-Hant.xlf | 6 +++--- .../JsonRpc/ServerModePerCallOutputDevice.cs | 3 ++- .../Services/IStopPoliciesService.cs | 4 ++-- .../Services/StopPoliciesService.cs | 10 +++++----- .../HelpInfoTests.cs | 2 -- .../HelpInfoTests.cs | 12 ------------ 21 files changed, 54 insertions(+), 64 deletions(-) diff --git a/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs b/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs index f8949c5f20..3e5c86f5f8 100644 --- a/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs +++ b/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs @@ -69,7 +69,7 @@ public async Task ConsumeAsync(IDataProducer dataProducer, IData value, Cancella ++_failCount > _maxFailedTests.Value) { await _capability.StopTestExecutionAsync(_testApplicationCancellationTokenSource.CancellationToken); - await _policiesService.ExecuteMaxFailedTestsCallbacksAsync(_testApplicationCancellationTokenSource.CancellationToken); + await _policiesService.ExecuteMaxFailedTestsCallbacksAsync(_maxFailedTests.Value, _testApplicationCancellationTokenSource.CancellationToken); } } } diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.cs index 823ddb5068..a8eef90a12 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.cs +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.cs @@ -90,7 +90,9 @@ public TerminalOutputDevice(ITestApplicationCancellationTokenSource testApplicat _clock = clock; policiesService.RegisterOnMaxFailedTestsCallback( - async _ => await DisplayAsync(this, new TextOutputDeviceData(PlatformResources.ReachedMaxFailedTestsMessage))); + async (maxFailedTests, _) => await DisplayAsync( + this, + new TextOutputDeviceData(string.Format(CultureInfo.InvariantCulture, PlatformResources.ReachedMaxFailedTestsMessage, maxFailedTests)))); policiesService.RegisterOnAbortCallback( () => diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/PlatformResources.resx b/src/Platform/Microsoft.Testing.Platform/Resources/PlatformResources.resx index 60e537cc96..f19bf8f0fc 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/PlatformResources.resx +++ b/src/Platform/Microsoft.Testing.Platform/Resources/PlatformResources.resx @@ -704,6 +704,7 @@ Takes one argument as string in the format <value>[h|m|s] where 'value' is Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted. - Test session is aborting due to reaching failures specified by the '--max-failed-tests' option. + Test session is aborting due to reaching failures ('{0}') specified by the '--max-failed-tests' option. + {0} is the number of max failed tests. diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.cs.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.cs.xlf index 3f8522f7cb..a0ec2301ab 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.cs.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.cs.xlf @@ -629,9 +629,9 @@ Může mít jenom jeden argument jako řetězec ve formátu <value>[h|m|s] - Test session is aborting due to reaching failures specified by the '--max-failed-tests' option. - Test session is aborting due to reaching failures specified by the '--max-failed-tests' option. - + Test session is aborting due to reaching failures ('{0}') specified by the '--max-failed-tests' option. + Test session is aborting due to reaching failures ('{0}') specified by the '--max-failed-tests' option. + {0} is the number of max failed tests. Retry failed after {0} times diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.de.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.de.xlf index 0809f66116..e6a2d5927d 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.de.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.de.xlf @@ -629,9 +629,9 @@ Nimmt ein Argument als Zeichenfolge im Format <value>[h|m|s], wobei "value - Test session is aborting due to reaching failures specified by the '--max-failed-tests' option. - Test session is aborting due to reaching failures specified by the '--max-failed-tests' option. - + Test session is aborting due to reaching failures ('{0}') specified by the '--max-failed-tests' option. + Test session is aborting due to reaching failures ('{0}') specified by the '--max-failed-tests' option. + {0} is the number of max failed tests. Retry failed after {0} times diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.es.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.es.xlf index 2711aafb0d..4264208a7f 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.es.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.es.xlf @@ -629,9 +629,9 @@ Toma un argumento como cadena con el formato <value>[h|m|s] donde 'value' - Test session is aborting due to reaching failures specified by the '--max-failed-tests' option. - Test session is aborting due to reaching failures specified by the '--max-failed-tests' option. - + Test session is aborting due to reaching failures ('{0}') specified by the '--max-failed-tests' option. + Test session is aborting due to reaching failures ('{0}') specified by the '--max-failed-tests' option. + {0} is the number of max failed tests. Retry failed after {0} times diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.fr.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.fr.xlf index 35879307c3..fa14196f93 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.fr.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.fr.xlf @@ -629,9 +629,9 @@ Prend un argument sous forme de chaîne au format <value>[h|m|s] où « v - Test session is aborting due to reaching failures specified by the '--max-failed-tests' option. - Test session is aborting due to reaching failures specified by the '--max-failed-tests' option. - + Test session is aborting due to reaching failures ('{0}') specified by the '--max-failed-tests' option. + Test session is aborting due to reaching failures ('{0}') specified by the '--max-failed-tests' option. + {0} is the number of max failed tests. Retry failed after {0} times diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.it.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.it.xlf index be692f5b08..bb1fcefb19 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.it.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.it.xlf @@ -629,9 +629,9 @@ Acquisisce un argomento come stringa nel formato <value>[h|m|s] dove 'valu - Test session is aborting due to reaching failures specified by the '--max-failed-tests' option. - Test session is aborting due to reaching failures specified by the '--max-failed-tests' option. - + Test session is aborting due to reaching failures ('{0}') specified by the '--max-failed-tests' option. + Test session is aborting due to reaching failures ('{0}') specified by the '--max-failed-tests' option. + {0} is the number of max failed tests. Retry failed after {0} times diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ja.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ja.xlf index fa60bacb12..645f258331 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ja.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ja.xlf @@ -630,9 +630,9 @@ Takes one argument as string in the format <value>[h|m|s] where 'value' is - Test session is aborting due to reaching failures specified by the '--max-failed-tests' option. - Test session is aborting due to reaching failures specified by the '--max-failed-tests' option. - + Test session is aborting due to reaching failures ('{0}') specified by the '--max-failed-tests' option. + Test session is aborting due to reaching failures ('{0}') specified by the '--max-failed-tests' option. + {0} is the number of max failed tests. Retry failed after {0} times diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ko.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ko.xlf index 97c7d28b76..abcce5e080 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ko.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ko.xlf @@ -629,9 +629,9 @@ Takes one argument as string in the format <value>[h|m|s] where 'value' is - Test session is aborting due to reaching failures specified by the '--max-failed-tests' option. - Test session is aborting due to reaching failures specified by the '--max-failed-tests' option. - + Test session is aborting due to reaching failures ('{0}') specified by the '--max-failed-tests' option. + Test session is aborting due to reaching failures ('{0}') specified by the '--max-failed-tests' option. + {0} is the number of max failed tests. Retry failed after {0} times diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pl.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pl.xlf index 073a6be265..ae0667f335 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pl.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pl.xlf @@ -629,9 +629,9 @@ Pobiera jeden argument jako ciąg w formacie <value>[h|m|s], gdzie element - Test session is aborting due to reaching failures specified by the '--max-failed-tests' option. - Test session is aborting due to reaching failures specified by the '--max-failed-tests' option. - + Test session is aborting due to reaching failures ('{0}') specified by the '--max-failed-tests' option. + Test session is aborting due to reaching failures ('{0}') specified by the '--max-failed-tests' option. + {0} is the number of max failed tests. Retry failed after {0} times diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pt-BR.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pt-BR.xlf index 6bb4f5739f..c8e311c033 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pt-BR.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pt-BR.xlf @@ -629,9 +629,9 @@ Recebe um argumento como cadeia de caracteres no formato <valor>[h|m|s] em - Test session is aborting due to reaching failures specified by the '--max-failed-tests' option. - Test session is aborting due to reaching failures specified by the '--max-failed-tests' option. - + Test session is aborting due to reaching failures ('{0}') specified by the '--max-failed-tests' option. + Test session is aborting due to reaching failures ('{0}') specified by the '--max-failed-tests' option. + {0} is the number of max failed tests. Retry failed after {0} times diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ru.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ru.xlf index ad42932e14..50e6881be8 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ru.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ru.xlf @@ -629,9 +629,9 @@ Takes one argument as string in the format <value>[h|m|s] where 'value' is - Test session is aborting due to reaching failures specified by the '--max-failed-tests' option. - Test session is aborting due to reaching failures specified by the '--max-failed-tests' option. - + Test session is aborting due to reaching failures ('{0}') specified by the '--max-failed-tests' option. + Test session is aborting due to reaching failures ('{0}') specified by the '--max-failed-tests' option. + {0} is the number of max failed tests. Retry failed after {0} times diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.tr.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.tr.xlf index ce820d699a..b9c4d4dd79 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.tr.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.tr.xlf @@ -629,9 +629,9 @@ Bir bağımsız değişkeni, 'value' değerinin kayan olduğu <value>[h|m| - Test session is aborting due to reaching failures specified by the '--max-failed-tests' option. - Test session is aborting due to reaching failures specified by the '--max-failed-tests' option. - + Test session is aborting due to reaching failures ('{0}') specified by the '--max-failed-tests' option. + Test session is aborting due to reaching failures ('{0}') specified by the '--max-failed-tests' option. + {0} is the number of max failed tests. Retry failed after {0} times diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hans.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hans.xlf index c512b5f49c..49cd8b43c6 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hans.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hans.xlf @@ -629,9 +629,9 @@ Takes one argument as string in the format <value>[h|m|s] where 'value' is - Test session is aborting due to reaching failures specified by the '--max-failed-tests' option. - Test session is aborting due to reaching failures specified by the '--max-failed-tests' option. - + Test session is aborting due to reaching failures ('{0}') specified by the '--max-failed-tests' option. + Test session is aborting due to reaching failures ('{0}') specified by the '--max-failed-tests' option. + {0} is the number of max failed tests. Retry failed after {0} times diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hant.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hant.xlf index ad367169f7..4b890accf8 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hant.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hant.xlf @@ -629,9 +629,9 @@ Takes one argument as string in the format <value>[h|m|s] where 'value' is - Test session is aborting due to reaching failures specified by the '--max-failed-tests' option. - Test session is aborting due to reaching failures specified by the '--max-failed-tests' option. - + Test session is aborting due to reaching failures ('{0}') specified by the '--max-failed-tests' option. + Test session is aborting due to reaching failures ('{0}') specified by the '--max-failed-tests' option. + {0} is the number of max failed tests. Retry failed after {0} times diff --git a/src/Platform/Microsoft.Testing.Platform/ServerMode/JsonRpc/ServerModePerCallOutputDevice.cs b/src/Platform/Microsoft.Testing.Platform/ServerMode/JsonRpc/ServerModePerCallOutputDevice.cs index 3c01a65dee..d2ff415e6a 100644 --- a/src/Platform/Microsoft.Testing.Platform/ServerMode/JsonRpc/ServerModePerCallOutputDevice.cs +++ b/src/Platform/Microsoft.Testing.Platform/ServerMode/JsonRpc/ServerModePerCallOutputDevice.cs @@ -28,7 +28,8 @@ public ServerModePerCallOutputDevice(FileLoggerProvider? fileLoggerProvider, ISt { _fileLoggerProvider = fileLoggerProvider; policiesService.RegisterOnMaxFailedTestsCallback( - async _ => await DisplayAsync(this, new TextOutputDeviceData(PlatformResources.ReachedMaxFailedTestsMessage))); + async (maxFailedTests, _) => await DisplayAsync( + this, new TextOutputDeviceData(string.Format(CultureInfo.InvariantCulture, PlatformResources.ReachedMaxFailedTestsMessage, maxFailedTests)))); } internal async Task InitializeAsync(IServerTestHost serverTestHost) diff --git a/src/Platform/Microsoft.Testing.Platform/Services/IStopPoliciesService.cs b/src/Platform/Microsoft.Testing.Platform/Services/IStopPoliciesService.cs index 911d9eba5a..d2b49d99d6 100644 --- a/src/Platform/Microsoft.Testing.Platform/Services/IStopPoliciesService.cs +++ b/src/Platform/Microsoft.Testing.Platform/Services/IStopPoliciesService.cs @@ -9,11 +9,11 @@ internal interface IStopPoliciesService bool IsAbortTriggered { get; } - void RegisterOnMaxFailedTestsCallback(Func callback); + void RegisterOnMaxFailedTestsCallback(Func callback); void RegisterOnAbortCallback(Func callback); - Task ExecuteMaxFailedTestsCallbacksAsync(CancellationToken cancellationToken); + Task ExecuteMaxFailedTestsCallbacksAsync(int maxFailedTests, CancellationToken cancellationToken); Task ExecuteAbortCallbacksAsync(); } diff --git a/src/Platform/Microsoft.Testing.Platform/Services/StopPoliciesService.cs b/src/Platform/Microsoft.Testing.Platform/Services/StopPoliciesService.cs index cf5d36f20b..ccd58e4fb0 100644 --- a/src/Platform/Microsoft.Testing.Platform/Services/StopPoliciesService.cs +++ b/src/Platform/Microsoft.Testing.Platform/Services/StopPoliciesService.cs @@ -12,7 +12,7 @@ public StopPoliciesService(ITestApplicationCancellationTokenSource testApplicati testApplicationCancellationTokenSource.CancellationToken.Register(async () => await ExecuteAbortCallbacksAsync()); #pragma warning restore VSTHRD101 // Avoid unsupported async delegates - private BlockingCollection>? _maxFailedTestsCallbacks; + private BlockingCollection>? _maxFailedTestsCallbacks; private BlockingCollection>? _abortCallbacks; public bool IsMaxFailedTestsTriggered { get; private set; } @@ -22,16 +22,16 @@ public StopPoliciesService(ITestApplicationCancellationTokenSource testApplicati private static void RegisterCallback(ref BlockingCollection? callbacks, T callback) => (callbacks ??= new()).Add(callback); - public async Task ExecuteMaxFailedTestsCallbacksAsync(CancellationToken cancellationToken) + public async Task ExecuteMaxFailedTestsCallbacksAsync(int maxFailedTests, CancellationToken cancellationToken) { if (_maxFailedTestsCallbacks is null) { return; } - foreach (Func callback in _maxFailedTestsCallbacks) + foreach (Func callback in _maxFailedTestsCallbacks) { - await callback.Invoke(cancellationToken); + await callback.Invoke(maxFailedTests, cancellationToken); } } @@ -48,7 +48,7 @@ public async Task ExecuteAbortCallbacksAsync() } } - public void RegisterOnMaxFailedTestsCallback(Func callback) + public void RegisterOnMaxFailedTestsCallback(Func callback) => RegisterCallback(ref _maxFailedTestsCallbacks, callback); public void RegisterOnAbortCallback(Func callback) diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/HelpInfoTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/HelpInfoTests.cs index e2aeac66b3..8894ffbc2d 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/HelpInfoTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/HelpInfoTests.cs @@ -59,8 +59,6 @@ Show the command line help. Display .NET test application information. --list-tests List available tests. - --max-failed-tests - Specifies a maximum number of test failures that, when exceeded, will abort the test run. --minimum-expected-tests Specifies the minimum number of tests that are expected to run. --results-directory diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpInfoTests.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpInfoTests.cs index 40c1c822ff..1f66b97679 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpInfoTests.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpInfoTests.cs @@ -55,8 +55,6 @@ Show the command line help. Display .NET test application information. --list-tests List available tests. - --max-failed-tests - Specifies a maximum number of test failures that, when exceeded, will abort the test run. --minimum-expected-tests Specifies the minimum number of tests that are expected to run. --results-directory @@ -220,10 +218,6 @@ Note that this is slowing down the test execution\. Arity: 0 Hidden: False Description: List available tests\. - --max-failed-tests - Arity: 1 - Hidden: False - Description: Specifies a maximum number of test failures that, when exceeded, will abort the test run\. --minimum-expected-tests Arity: 0\.\.1 Hidden: False @@ -323,8 +317,6 @@ Show the command line help. Display .NET test application information. --list-tests List available tests. - --max-failed-tests - Specifies a maximum number of test failures that, when exceeded, will abort the test run. --minimum-expected-tests Specifies the minimum number of tests that are expected to run. --results-directory @@ -500,10 +492,6 @@ Note that this is slowing down the test execution. Arity: 0 Hidden: False Description: List available tests. - --max-failed-tests - Arity: 1 - Hidden: False - Description: Specifies a maximum number of test failures that, when exceeded, will abort the test run. --minimum-expected-tests Arity: 0..1 Hidden: False From ceae0ccc17a6514b939ef03be7c348b96ab739c7 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Fri, 6 Dec 2024 18:49:42 +0100 Subject: [PATCH 28/64] More progress --- .../TestApplicationBuilderExtensions.cs | 1 + .../TestApplicationBuilderExtensions.cs | 10 +++++++++ .../PublicAPI.Unshipped.txt | 1 + .../Hosts/TestHostBuilder.cs | 2 +- .../PublicAPI/PublicAPI.Unshipped.txt | 22 +++++++++---------- ...axFailedTestsCommandLineOptionsProvider.cs | 2 ++ 6 files changed, 25 insertions(+), 13 deletions(-) diff --git a/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/TestApplicationBuilderExtensions.cs b/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/TestApplicationBuilderExtensions.cs index 05d2fa2481..9dd9b985f3 100644 --- a/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/TestApplicationBuilderExtensions.cs +++ b/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/TestApplicationBuilderExtensions.cs @@ -20,6 +20,7 @@ public static void AddMSTest(this ITestApplicationBuilder testApplicationBuilder testApplicationBuilder.AddRunSettingsService(extension); testApplicationBuilder.AddTestCaseFilterService(extension); testApplicationBuilder.AddTestRunParametersService(extension); + testApplicationBuilder.AddMaxFailedTestsService(); testApplicationBuilder.AddRunSettingsEnvironmentVariableProvider(extension); testApplicationBuilder.RegisterTestFramework( serviceProvider => new TestFrameworkCapabilities( diff --git a/src/Platform/Microsoft.Testing.Extensions.VSTestBridge/Helpers/TestApplicationBuilderExtensions.cs b/src/Platform/Microsoft.Testing.Extensions.VSTestBridge/Helpers/TestApplicationBuilderExtensions.cs index 9ec3739da9..7cc89ae29b 100644 --- a/src/Platform/Microsoft.Testing.Extensions.VSTestBridge/Helpers/TestApplicationBuilderExtensions.cs +++ b/src/Platform/Microsoft.Testing.Extensions.VSTestBridge/Helpers/TestApplicationBuilderExtensions.cs @@ -8,6 +8,7 @@ using Microsoft.Testing.Platform.Extensions; using Microsoft.Testing.Platform.Helpers; using Microsoft.Testing.Platform.Services; +using Microsoft.Testing.Platform.TestFramework; namespace Microsoft.Testing.Extensions.VSTestBridge.Helpers; @@ -47,6 +48,15 @@ public static void AddRunSettingsService(this ITestApplicationBuilder builder, I public static void AddTestRunParametersService(this ITestApplicationBuilder builder, IExtension extension) => builder.CommandLine.AddProvider(() => new TestRunParametersCommandLineOptionsProvider(extension)); + /// + /// Registers the command-line options provider for '--max-failed-tests'. + /// + /// The test application builder. + public static void AddMaxFailedTestsService(this ITestApplicationBuilder builder) +#pragma warning disable TPEXP // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + => builder.CommandLine.AddProvider(() => new MaxFailedTestsCommandLineOptionsProvider()); +#pragma warning restore TPEXP // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + /// /// Register the environment variable provider. /// diff --git a/src/Platform/Microsoft.Testing.Extensions.VSTestBridge/PublicAPI.Unshipped.txt b/src/Platform/Microsoft.Testing.Extensions.VSTestBridge/PublicAPI.Unshipped.txt index c27c99f4e5..421454eed9 100644 --- a/src/Platform/Microsoft.Testing.Extensions.VSTestBridge/PublicAPI.Unshipped.txt +++ b/src/Platform/Microsoft.Testing.Extensions.VSTestBridge/PublicAPI.Unshipped.txt @@ -1,2 +1,3 @@ #nullable enable +static Microsoft.Testing.Extensions.VSTestBridge.Helpers.TestApplicationBuilderExtensions.AddMaxFailedTestsService(this Microsoft.Testing.Platform.Builder.ITestApplicationBuilder! builder) -> void static Microsoft.Testing.Extensions.VSTestBridge.Helpers.TestApplicationBuilderExtensions.AddRunSettingsEnvironmentVariableProvider(this Microsoft.Testing.Platform.Builder.ITestApplicationBuilder! builder, Microsoft.Testing.Platform.Extensions.IExtension! extension) -> void diff --git a/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs b/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs index 4d0bf8c5d3..ac4f7b6828 100644 --- a/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs +++ b/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs @@ -207,7 +207,7 @@ public async Task BuildAsync( commandLineOptionsProxy.SetCommandLineOptions(commandLineHandler); // This is needed by output device. - var policiesService = new StopPoliciesService(); + var policiesService = new StopPoliciesService(testApplicationCancellationTokenSource); serviceProvider.AddService(policiesService); bool hasServerFlag = commandLineHandler.TryGetOptionArgumentList(PlatformCommandLineProvider.ServerOptionKey, out string[]? protocolName); diff --git a/src/Platform/Microsoft.Testing.Platform/PublicAPI/PublicAPI.Unshipped.txt b/src/Platform/Microsoft.Testing.Platform/PublicAPI/PublicAPI.Unshipped.txt index 44fd6896ef..5781934149 100644 --- a/src/Platform/Microsoft.Testing.Platform/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/Platform/Microsoft.Testing.Platform/PublicAPI/PublicAPI.Unshipped.txt @@ -6,23 +6,21 @@ Microsoft.Testing.Platform.OutputDevice.ErrorMessageOutputDeviceData.Message.get Microsoft.Testing.Platform.OutputDevice.WarningMessageOutputDeviceData Microsoft.Testing.Platform.OutputDevice.WarningMessageOutputDeviceData.Message.get -> string! Microsoft.Testing.Platform.OutputDevice.WarningMessageOutputDeviceData.WarningMessageOutputDeviceData(string! message) -> void -Microsoft.Testing.Platform.TestFramework.MaxFailedTestsCommandLineOptionsProvider -Microsoft.Testing.Platform.TestFramework.MaxFailedTestsCommandLineOptionsProvider.Description.get -> string! -Microsoft.Testing.Platform.TestFramework.MaxFailedTestsCommandLineOptionsProvider.DisplayName.get -> string! -Microsoft.Testing.Platform.TestFramework.MaxFailedTestsCommandLineOptionsProvider.GetCommandLineOptions() -> System.Collections.Generic.IReadOnlyCollection! -Microsoft.Testing.Platform.TestFramework.MaxFailedTestsCommandLineOptionsProvider.IsEnabledAsync() -> System.Threading.Tasks.Task! -Microsoft.Testing.Platform.TestFramework.MaxFailedTestsCommandLineOptionsProvider.MaxFailedTestsCommandLineOptionsProvider() -> void -Microsoft.Testing.Platform.TestFramework.MaxFailedTestsCommandLineOptionsProvider.Uid.get -> string! -Microsoft.Testing.Platform.TestFramework.MaxFailedTestsCommandLineOptionsProvider.ValidateCommandLineOptionsAsync(Microsoft.Testing.Platform.CommandLine.ICommandLineOptions! commandLineOptions) -> System.Threading.Tasks.Task! -Microsoft.Testing.Platform.TestFramework.MaxFailedTestsCommandLineOptionsProvider.ValidateOptionArgumentsAsync(Microsoft.Testing.Platform.Extensions.CommandLine.CommandLineOption! commandOption, string![]! arguments) -> System.Threading.Tasks.Task! -Microsoft.Testing.Platform.TestFramework.MaxFailedTestsCommandLineOptionsProvider.Version.get -> string! [TPEXP]Microsoft.Testing.Platform.Capabilities.TestFramework.IGracefulStopTestExecutionCapability [TPEXP]Microsoft.Testing.Platform.Capabilities.TestFramework.IGracefulStopTestExecutionCapability.StopTestExecutionAsync(System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task! -[TPEXP]Microsoft.Testing.Platform.Capabilities.TestFramework.IStopGracefullyTestExecutionCapability -[TPEXP]Microsoft.Testing.Platform.Capabilities.TestFramework.IStopGracefullyTestExecutionCapability.StopTestExecutionAsync(System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task! [TPEXP]Microsoft.Testing.Platform.Extensions.Messages.StandardOutputProperty [TPEXP]Microsoft.Testing.Platform.Extensions.Messages.StandardOutputProperty.StandardOutput.get -> string! [TPEXP]Microsoft.Testing.Platform.Extensions.Messages.StandardOutputProperty.StandardOutput.init -> void +[TPEXP]Microsoft.Testing.Platform.TestFramework.MaxFailedTestsCommandLineOptionsProvider +[TPEXP]Microsoft.Testing.Platform.TestFramework.MaxFailedTestsCommandLineOptionsProvider.Description.get -> string! +[TPEXP]Microsoft.Testing.Platform.TestFramework.MaxFailedTestsCommandLineOptionsProvider.DisplayName.get -> string! +[TPEXP]Microsoft.Testing.Platform.TestFramework.MaxFailedTestsCommandLineOptionsProvider.GetCommandLineOptions() -> System.Collections.Generic.IReadOnlyCollection! +[TPEXP]Microsoft.Testing.Platform.TestFramework.MaxFailedTestsCommandLineOptionsProvider.IsEnabledAsync() -> System.Threading.Tasks.Task! +[TPEXP]Microsoft.Testing.Platform.TestFramework.MaxFailedTestsCommandLineOptionsProvider.MaxFailedTestsCommandLineOptionsProvider() -> void +[TPEXP]Microsoft.Testing.Platform.TestFramework.MaxFailedTestsCommandLineOptionsProvider.Uid.get -> string! +[TPEXP]Microsoft.Testing.Platform.TestFramework.MaxFailedTestsCommandLineOptionsProvider.ValidateCommandLineOptionsAsync(Microsoft.Testing.Platform.CommandLine.ICommandLineOptions! commandLineOptions) -> System.Threading.Tasks.Task! +[TPEXP]Microsoft.Testing.Platform.TestFramework.MaxFailedTestsCommandLineOptionsProvider.ValidateOptionArgumentsAsync(Microsoft.Testing.Platform.Extensions.CommandLine.CommandLineOption! commandOption, string![]! arguments) -> System.Threading.Tasks.Task! +[TPEXP]Microsoft.Testing.Platform.TestFramework.MaxFailedTestsCommandLineOptionsProvider.Version.get -> string! [TPEXP]virtual Microsoft.Testing.Platform.Extensions.Messages.StandardOutputProperty.EqualityContract.get -> System.Type! [TPEXP]override Microsoft.Testing.Platform.Extensions.Messages.StandardOutputProperty.ToString() -> string! [TPEXP]virtual Microsoft.Testing.Platform.Extensions.Messages.StandardOutputProperty.PrintMembers(System.Text.StringBuilder! builder) -> bool diff --git a/src/Platform/Microsoft.Testing.Platform/TestFramework/MaxFailedTestsCommandLineOptionsProvider.cs b/src/Platform/Microsoft.Testing.Platform/TestFramework/MaxFailedTestsCommandLineOptionsProvider.cs index 71390c27e7..9e02dd0a9d 100644 --- a/src/Platform/Microsoft.Testing.Platform/TestFramework/MaxFailedTestsCommandLineOptionsProvider.cs +++ b/src/Platform/Microsoft.Testing.Platform/TestFramework/MaxFailedTestsCommandLineOptionsProvider.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +using System.Diagnostics.CodeAnalysis; using System.Globalization; using Microsoft.Testing.Platform.CommandLine; @@ -11,6 +12,7 @@ namespace Microsoft.Testing.Platform.TestFramework; +[Experimental("TPEXP", UrlFormat = "https://aka.ms/testingplatform/diagnostics#{0}")] public sealed class MaxFailedTestsCommandLineOptionsProvider : ICommandLineOptionsProvider { // TODO: We have 'minimum-expected-tests', so should we use "maximum" instead of "max" here as well for consistency? From 1634d4041e62081119d5cd3ab431c5bcb8bd4e7a Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Fri, 6 Dec 2024 19:45:59 +0100 Subject: [PATCH 29/64] Small fix --- .../TestFramework/MaxFailedTestsCommandLineOptionsProvider.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Platform/Microsoft.Testing.Platform/TestFramework/MaxFailedTestsCommandLineOptionsProvider.cs b/src/Platform/Microsoft.Testing.Platform/TestFramework/MaxFailedTestsCommandLineOptionsProvider.cs index 9e02dd0a9d..863e39a256 100644 --- a/src/Platform/Microsoft.Testing.Platform/TestFramework/MaxFailedTestsCommandLineOptionsProvider.cs +++ b/src/Platform/Microsoft.Testing.Platform/TestFramework/MaxFailedTestsCommandLineOptionsProvider.cs @@ -36,7 +36,8 @@ public IReadOnlyCollection GetCommandLineOptions() public Task IsEnabledAsync() => Task.FromResult(true); - public Task ValidateCommandLineOptionsAsync(ICommandLineOptions commandLineOptions) => throw new NotImplementedException(); + public Task ValidateCommandLineOptionsAsync(ICommandLineOptions commandLineOptions) + => ValidationResult.ValidTask; public Task ValidateOptionArgumentsAsync(CommandLineOption commandOption, string[] arguments) { From d2a2c82924784fe285faf0f0c6c2309d76c68bd1 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Fri, 6 Dec 2024 20:06:55 +0100 Subject: [PATCH 30/64] Fix test failures --- .../MSTest.Acceptance.IntegrationTests/HelpInfoTests.cs | 2 ++ .../Services/TestApplicationResultTests.cs | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/HelpInfoTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/HelpInfoTests.cs index 8894ffbc2d..e2aeac66b3 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/HelpInfoTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/HelpInfoTests.cs @@ -59,6 +59,8 @@ Show the command line help. Display .NET test application information. --list-tests List available tests. + --max-failed-tests + Specifies a maximum number of test failures that, when exceeded, will abort the test run. --minimum-expected-tests Specifies the minimum number of tests that are expected to run. --results-directory diff --git a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Services/TestApplicationResultTests.cs b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Services/TestApplicationResultTests.cs index 6da2696496..a2bccd4168 100644 --- a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Services/TestApplicationResultTests.cs +++ b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Services/TestApplicationResultTests.cs @@ -77,7 +77,7 @@ public async Task GetProcessExitCodeAsync_If_Canceled_Returns_TestSessionAborted }); TestApplicationResult testApplicationResult - = new(new Mock().Object, new Mock().Object, new Mock().Object, new Mock().Object); + = new(new Mock().Object, new Mock().Object, new Mock().Object, new StopPoliciesService(testApplicationCancellationTokenSource.Object)); await testApplicationResult.ConsumeAsync(new DummyProducer(), new TestNodeUpdateMessage( default, From 60f6e416d5405422663e2b63b6854c2673c17e1e Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Sat, 7 Dec 2024 09:00:27 +0100 Subject: [PATCH 31/64] Adjust cancellation --- .../Services/StopPoliciesService.cs | 16 ++++++++++++++-- .../Services/TestApplicationResultTests.cs | 5 +++-- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/Platform/Microsoft.Testing.Platform/Services/StopPoliciesService.cs b/src/Platform/Microsoft.Testing.Platform/Services/StopPoliciesService.cs index ccd58e4fb0..e2b9a1aff0 100644 --- a/src/Platform/Microsoft.Testing.Platform/Services/StopPoliciesService.cs +++ b/src/Platform/Microsoft.Testing.Platform/Services/StopPoliciesService.cs @@ -7,10 +7,22 @@ namespace Microsoft.Testing.Platform.Services; internal sealed class StopPoliciesService : IStopPoliciesService { - public StopPoliciesService(ITestApplicationCancellationTokenSource testApplicationCancellationTokenSource) => + public StopPoliciesService(ITestApplicationCancellationTokenSource testApplicationCancellationTokenSource) + { + // This happens in mocked tests because the test will do the Cancel on the first CancellationToken property access. + // In theory, cancellation may happen in practice fast enough before StopPoliciesService is created. + // TODO: Do we have a race here? i.e, IsCancellationRequested is seen as false, then cancel happen before the CT.Register happens? + if (testApplicationCancellationTokenSource.CancellationToken.IsCancellationRequested) + { + _ = ExecuteAbortCallbacksAsync(); + } + else + { #pragma warning disable VSTHRD101 // Avoid unsupported async delegates - testApplicationCancellationTokenSource.CancellationToken.Register(async () => await ExecuteAbortCallbacksAsync()); + testApplicationCancellationTokenSource.CancellationToken.Register(async () => await ExecuteAbortCallbacksAsync()); #pragma warning restore VSTHRD101 // Avoid unsupported async delegates + } + } private BlockingCollection>? _maxFailedTestsCallbacks; private BlockingCollection>? _abortCallbacks; diff --git a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Services/TestApplicationResultTests.cs b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Services/TestApplicationResultTests.cs index a2bccd4168..804d5aecf0 100644 --- a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Services/TestApplicationResultTests.cs +++ b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Services/TestApplicationResultTests.cs @@ -68,10 +68,11 @@ public async Task GetProcessExitCodeAsync_If_Failed_Tests_Returns_AtLeastOneTest public async Task GetProcessExitCodeAsync_If_Canceled_Returns_TestSessionAborted() { Mock testApplicationCancellationTokenSource = new(); + // CTS should not be created in SetupGet so that the mocked ITestApplicationCancellationTokenSource returns the same instance on every access + // which is the case in the real production implementation. + CancellationTokenSource cancellationTokenSource = new(); testApplicationCancellationTokenSource.SetupGet(x => x.CancellationToken).Returns(() => { - CancellationTokenSource cancellationTokenSource = new(); - cancellationTokenSource.Cancel(); return cancellationTokenSource.Token; }); From 0a9999a1bbfc42084bcfe9b4cb69bc685dca0932 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Sat, 7 Dec 2024 09:20:14 +0100 Subject: [PATCH 32/64] Adjust for potential race conditions --- .../Services/StopPoliciesService.cs | 36 +++++++++++++++---- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/src/Platform/Microsoft.Testing.Platform/Services/StopPoliciesService.cs b/src/Platform/Microsoft.Testing.Platform/Services/StopPoliciesService.cs index e2b9a1aff0..eaca20bd11 100644 --- a/src/Platform/Microsoft.Testing.Platform/Services/StopPoliciesService.cs +++ b/src/Platform/Microsoft.Testing.Platform/Services/StopPoliciesService.cs @@ -7,21 +7,32 @@ namespace Microsoft.Testing.Platform.Services; internal sealed class StopPoliciesService : IStopPoliciesService { + private readonly Lock _abortLock = new(); + public StopPoliciesService(ITestApplicationCancellationTokenSource testApplicationCancellationTokenSource) { +#pragma warning disable VSTHRD101 // Avoid unsupported async delegates + testApplicationCancellationTokenSource.CancellationToken.Register(async () => await ExecuteAbortCallbacksAsync()); +#pragma warning restore VSTHRD101 // Avoid unsupported async delegates + // This happens in mocked tests because the test will do the Cancel on the first CancellationToken property access. // In theory, cancellation may happen in practice fast enough before StopPoliciesService is created. - // TODO: Do we have a race here? i.e, IsCancellationRequested is seen as false, then cancel happen before the CT.Register happens? if (testApplicationCancellationTokenSource.CancellationToken.IsCancellationRequested) { _ = ExecuteAbortCallbacksAsync(); } - else - { -#pragma warning disable VSTHRD101 // Avoid unsupported async delegates - testApplicationCancellationTokenSource.CancellationToken.Register(async () => await ExecuteAbortCallbacksAsync()); -#pragma warning restore VSTHRD101 // Avoid unsupported async delegates - } + + // NOTE::: Don't move the CancellationToken.Register call to an "else" here. + // If the call is moved to else, then this race may happen: + // 1. Check IsCancellationRequested -> false + // 2. Cancellation happens + // 3. Go to the "else" and do CancellationToken.Register which will never happen because + // cancellation already happened after IsCancellationRequested check and before CancellationToken.Register. + // + // With the current implementation above, we always do the Register first. + // Then, if IsCancellationRequested was false, we are sure the register already happened and we get the callback when cancelled. + // However, if we found IsCancellationRequested to be true, we are not sure if cancellation happened before Register or not. + // So, we may call ExecuteAbortCallbacksAsync twice. This is fine, we handle that with a lock inside ExecuteAbortCallbacksAsync. } private BlockingCollection>? _maxFailedTestsCallbacks; @@ -36,6 +47,7 @@ private static void RegisterCallback(ref BlockingCollection? callbacks, T public async Task ExecuteMaxFailedTestsCallbacksAsync(int maxFailedTests, CancellationToken cancellationToken) { + IsMaxFailedTestsTriggered = true; if (_maxFailedTestsCallbacks is null) { return; @@ -49,6 +61,16 @@ public async Task ExecuteMaxFailedTestsCallbacksAsync(int maxFailedTests, Cancel public async Task ExecuteAbortCallbacksAsync() { + lock (_abortLock) + { + if (IsAbortTriggered) + { + return; + } + + IsAbortTriggered = true; + } + if (_abortCallbacks is null) { return; From bf81e2066acee8f5a8f15716fe7fda3c92018794 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Mon, 9 Dec 2024 08:26:42 +0100 Subject: [PATCH 33/64] Rename --- .../Helpers/TestApplicationBuilderExtensions.cs | 2 +- .../IGracefulStopTestExecutionCapability.cs | 2 +- .../Resources/PlatformResources.resx | 6 +++--- .../Resources/xlf/PlatformResources.cs.xlf | 12 ++++++------ .../Resources/xlf/PlatformResources.de.xlf | 12 ++++++------ .../Resources/xlf/PlatformResources.es.xlf | 12 ++++++------ .../Resources/xlf/PlatformResources.fr.xlf | 12 ++++++------ .../Resources/xlf/PlatformResources.it.xlf | 12 ++++++------ .../Resources/xlf/PlatformResources.ja.xlf | 12 ++++++------ .../Resources/xlf/PlatformResources.ko.xlf | 12 ++++++------ .../Resources/xlf/PlatformResources.pl.xlf | 12 ++++++------ .../Resources/xlf/PlatformResources.pt-BR.xlf | 12 ++++++------ .../Resources/xlf/PlatformResources.ru.xlf | 12 ++++++------ .../Resources/xlf/PlatformResources.tr.xlf | 12 ++++++------ .../Resources/xlf/PlatformResources.zh-Hans.xlf | 12 ++++++------ .../Resources/xlf/PlatformResources.zh-Hant.xlf | 12 ++++++------ .../MaxFailedTestsCommandLineOptionsProvider.cs | 2 +- .../HelpInfoTests.cs | 2 -- 18 files changed, 84 insertions(+), 86 deletions(-) diff --git a/src/Platform/Microsoft.Testing.Extensions.VSTestBridge/Helpers/TestApplicationBuilderExtensions.cs b/src/Platform/Microsoft.Testing.Extensions.VSTestBridge/Helpers/TestApplicationBuilderExtensions.cs index 7cc89ae29b..97aeeb6548 100644 --- a/src/Platform/Microsoft.Testing.Extensions.VSTestBridge/Helpers/TestApplicationBuilderExtensions.cs +++ b/src/Platform/Microsoft.Testing.Extensions.VSTestBridge/Helpers/TestApplicationBuilderExtensions.cs @@ -49,7 +49,7 @@ public static void AddTestRunParametersService(this ITestApplicationBuilder buil => builder.CommandLine.AddProvider(() => new TestRunParametersCommandLineOptionsProvider(extension)); /// - /// Registers the command-line options provider for '--max-failed-tests'. + /// Registers the command-line options provider for '--maximum-failed-tests'. /// /// The test application builder. public static void AddMaxFailedTestsService(this ITestApplicationBuilder builder) diff --git a/src/Platform/Microsoft.Testing.Platform/Capabilities/TestFramework/IGracefulStopTestExecutionCapability.cs b/src/Platform/Microsoft.Testing.Platform/Capabilities/TestFramework/IGracefulStopTestExecutionCapability.cs index 6f0815e440..ee8a92dee3 100644 --- a/src/Platform/Microsoft.Testing.Platform/Capabilities/TestFramework/IGracefulStopTestExecutionCapability.cs +++ b/src/Platform/Microsoft.Testing.Platform/Capabilities/TestFramework/IGracefulStopTestExecutionCapability.cs @@ -7,7 +7,7 @@ namespace Microsoft.Testing.Platform.Capabilities.TestFramework; /// /// A capability to support stopping test execution gracefully, without cancelling/aborting everything. -/// This is used to support '--max-failed-tests'. +/// This is used to support '--maximum-failed-tests'. /// /// /// Test frameworks can choose to run any needed cleanup when cancellation is requested. diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/PlatformResources.resx b/src/Platform/Microsoft.Testing.Platform/Resources/PlatformResources.resx index f19bf8f0fc..4ecc51402f 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/PlatformResources.resx +++ b/src/Platform/Microsoft.Testing.Platform/Resources/PlatformResources.resx @@ -698,13 +698,13 @@ Takes one argument as string in the format <value>[h|m|s] where 'value' is Specifies a maximum number of test failures that, when exceeded, will abort the test run. - The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid. + The option '--maximum-failed-tests' must be a positive integer. The value '{0}' is not valid. - Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + Extension used to support '--maximum-failed-tests'. When a given failures threshold is reached, the test run will be aborted. - Test session is aborting due to reaching failures ('{0}') specified by the '--max-failed-tests' option. + Test session is aborting due to reaching failures ('{0}') specified by the '--maximum-failed-tests' option. {0} is the number of max failed tests. diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.cs.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.cs.xlf index a0ec2301ab..12940b3af5 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.cs.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.cs.xlf @@ -3,8 +3,8 @@ - Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted. - Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + Extension used to support '--maximum-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + Extension used to support '--maximum-failed-tests'. When a given failures threshold is reached, the test run will be aborted. @@ -377,8 +377,8 @@ - The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid. - The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid. + The option '--maximum-failed-tests' must be a positive integer. The value '{0}' is not valid. + The option '--maximum-failed-tests' must be a positive integer. The value '{0}' is not valid. @@ -629,8 +629,8 @@ Může mít jenom jeden argument jako řetězec ve formátu <value>[h|m|s] - Test session is aborting due to reaching failures ('{0}') specified by the '--max-failed-tests' option. - Test session is aborting due to reaching failures ('{0}') specified by the '--max-failed-tests' option. + Test session is aborting due to reaching failures ('{0}') specified by the '--maximum-failed-tests' option. + Test session is aborting due to reaching failures ('{0}') specified by the '--maximum-failed-tests' option. {0} is the number of max failed tests. diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.de.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.de.xlf index e6a2d5927d..1d6ffed6a4 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.de.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.de.xlf @@ -3,8 +3,8 @@ - Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted. - Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + Extension used to support '--maximum-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + Extension used to support '--maximum-failed-tests'. When a given failures threshold is reached, the test run will be aborted. @@ -377,8 +377,8 @@ - The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid. - The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid. + The option '--maximum-failed-tests' must be a positive integer. The value '{0}' is not valid. + The option '--maximum-failed-tests' must be a positive integer. The value '{0}' is not valid. @@ -629,8 +629,8 @@ Nimmt ein Argument als Zeichenfolge im Format <value>[h|m|s], wobei "value - Test session is aborting due to reaching failures ('{0}') specified by the '--max-failed-tests' option. - Test session is aborting due to reaching failures ('{0}') specified by the '--max-failed-tests' option. + Test session is aborting due to reaching failures ('{0}') specified by the '--maximum-failed-tests' option. + Test session is aborting due to reaching failures ('{0}') specified by the '--maximum-failed-tests' option. {0} is the number of max failed tests. diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.es.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.es.xlf index 4264208a7f..e55ae0c673 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.es.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.es.xlf @@ -3,8 +3,8 @@ - Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted. - Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + Extension used to support '--maximum-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + Extension used to support '--maximum-failed-tests'. When a given failures threshold is reached, the test run will be aborted. @@ -377,8 +377,8 @@ - The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid. - The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid. + The option '--maximum-failed-tests' must be a positive integer. The value '{0}' is not valid. + The option '--maximum-failed-tests' must be a positive integer. The value '{0}' is not valid. @@ -629,8 +629,8 @@ Toma un argumento como cadena con el formato <value>[h|m|s] donde 'value' - Test session is aborting due to reaching failures ('{0}') specified by the '--max-failed-tests' option. - Test session is aborting due to reaching failures ('{0}') specified by the '--max-failed-tests' option. + Test session is aborting due to reaching failures ('{0}') specified by the '--maximum-failed-tests' option. + Test session is aborting due to reaching failures ('{0}') specified by the '--maximum-failed-tests' option. {0} is the number of max failed tests. diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.fr.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.fr.xlf index fa14196f93..fff94cb5d4 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.fr.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.fr.xlf @@ -3,8 +3,8 @@ - Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted. - Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + Extension used to support '--maximum-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + Extension used to support '--maximum-failed-tests'. When a given failures threshold is reached, the test run will be aborted. @@ -377,8 +377,8 @@ - The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid. - The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid. + The option '--maximum-failed-tests' must be a positive integer. The value '{0}' is not valid. + The option '--maximum-failed-tests' must be a positive integer. The value '{0}' is not valid. @@ -629,8 +629,8 @@ Prend un argument sous forme de chaîne au format <value>[h|m|s] où « v - Test session is aborting due to reaching failures ('{0}') specified by the '--max-failed-tests' option. - Test session is aborting due to reaching failures ('{0}') specified by the '--max-failed-tests' option. + Test session is aborting due to reaching failures ('{0}') specified by the '--maximum-failed-tests' option. + Test session is aborting due to reaching failures ('{0}') specified by the '--maximum-failed-tests' option. {0} is the number of max failed tests. diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.it.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.it.xlf index bb1fcefb19..73e36ea530 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.it.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.it.xlf @@ -3,8 +3,8 @@ - Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted. - Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + Extension used to support '--maximum-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + Extension used to support '--maximum-failed-tests'. When a given failures threshold is reached, the test run will be aborted. @@ -377,8 +377,8 @@ - The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid. - The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid. + The option '--maximum-failed-tests' must be a positive integer. The value '{0}' is not valid. + The option '--maximum-failed-tests' must be a positive integer. The value '{0}' is not valid. @@ -629,8 +629,8 @@ Acquisisce un argomento come stringa nel formato <value>[h|m|s] dove 'valu - Test session is aborting due to reaching failures ('{0}') specified by the '--max-failed-tests' option. - Test session is aborting due to reaching failures ('{0}') specified by the '--max-failed-tests' option. + Test session is aborting due to reaching failures ('{0}') specified by the '--maximum-failed-tests' option. + Test session is aborting due to reaching failures ('{0}') specified by the '--maximum-failed-tests' option. {0} is the number of max failed tests. diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ja.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ja.xlf index 645f258331..584f0dec07 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ja.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ja.xlf @@ -3,8 +3,8 @@ - Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted. - Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + Extension used to support '--maximum-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + Extension used to support '--maximum-failed-tests'. When a given failures threshold is reached, the test run will be aborted. @@ -377,8 +377,8 @@ - The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid. - The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid. + The option '--maximum-failed-tests' must be a positive integer. The value '{0}' is not valid. + The option '--maximum-failed-tests' must be a positive integer. The value '{0}' is not valid. @@ -630,8 +630,8 @@ Takes one argument as string in the format <value>[h|m|s] where 'value' is - Test session is aborting due to reaching failures ('{0}') specified by the '--max-failed-tests' option. - Test session is aborting due to reaching failures ('{0}') specified by the '--max-failed-tests' option. + Test session is aborting due to reaching failures ('{0}') specified by the '--maximum-failed-tests' option. + Test session is aborting due to reaching failures ('{0}') specified by the '--maximum-failed-tests' option. {0} is the number of max failed tests. diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ko.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ko.xlf index abcce5e080..e722f501a6 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ko.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ko.xlf @@ -3,8 +3,8 @@ - Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted. - Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + Extension used to support '--maximum-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + Extension used to support '--maximum-failed-tests'. When a given failures threshold is reached, the test run will be aborted. @@ -377,8 +377,8 @@ - The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid. - The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid. + The option '--maximum-failed-tests' must be a positive integer. The value '{0}' is not valid. + The option '--maximum-failed-tests' must be a positive integer. The value '{0}' is not valid. @@ -629,8 +629,8 @@ Takes one argument as string in the format <value>[h|m|s] where 'value' is - Test session is aborting due to reaching failures ('{0}') specified by the '--max-failed-tests' option. - Test session is aborting due to reaching failures ('{0}') specified by the '--max-failed-tests' option. + Test session is aborting due to reaching failures ('{0}') specified by the '--maximum-failed-tests' option. + Test session is aborting due to reaching failures ('{0}') specified by the '--maximum-failed-tests' option. {0} is the number of max failed tests. diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pl.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pl.xlf index ae0667f335..87b9b7ce46 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pl.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pl.xlf @@ -3,8 +3,8 @@ - Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted. - Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + Extension used to support '--maximum-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + Extension used to support '--maximum-failed-tests'. When a given failures threshold is reached, the test run will be aborted. @@ -377,8 +377,8 @@ - The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid. - The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid. + The option '--maximum-failed-tests' must be a positive integer. The value '{0}' is not valid. + The option '--maximum-failed-tests' must be a positive integer. The value '{0}' is not valid. @@ -629,8 +629,8 @@ Pobiera jeden argument jako ciąg w formacie <value>[h|m|s], gdzie element - Test session is aborting due to reaching failures ('{0}') specified by the '--max-failed-tests' option. - Test session is aborting due to reaching failures ('{0}') specified by the '--max-failed-tests' option. + Test session is aborting due to reaching failures ('{0}') specified by the '--maximum-failed-tests' option. + Test session is aborting due to reaching failures ('{0}') specified by the '--maximum-failed-tests' option. {0} is the number of max failed tests. diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pt-BR.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pt-BR.xlf index c8e311c033..6a29715741 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pt-BR.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pt-BR.xlf @@ -3,8 +3,8 @@ - Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted. - Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + Extension used to support '--maximum-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + Extension used to support '--maximum-failed-tests'. When a given failures threshold is reached, the test run will be aborted. @@ -377,8 +377,8 @@ - The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid. - The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid. + The option '--maximum-failed-tests' must be a positive integer. The value '{0}' is not valid. + The option '--maximum-failed-tests' must be a positive integer. The value '{0}' is not valid. @@ -629,8 +629,8 @@ Recebe um argumento como cadeia de caracteres no formato <valor>[h|m|s] em - Test session is aborting due to reaching failures ('{0}') specified by the '--max-failed-tests' option. - Test session is aborting due to reaching failures ('{0}') specified by the '--max-failed-tests' option. + Test session is aborting due to reaching failures ('{0}') specified by the '--maximum-failed-tests' option. + Test session is aborting due to reaching failures ('{0}') specified by the '--maximum-failed-tests' option. {0} is the number of max failed tests. diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ru.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ru.xlf index 50e6881be8..1ba4e6fd10 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ru.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ru.xlf @@ -3,8 +3,8 @@ - Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted. - Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + Extension used to support '--maximum-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + Extension used to support '--maximum-failed-tests'. When a given failures threshold is reached, the test run will be aborted. @@ -377,8 +377,8 @@ - The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid. - The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid. + The option '--maximum-failed-tests' must be a positive integer. The value '{0}' is not valid. + The option '--maximum-failed-tests' must be a positive integer. The value '{0}' is not valid. @@ -629,8 +629,8 @@ Takes one argument as string in the format <value>[h|m|s] where 'value' is - Test session is aborting due to reaching failures ('{0}') specified by the '--max-failed-tests' option. - Test session is aborting due to reaching failures ('{0}') specified by the '--max-failed-tests' option. + Test session is aborting due to reaching failures ('{0}') specified by the '--maximum-failed-tests' option. + Test session is aborting due to reaching failures ('{0}') specified by the '--maximum-failed-tests' option. {0} is the number of max failed tests. diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.tr.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.tr.xlf index b9c4d4dd79..a688ecf09c 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.tr.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.tr.xlf @@ -3,8 +3,8 @@ - Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted. - Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + Extension used to support '--maximum-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + Extension used to support '--maximum-failed-tests'. When a given failures threshold is reached, the test run will be aborted. @@ -377,8 +377,8 @@ - The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid. - The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid. + The option '--maximum-failed-tests' must be a positive integer. The value '{0}' is not valid. + The option '--maximum-failed-tests' must be a positive integer. The value '{0}' is not valid. @@ -629,8 +629,8 @@ Bir bağımsız değişkeni, 'value' değerinin kayan olduğu <value>[h|m| - Test session is aborting due to reaching failures ('{0}') specified by the '--max-failed-tests' option. - Test session is aborting due to reaching failures ('{0}') specified by the '--max-failed-tests' option. + Test session is aborting due to reaching failures ('{0}') specified by the '--maximum-failed-tests' option. + Test session is aborting due to reaching failures ('{0}') specified by the '--maximum-failed-tests' option. {0} is the number of max failed tests. diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hans.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hans.xlf index 49cd8b43c6..724c4e30b8 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hans.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hans.xlf @@ -3,8 +3,8 @@ - Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted. - Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + Extension used to support '--maximum-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + Extension used to support '--maximum-failed-tests'. When a given failures threshold is reached, the test run will be aborted. @@ -377,8 +377,8 @@ - The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid. - The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid. + The option '--maximum-failed-tests' must be a positive integer. The value '{0}' is not valid. + The option '--maximum-failed-tests' must be a positive integer. The value '{0}' is not valid. @@ -629,8 +629,8 @@ Takes one argument as string in the format <value>[h|m|s] where 'value' is - Test session is aborting due to reaching failures ('{0}') specified by the '--max-failed-tests' option. - Test session is aborting due to reaching failures ('{0}') specified by the '--max-failed-tests' option. + Test session is aborting due to reaching failures ('{0}') specified by the '--maximum-failed-tests' option. + Test session is aborting due to reaching failures ('{0}') specified by the '--maximum-failed-tests' option. {0} is the number of max failed tests. diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hant.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hant.xlf index 4b890accf8..88908fcba5 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hant.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hant.xlf @@ -3,8 +3,8 @@ - Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted. - Extension used to support '--max-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + Extension used to support '--maximum-failed-tests'. When a given failures threshold is reached, the test run will be aborted. + Extension used to support '--maximum-failed-tests'. When a given failures threshold is reached, the test run will be aborted. @@ -377,8 +377,8 @@ - The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid. - The option '--max-failed-tests' must be a positive integer. The value '{0}' is not valid. + The option '--maximum-failed-tests' must be a positive integer. The value '{0}' is not valid. + The option '--maximum-failed-tests' must be a positive integer. The value '{0}' is not valid. @@ -629,8 +629,8 @@ Takes one argument as string in the format <value>[h|m|s] where 'value' is - Test session is aborting due to reaching failures ('{0}') specified by the '--max-failed-tests' option. - Test session is aborting due to reaching failures ('{0}') specified by the '--max-failed-tests' option. + Test session is aborting due to reaching failures ('{0}') specified by the '--maximum-failed-tests' option. + Test session is aborting due to reaching failures ('{0}') specified by the '--maximum-failed-tests' option. {0} is the number of max failed tests. diff --git a/src/Platform/Microsoft.Testing.Platform/TestFramework/MaxFailedTestsCommandLineOptionsProvider.cs b/src/Platform/Microsoft.Testing.Platform/TestFramework/MaxFailedTestsCommandLineOptionsProvider.cs index 863e39a256..534bee263f 100644 --- a/src/Platform/Microsoft.Testing.Platform/TestFramework/MaxFailedTestsCommandLineOptionsProvider.cs +++ b/src/Platform/Microsoft.Testing.Platform/TestFramework/MaxFailedTestsCommandLineOptionsProvider.cs @@ -16,7 +16,7 @@ namespace Microsoft.Testing.Platform.TestFramework; public sealed class MaxFailedTestsCommandLineOptionsProvider : ICommandLineOptionsProvider { // TODO: We have 'minimum-expected-tests', so should we use "maximum" instead of "max" here as well for consistency? - internal const string MaxFailedTestsOptionKey = "max-failed-tests"; + internal const string MaxFailedTestsOptionKey = "maximum-failed-tests"; private static readonly IReadOnlyCollection OptionsCache = [ diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/HelpInfoTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/HelpInfoTests.cs index e2aeac66b3..8894ffbc2d 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/HelpInfoTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/HelpInfoTests.cs @@ -59,8 +59,6 @@ Show the command line help. Display .NET test application information. --list-tests List available tests. - --max-failed-tests - Specifies a maximum number of test failures that, when exceeded, will abort the test run. --minimum-expected-tests Specifies the minimum number of tests that are expected to run. --results-directory From 5fb51f4506b5621d247de9a1e6b1286b6eb4d687 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Mon, 9 Dec 2024 10:51:24 +0100 Subject: [PATCH 34/64] MSTest side of the feature --- .../Execution/ClassCleanupManager.cs | 15 +++++++++++++++ .../Execution/TestExecutionManager.cs | 5 +++++ .../Execution/UnitTestRunner.cs | 2 ++ 3 files changed, 22 insertions(+) diff --git a/src/Adapter/MSTest.TestAdapter/Execution/ClassCleanupManager.cs b/src/Adapter/MSTest.TestAdapter/Execution/ClassCleanupManager.cs index 8f6d5d75f2..a97db594f9 100644 --- a/src/Adapter/MSTest.TestAdapter/Execution/ClassCleanupManager.cs +++ b/src/Adapter/MSTest.TestAdapter/Execution/ClassCleanupManager.cs @@ -62,4 +62,19 @@ public void MarkTestComplete(TestMethodInfo testMethodInfo, TestMethod testMetho ShouldRunEndOfAssemblyCleanup = _remainingTestsByClass.IsEmpty; } } + + internal void ForceCleanup(TypeCache typeCache) + { + IEnumerable classInfoCache = typeCache.ClassInfoListWithExecutableCleanupMethods; + foreach (TestClassInfo classInfo in classInfoCache) + { + classInfo.ExecuteClassCleanup(); + } + + IEnumerable assemblyInfoCache = typeCache.AssemblyInfoListWithExecutableCleanupMethods; + foreach (TestAssemblyInfo assemblyInfo in assemblyInfoCache) + { + assemblyInfo.ExecuteAssemblyCleanup(); + } + } } diff --git a/src/Adapter/MSTest.TestAdapter/Execution/TestExecutionManager.cs b/src/Adapter/MSTest.TestAdapter/Execution/TestExecutionManager.cs index 99461b05bc..c676d2932c 100644 --- a/src/Adapter/MSTest.TestAdapter/Execution/TestExecutionManager.cs +++ b/src/Adapter/MSTest.TestAdapter/Execution/TestExecutionManager.cs @@ -399,6 +399,11 @@ private void ExecuteTestsInSource(IEnumerable tests, IRunContext? runC ExecuteTestsWithTestRunner(testsToRun, frameworkHandle, source, sourceLevelParameters, testRunner); } + if (MSTestStopGracefullyTestExecutionCapability.Instance.IsStopRequested) + { + testRunner.ForceCleanup(); + } + PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo("Executed tests belonging to source {0}", source); } diff --git a/src/Adapter/MSTest.TestAdapter/Execution/UnitTestRunner.cs b/src/Adapter/MSTest.TestAdapter/Execution/UnitTestRunner.cs index fae1a9d7bc..aabe37dbaa 100644 --- a/src/Adapter/MSTest.TestAdapter/Execution/UnitTestRunner.cs +++ b/src/Adapter/MSTest.TestAdapter/Execution/UnitTestRunner.cs @@ -367,4 +367,6 @@ private bool IsTestMethodRunnable( notRunnableResult = null; return true; } + + internal void ForceCleanup() => _classCleanupManager.ForceCleanup(_typeCache); } From f928c078f2d1f6ed05605d6378c0aeabbe00a641 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Mon, 9 Dec 2024 11:39:51 +0100 Subject: [PATCH 35/64] Progress --- .../Execution/ClassCleanupManager.cs | 2 +- .../MSTest.TestAdapter/Execution/UnitTestRunner.cs | 2 +- .../Extensions/AbortForMaxFailedTestsExtension.cs | 2 +- .../MaxFailedTestsCommandLineOptionsProvider.cs | 10 ++++++---- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/Adapter/MSTest.TestAdapter/Execution/ClassCleanupManager.cs b/src/Adapter/MSTest.TestAdapter/Execution/ClassCleanupManager.cs index a97db594f9..6cd4c78279 100644 --- a/src/Adapter/MSTest.TestAdapter/Execution/ClassCleanupManager.cs +++ b/src/Adapter/MSTest.TestAdapter/Execution/ClassCleanupManager.cs @@ -63,7 +63,7 @@ public void MarkTestComplete(TestMethodInfo testMethodInfo, TestMethod testMetho } } - internal void ForceCleanup(TypeCache typeCache) + internal static void ForceCleanup(TypeCache typeCache) { IEnumerable classInfoCache = typeCache.ClassInfoListWithExecutableCleanupMethods; foreach (TestClassInfo classInfo in classInfoCache) diff --git a/src/Adapter/MSTest.TestAdapter/Execution/UnitTestRunner.cs b/src/Adapter/MSTest.TestAdapter/Execution/UnitTestRunner.cs index aabe37dbaa..a05a8afb13 100644 --- a/src/Adapter/MSTest.TestAdapter/Execution/UnitTestRunner.cs +++ b/src/Adapter/MSTest.TestAdapter/Execution/UnitTestRunner.cs @@ -368,5 +368,5 @@ private bool IsTestMethodRunnable( return true; } - internal void ForceCleanup() => _classCleanupManager.ForceCleanup(_typeCache); + internal void ForceCleanup() => ClassCleanupManager.ForceCleanup(_typeCache); } diff --git a/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs b/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs index 3e5c86f5f8..b92fa71862 100644 --- a/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs +++ b/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs @@ -29,7 +29,7 @@ public AbortForMaxFailedTestsExtension( { if (commandLineOptions.TryGetOptionArgumentList(MaxFailedTestsCommandLineOptionsProvider.MaxFailedTestsOptionKey, out string[]? args) && int.TryParse(args[0], out int maxFailedTests) && - maxFailedTests > 0) + maxFailedTests >= 0) { _maxFailedTests = maxFailedTests; } diff --git a/src/Platform/Microsoft.Testing.Platform/TestFramework/MaxFailedTestsCommandLineOptionsProvider.cs b/src/Platform/Microsoft.Testing.Platform/TestFramework/MaxFailedTestsCommandLineOptionsProvider.cs index 534bee263f..8903fde1d4 100644 --- a/src/Platform/Microsoft.Testing.Platform/TestFramework/MaxFailedTestsCommandLineOptionsProvider.cs +++ b/src/Platform/Microsoft.Testing.Platform/TestFramework/MaxFailedTestsCommandLineOptionsProvider.cs @@ -44,10 +44,12 @@ public Task ValidateOptionArgumentsAsync(CommandLineOption com if (commandOption.Name == MaxFailedTestsOptionKey) { string arg = arguments[0]; - if (!int.TryParse(arg, out int maxFailedTestsResult) || maxFailedTestsResult <= 0) - { - return ValidationResult.InvalidTask(string.Format(CultureInfo.InvariantCulture, PlatformResources.MaxFailedTestsMustBePositive, arg)); - } + // We consider --maximum-failed-tests 0 as valid. + // The idea is that we stop the execution when we *exceed* the max failed tests, not when *reach*. + // So zero means, stop execution on the first failure. + return int.TryParse(arg, out int maxFailedTestsResult) && maxFailedTestsResult >= 0 + ? ValidationResult.ValidTask + : ValidationResult.InvalidTask(string.Format(CultureInfo.InvariantCulture, PlatformResources.MaxFailedTestsMustBePositive, arg)); } throw ApplicationStateGuard.Unreachable(); From 43ba49edf2a3cd7a458c483b3410b6fefef67148 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Mon, 9 Dec 2024 12:14:28 +0100 Subject: [PATCH 36/64] Minor fixes --- .../Extensions/AbortForMaxFailedTestsExtension.cs | 4 +++- .../TestFramework/MaxFailedTestsCommandLineOptionsProvider.cs | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs b/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs index b92fa71862..84aa56de6b 100644 --- a/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs +++ b/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs @@ -66,7 +66,9 @@ public async Task ConsumeAsync(IDataProducer dataProducer, IData value, Cancella TestNodeStateProperty testNodeStateProperty = node.TestNode.Properties.Single(); if (TestNodePropertiesCategories.WellKnownTestNodeTestRunOutcomeFailedProperties.Any(t => t == testNodeStateProperty.GetType()) && - ++_failCount > _maxFailedTests.Value) + ++_failCount > _maxFailedTests.Value && + // If already triggered, don't do it again. + _policiesService.IsMaxFailedTestsTriggered) { await _capability.StopTestExecutionAsync(_testApplicationCancellationTokenSource.CancellationToken); await _policiesService.ExecuteMaxFailedTestsCallbacksAsync(_maxFailedTests.Value, _testApplicationCancellationTokenSource.CancellationToken); diff --git a/src/Platform/Microsoft.Testing.Platform/TestFramework/MaxFailedTestsCommandLineOptionsProvider.cs b/src/Platform/Microsoft.Testing.Platform/TestFramework/MaxFailedTestsCommandLineOptionsProvider.cs index 8903fde1d4..1b198be0ae 100644 --- a/src/Platform/Microsoft.Testing.Platform/TestFramework/MaxFailedTestsCommandLineOptionsProvider.cs +++ b/src/Platform/Microsoft.Testing.Platform/TestFramework/MaxFailedTestsCommandLineOptionsProvider.cs @@ -15,7 +15,6 @@ namespace Microsoft.Testing.Platform.TestFramework; [Experimental("TPEXP", UrlFormat = "https://aka.ms/testingplatform/diagnostics#{0}")] public sealed class MaxFailedTestsCommandLineOptionsProvider : ICommandLineOptionsProvider { - // TODO: We have 'minimum-expected-tests', so should we use "maximum" instead of "max" here as well for consistency? internal const string MaxFailedTestsOptionKey = "maximum-failed-tests"; private static readonly IReadOnlyCollection OptionsCache = From 7199f00917bdd0d73d84fced0f3adff298f66b3c Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Mon, 9 Dec 2024 12:35:18 +0100 Subject: [PATCH 37/64] Cleanup --- .../TestApplicationBuilderExtensions.cs | 5 ++++- .../Helpers/TestApplicationBuilderExtensions.cs | 9 --------- .../MaxFailedTestsCommandLineOptionsProvider.cs | 15 ++++++--------- .../Helpers/TestApplicationBuilderExtensions.cs | 8 ++++++++ 4 files changed, 18 insertions(+), 19 deletions(-) rename src/Platform/Microsoft.Testing.Platform/{TestFramework => CommandLine}/MaxFailedTestsCommandLineOptionsProvider.cs (73%) diff --git a/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/TestApplicationBuilderExtensions.cs b/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/TestApplicationBuilderExtensions.cs index 9dd9b985f3..15109cc58d 100644 --- a/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/TestApplicationBuilderExtensions.cs +++ b/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/TestApplicationBuilderExtensions.cs @@ -8,6 +8,7 @@ using Microsoft.Testing.Extensions.VSTestBridge.Helpers; using Microsoft.Testing.Platform.Builder; using Microsoft.Testing.Platform.Capabilities.TestFramework; +using Microsoft.Testing.Platform.Helpers; using Microsoft.Testing.Platform.Services; namespace Microsoft.VisualStudio.TestTools.UnitTesting; @@ -20,7 +21,9 @@ public static void AddMSTest(this ITestApplicationBuilder testApplicationBuilder testApplicationBuilder.AddRunSettingsService(extension); testApplicationBuilder.AddTestCaseFilterService(extension); testApplicationBuilder.AddTestRunParametersService(extension); - testApplicationBuilder.AddMaxFailedTestsService(); +#pragma warning disable TPEXP // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + testApplicationBuilder.AddMaxFailedTestsService(extension); +#pragma warning restore TPEXP // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. testApplicationBuilder.AddRunSettingsEnvironmentVariableProvider(extension); testApplicationBuilder.RegisterTestFramework( serviceProvider => new TestFrameworkCapabilities( diff --git a/src/Platform/Microsoft.Testing.Extensions.VSTestBridge/Helpers/TestApplicationBuilderExtensions.cs b/src/Platform/Microsoft.Testing.Extensions.VSTestBridge/Helpers/TestApplicationBuilderExtensions.cs index 97aeeb6548..ca611010c2 100644 --- a/src/Platform/Microsoft.Testing.Extensions.VSTestBridge/Helpers/TestApplicationBuilderExtensions.cs +++ b/src/Platform/Microsoft.Testing.Extensions.VSTestBridge/Helpers/TestApplicationBuilderExtensions.cs @@ -48,15 +48,6 @@ public static void AddRunSettingsService(this ITestApplicationBuilder builder, I public static void AddTestRunParametersService(this ITestApplicationBuilder builder, IExtension extension) => builder.CommandLine.AddProvider(() => new TestRunParametersCommandLineOptionsProvider(extension)); - /// - /// Registers the command-line options provider for '--maximum-failed-tests'. - /// - /// The test application builder. - public static void AddMaxFailedTestsService(this ITestApplicationBuilder builder) -#pragma warning disable TPEXP // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. - => builder.CommandLine.AddProvider(() => new MaxFailedTestsCommandLineOptionsProvider()); -#pragma warning restore TPEXP // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. - /// /// Register the environment variable provider. /// diff --git a/src/Platform/Microsoft.Testing.Platform/TestFramework/MaxFailedTestsCommandLineOptionsProvider.cs b/src/Platform/Microsoft.Testing.Platform/CommandLine/MaxFailedTestsCommandLineOptionsProvider.cs similarity index 73% rename from src/Platform/Microsoft.Testing.Platform/TestFramework/MaxFailedTestsCommandLineOptionsProvider.cs rename to src/Platform/Microsoft.Testing.Platform/CommandLine/MaxFailedTestsCommandLineOptionsProvider.cs index 1b198be0ae..0eb90ed5de 100644 --- a/src/Platform/Microsoft.Testing.Platform/TestFramework/MaxFailedTestsCommandLineOptionsProvider.cs +++ b/src/Platform/Microsoft.Testing.Platform/CommandLine/MaxFailedTestsCommandLineOptionsProvider.cs @@ -1,19 +1,16 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using System.Diagnostics.CodeAnalysis; using System.Globalization; -using Microsoft.Testing.Platform.CommandLine; using Microsoft.Testing.Platform.Extensions; using Microsoft.Testing.Platform.Extensions.CommandLine; using Microsoft.Testing.Platform.Helpers; using Microsoft.Testing.Platform.Resources; -namespace Microsoft.Testing.Platform.TestFramework; +namespace Microsoft.Testing.Platform.CommandLine; -[Experimental("TPEXP", UrlFormat = "https://aka.ms/testingplatform/diagnostics#{0}")] -public sealed class MaxFailedTestsCommandLineOptionsProvider : ICommandLineOptionsProvider +internal sealed class MaxFailedTestsCommandLineOptionsProvider(IExtension extension) : ICommandLineOptionsProvider { internal const string MaxFailedTestsOptionKey = "maximum-failed-tests"; @@ -22,13 +19,13 @@ public sealed class MaxFailedTestsCommandLineOptionsProvider : ICommandLineOptio new(MaxFailedTestsOptionKey, PlatformResources.PlatformCommandLineMaxFailedTestsOptionDescription, ArgumentArity.ExactlyOne, false, isBuiltIn: true), ]; - public string Uid => nameof(MaxFailedTestsCommandLineOptionsProvider); + public string Uid => extension.Uid; - public string Version => AppVersion.DefaultSemVer; + public string Version => extension.Version; - public string DisplayName => PlatformResources.PlatformCommandLineProviderDisplayName; /*TODO: New display name?*/ + public string DisplayName => extension.DisplayName; - public string Description => PlatformResources.PlatformCommandLineProviderDescription; /*TODO: New description?*/ + public string Description => extension.Description; public IReadOnlyCollection GetCommandLineOptions() => OptionsCache; diff --git a/src/Platform/Microsoft.Testing.Platform/Helpers/TestApplicationBuilderExtensions.cs b/src/Platform/Microsoft.Testing.Platform/Helpers/TestApplicationBuilderExtensions.cs index 512dc9e6a8..7ff28fc10e 100644 --- a/src/Platform/Microsoft.Testing.Platform/Helpers/TestApplicationBuilderExtensions.cs +++ b/src/Platform/Microsoft.Testing.Platform/Helpers/TestApplicationBuilderExtensions.cs @@ -15,4 +15,12 @@ public static class TestApplicationBuilderExtensions { public static void AddTreeNodeFilterService(this ITestApplicationBuilder testApplicationBuilder, IExtension extension) => testApplicationBuilder.CommandLine.AddProvider(() => new TreeNodeFilterCommandLineOptionsProvider(extension)); + + /// + /// Registers the command-line options provider for '--maximum-failed-tests'. + /// + /// The test application builder. + [Experimental("TPEXP", UrlFormat = "https://aka.ms/testingplatform/diagnostics#{0}")] + public static void AddMaxFailedTestsService(this ITestApplicationBuilder builder, IExtension extension) + => builder.CommandLine.AddProvider(() => new MaxFailedTestsCommandLineOptionsProvider(extension)); } From f57c0a62139f3920ecea2b8a26239b81c7bc3934 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Mon, 9 Dec 2024 12:53:39 +0100 Subject: [PATCH 38/64] TODO: --- .../CommandLine/MaxFailedTestsCommandLineOptionsProvider.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Platform/Microsoft.Testing.Platform/CommandLine/MaxFailedTestsCommandLineOptionsProvider.cs b/src/Platform/Microsoft.Testing.Platform/CommandLine/MaxFailedTestsCommandLineOptionsProvider.cs index 0eb90ed5de..84d1ad47b3 100644 --- a/src/Platform/Microsoft.Testing.Platform/CommandLine/MaxFailedTestsCommandLineOptionsProvider.cs +++ b/src/Platform/Microsoft.Testing.Platform/CommandLine/MaxFailedTestsCommandLineOptionsProvider.cs @@ -16,7 +16,7 @@ internal sealed class MaxFailedTestsCommandLineOptionsProvider(IExtension extens private static readonly IReadOnlyCollection OptionsCache = [ - new(MaxFailedTestsOptionKey, PlatformResources.PlatformCommandLineMaxFailedTestsOptionDescription, ArgumentArity.ExactlyOne, false, isBuiltIn: true), + new(MaxFailedTestsOptionKey, PlatformResources.PlatformCommandLineMaxFailedTestsOptionDescription, ArgumentArity.ExactlyOne, isHidden: false/*TODO: Should we pass isBuiltIn: true??*/), ]; public string Uid => extension.Uid; From 3a4f0caf6c4381a08e926e652e787e2fa0d49c12 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Mon, 9 Dec 2024 12:56:02 +0100 Subject: [PATCH 39/64] Update help tests --- .../MSTest.Acceptance.IntegrationTests/HelpInfoTests.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/HelpInfoTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/HelpInfoTests.cs index 8894ffbc2d..4e167cded7 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/HelpInfoTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/HelpInfoTests.cs @@ -71,6 +71,8 @@ Takes one argument as string in the format [h|m|s] where 'value' is float Extension options: --filter Filters tests using the given expression. For more information, see the Filter option details section. For more information and examples on how to use selective unit test filtering, see https://learn.microsoft.com/dotnet/core/testing/selective-unit-tests. + --maximum-failed-tests + Specifies a maximum number of test failures that, when exceeded, will abort the test run. --no-ansi Disable outputting ANSI escape characters to screen. --no-progress @@ -109,6 +111,10 @@ public async Task Info_WhenMSTestExtensionRegistered_OutputInfoContentOfRegister Arity: 1 Hidden: False Description: Filters tests using the given expression. For more information, see the Filter option details section. For more information and examples on how to use selective unit test filtering, see https://learn.microsoft.com/dotnet/core/testing/selective-unit-tests. + --maximum-failed-tests + Arity: 1 + Hidden: False + Description: Specifies a maximum number of test failures that, when exceeded, will abort the test run. --test-parameter Arity: 1..N Hidden: False From a249f8fc7d2f943b286a9aaad8051930bd3297a1 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Mon, 9 Dec 2024 13:05:40 +0100 Subject: [PATCH 40/64] Fix build --- .../Extensions/AbortForMaxFailedTestsExtension.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs b/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs index 84aa56de6b..a649771f07 100644 --- a/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs +++ b/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs @@ -9,7 +9,6 @@ using Microsoft.Testing.Platform.Messages; using Microsoft.Testing.Platform.Resources; using Microsoft.Testing.Platform.Services; -using Microsoft.Testing.Platform.TestFramework; namespace Microsoft.Testing.Platform.Extensions; From 6f8474381d52cac40c298403c5ecb8d8c6bff67e Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Mon, 9 Dec 2024 13:07:38 +0100 Subject: [PATCH 41/64] Fix build errors --- .../PublicAPI/PublicAPI.Unshipped.txt | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/Platform/Microsoft.Testing.Platform/PublicAPI/PublicAPI.Unshipped.txt b/src/Platform/Microsoft.Testing.Platform/PublicAPI/PublicAPI.Unshipped.txt index 5781934149..1d1c2b5ca1 100644 --- a/src/Platform/Microsoft.Testing.Platform/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/Platform/Microsoft.Testing.Platform/PublicAPI/PublicAPI.Unshipped.txt @@ -11,16 +11,6 @@ Microsoft.Testing.Platform.OutputDevice.WarningMessageOutputDeviceData.WarningMe [TPEXP]Microsoft.Testing.Platform.Extensions.Messages.StandardOutputProperty [TPEXP]Microsoft.Testing.Platform.Extensions.Messages.StandardOutputProperty.StandardOutput.get -> string! [TPEXP]Microsoft.Testing.Platform.Extensions.Messages.StandardOutputProperty.StandardOutput.init -> void -[TPEXP]Microsoft.Testing.Platform.TestFramework.MaxFailedTestsCommandLineOptionsProvider -[TPEXP]Microsoft.Testing.Platform.TestFramework.MaxFailedTestsCommandLineOptionsProvider.Description.get -> string! -[TPEXP]Microsoft.Testing.Platform.TestFramework.MaxFailedTestsCommandLineOptionsProvider.DisplayName.get -> string! -[TPEXP]Microsoft.Testing.Platform.TestFramework.MaxFailedTestsCommandLineOptionsProvider.GetCommandLineOptions() -> System.Collections.Generic.IReadOnlyCollection! -[TPEXP]Microsoft.Testing.Platform.TestFramework.MaxFailedTestsCommandLineOptionsProvider.IsEnabledAsync() -> System.Threading.Tasks.Task! -[TPEXP]Microsoft.Testing.Platform.TestFramework.MaxFailedTestsCommandLineOptionsProvider.MaxFailedTestsCommandLineOptionsProvider() -> void -[TPEXP]Microsoft.Testing.Platform.TestFramework.MaxFailedTestsCommandLineOptionsProvider.Uid.get -> string! -[TPEXP]Microsoft.Testing.Platform.TestFramework.MaxFailedTestsCommandLineOptionsProvider.ValidateCommandLineOptionsAsync(Microsoft.Testing.Platform.CommandLine.ICommandLineOptions! commandLineOptions) -> System.Threading.Tasks.Task! -[TPEXP]Microsoft.Testing.Platform.TestFramework.MaxFailedTestsCommandLineOptionsProvider.ValidateOptionArgumentsAsync(Microsoft.Testing.Platform.Extensions.CommandLine.CommandLineOption! commandOption, string![]! arguments) -> System.Threading.Tasks.Task! -[TPEXP]Microsoft.Testing.Platform.TestFramework.MaxFailedTestsCommandLineOptionsProvider.Version.get -> string! [TPEXP]virtual Microsoft.Testing.Platform.Extensions.Messages.StandardOutputProperty.EqualityContract.get -> System.Type! [TPEXP]override Microsoft.Testing.Platform.Extensions.Messages.StandardOutputProperty.ToString() -> string! [TPEXP]virtual Microsoft.Testing.Platform.Extensions.Messages.StandardOutputProperty.PrintMembers(System.Text.StringBuilder! builder) -> bool From 7b5b39613e4d44d75f15ea7fb359679b64191d12 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Mon, 9 Dec 2024 13:08:59 +0100 Subject: [PATCH 42/64] Fix one more build error --- .../Helpers/TestApplicationBuilderExtensions.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Platform/Microsoft.Testing.Extensions.VSTestBridge/Helpers/TestApplicationBuilderExtensions.cs b/src/Platform/Microsoft.Testing.Extensions.VSTestBridge/Helpers/TestApplicationBuilderExtensions.cs index ca611010c2..9ec3739da9 100644 --- a/src/Platform/Microsoft.Testing.Extensions.VSTestBridge/Helpers/TestApplicationBuilderExtensions.cs +++ b/src/Platform/Microsoft.Testing.Extensions.VSTestBridge/Helpers/TestApplicationBuilderExtensions.cs @@ -8,7 +8,6 @@ using Microsoft.Testing.Platform.Extensions; using Microsoft.Testing.Platform.Helpers; using Microsoft.Testing.Platform.Services; -using Microsoft.Testing.Platform.TestFramework; namespace Microsoft.Testing.Extensions.VSTestBridge.Helpers; From 8ec5e060c99cfa87883fd78e0cdded2133818ee4 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Mon, 9 Dec 2024 13:10:09 +0100 Subject: [PATCH 43/64] Fix PublicAPI.Unshipped.txt --- .../PublicAPI.Unshipped.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Platform/Microsoft.Testing.Extensions.VSTestBridge/PublicAPI.Unshipped.txt b/src/Platform/Microsoft.Testing.Extensions.VSTestBridge/PublicAPI.Unshipped.txt index 421454eed9..c27c99f4e5 100644 --- a/src/Platform/Microsoft.Testing.Extensions.VSTestBridge/PublicAPI.Unshipped.txt +++ b/src/Platform/Microsoft.Testing.Extensions.VSTestBridge/PublicAPI.Unshipped.txt @@ -1,3 +1,2 @@ #nullable enable -static Microsoft.Testing.Extensions.VSTestBridge.Helpers.TestApplicationBuilderExtensions.AddMaxFailedTestsService(this Microsoft.Testing.Platform.Builder.ITestApplicationBuilder! builder) -> void static Microsoft.Testing.Extensions.VSTestBridge.Helpers.TestApplicationBuilderExtensions.AddRunSettingsEnvironmentVariableProvider(this Microsoft.Testing.Platform.Builder.ITestApplicationBuilder! builder, Microsoft.Testing.Platform.Extensions.IExtension! extension) -> void From 8a2379909d244dd095f528717f0b9f623ac93c7d Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Mon, 9 Dec 2024 13:16:07 +0100 Subject: [PATCH 44/64] Rename capability on MSTest side --- .../MSTest.TestAdapter/Execution/TestExecutionManager.cs | 4 ++-- ...lity.cs => MSTestGracefulStopTestExecutionCapability.cs} | 6 +++--- .../TestApplicationBuilderExtensions.cs | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) rename src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/{MSTestStopGracefullyTestExecutionCapability.cs => MSTestGracefulStopTestExecutionCapability.cs} (78%) diff --git a/src/Adapter/MSTest.TestAdapter/Execution/TestExecutionManager.cs b/src/Adapter/MSTest.TestAdapter/Execution/TestExecutionManager.cs index c676d2932c..4e3f356fc5 100644 --- a/src/Adapter/MSTest.TestAdapter/Execution/TestExecutionManager.cs +++ b/src/Adapter/MSTest.TestAdapter/Execution/TestExecutionManager.cs @@ -399,7 +399,7 @@ private void ExecuteTestsInSource(IEnumerable tests, IRunContext? runC ExecuteTestsWithTestRunner(testsToRun, frameworkHandle, source, sourceLevelParameters, testRunner); } - if (MSTestStopGracefullyTestExecutionCapability.Instance.IsStopRequested) + if (MSTestGracefulStopTestExecutionCapability.Instance.IsStopRequested) { testRunner.ForceCleanup(); } @@ -424,7 +424,7 @@ private void ExecuteTestsWithTestRunner( foreach (TestCase currentTest in orderedTests) { _testRunCancellationToken?.ThrowIfCancellationRequested(); - if (MSTestStopGracefullyTestExecutionCapability.Instance.IsStopRequested) + if (MSTestGracefulStopTestExecutionCapability.Instance.IsStopRequested) { break; } diff --git a/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/MSTestStopGracefullyTestExecutionCapability.cs b/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/MSTestGracefulStopTestExecutionCapability.cs similarity index 78% rename from src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/MSTestStopGracefullyTestExecutionCapability.cs rename to src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/MSTestGracefulStopTestExecutionCapability.cs index 8aa631b204..9163689757 100644 --- a/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/MSTestStopGracefullyTestExecutionCapability.cs +++ b/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/MSTestGracefulStopTestExecutionCapability.cs @@ -6,14 +6,14 @@ namespace Microsoft.VisualStudio.TestTools.UnitTesting; #pragma warning disable TPEXP // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. -internal sealed class MSTestStopGracefullyTestExecutionCapability : IGracefulStopTestExecutionCapability +internal sealed class MSTestGracefulStopTestExecutionCapability : IGracefulStopTestExecutionCapability #pragma warning restore TPEXP // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. { - private MSTestStopGracefullyTestExecutionCapability() + private MSTestGracefulStopTestExecutionCapability() { } - public static MSTestStopGracefullyTestExecutionCapability Instance { get; } = new(); + public static MSTestGracefulStopTestExecutionCapability Instance { get; } = new(); // TODO: Respect this properly to ensure cleanups are run. public bool IsStopRequested { get; private set; } diff --git a/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/TestApplicationBuilderExtensions.cs b/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/TestApplicationBuilderExtensions.cs index 15109cc58d..aab6a1b6d6 100644 --- a/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/TestApplicationBuilderExtensions.cs +++ b/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/TestApplicationBuilderExtensions.cs @@ -30,7 +30,7 @@ public static void AddMSTest(this ITestApplicationBuilder testApplicationBuilder new VSTestBridgeExtensionBaseCapabilities(), #pragma warning disable TPEXP // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. new MSTestBannerCapability(serviceProvider.GetRequiredService()), - MSTestStopGracefullyTestExecutionCapability.Instance), + MSTestGracefulStopTestExecutionCapability.Instance), #pragma warning restore TPEXP // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. (capabilities, serviceProvider) => new MSTestBridgedTestFramework(extension, getTestAssemblies, serviceProvider, capabilities)); } From e0dbaf17a1c1252a9fb2a98357bf5e19fb89afc1 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Mon, 9 Dec 2024 13:17:42 +0100 Subject: [PATCH 45/64] Remove outdated TODO --- .../MSTestGracefulStopTestExecutionCapability.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/MSTestGracefulStopTestExecutionCapability.cs b/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/MSTestGracefulStopTestExecutionCapability.cs index 9163689757..a8ef89ff47 100644 --- a/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/MSTestGracefulStopTestExecutionCapability.cs +++ b/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/MSTestGracefulStopTestExecutionCapability.cs @@ -15,7 +15,6 @@ private MSTestGracefulStopTestExecutionCapability() public static MSTestGracefulStopTestExecutionCapability Instance { get; } = new(); - // TODO: Respect this properly to ensure cleanups are run. public bool IsStopRequested { get; private set; } public Task StopTestExecutionAsync(CancellationToken cancellationToken) From 92f3d8d5354cb6cc7ccf6e3724f17e958d6315e1 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Mon, 9 Dec 2024 14:16:00 +0100 Subject: [PATCH 46/64] Rename public method, add test --- .../TestApplicationBuilderExtensions.cs | 2 +- .../AbortForMaxFailedTestsExtension.cs | 2 +- .../TestApplicationBuilderExtensions.cs | 2 +- .../MaxFailedTestsExtensionTests.cs | 176 ++++++++++++++++++ 4 files changed, 179 insertions(+), 3 deletions(-) create mode 100644 test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MaxFailedTestsExtensionTests.cs diff --git a/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/TestApplicationBuilderExtensions.cs b/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/TestApplicationBuilderExtensions.cs index aab6a1b6d6..3cb51f0732 100644 --- a/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/TestApplicationBuilderExtensions.cs +++ b/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/TestApplicationBuilderExtensions.cs @@ -22,7 +22,7 @@ public static void AddMSTest(this ITestApplicationBuilder testApplicationBuilder testApplicationBuilder.AddTestCaseFilterService(extension); testApplicationBuilder.AddTestRunParametersService(extension); #pragma warning disable TPEXP // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. - testApplicationBuilder.AddMaxFailedTestsService(extension); + testApplicationBuilder.AddMaximumFailedTestsService(extension); #pragma warning restore TPEXP // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. testApplicationBuilder.AddRunSettingsEnvironmentVariableProvider(extension); testApplicationBuilder.RegisterTestFramework( diff --git a/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs b/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs index a649771f07..3b0765e082 100644 --- a/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs +++ b/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs @@ -67,7 +67,7 @@ public async Task ConsumeAsync(IDataProducer dataProducer, IData value, Cancella if (TestNodePropertiesCategories.WellKnownTestNodeTestRunOutcomeFailedProperties.Any(t => t == testNodeStateProperty.GetType()) && ++_failCount > _maxFailedTests.Value && // If already triggered, don't do it again. - _policiesService.IsMaxFailedTestsTriggered) + !_policiesService.IsMaxFailedTestsTriggered) { await _capability.StopTestExecutionAsync(_testApplicationCancellationTokenSource.CancellationToken); await _policiesService.ExecuteMaxFailedTestsCallbacksAsync(_maxFailedTests.Value, _testApplicationCancellationTokenSource.CancellationToken); diff --git a/src/Platform/Microsoft.Testing.Platform/Helpers/TestApplicationBuilderExtensions.cs b/src/Platform/Microsoft.Testing.Platform/Helpers/TestApplicationBuilderExtensions.cs index 7ff28fc10e..bae0af6c00 100644 --- a/src/Platform/Microsoft.Testing.Platform/Helpers/TestApplicationBuilderExtensions.cs +++ b/src/Platform/Microsoft.Testing.Platform/Helpers/TestApplicationBuilderExtensions.cs @@ -21,6 +21,6 @@ public static void AddTreeNodeFilterService(this ITestApplicationBuilder testApp /// /// The test application builder. [Experimental("TPEXP", UrlFormat = "https://aka.ms/testingplatform/diagnostics#{0}")] - public static void AddMaxFailedTestsService(this ITestApplicationBuilder builder, IExtension extension) + public static void AddMaximumFailedTestsService(this ITestApplicationBuilder builder, IExtension extension) => builder.CommandLine.AddProvider(() => new MaxFailedTestsCommandLineOptionsProvider(extension)); } diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MaxFailedTestsExtensionTests.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MaxFailedTestsExtensionTests.cs new file mode 100644 index 0000000000..02700e1d2a --- /dev/null +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MaxFailedTestsExtensionTests.cs @@ -0,0 +1,176 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Runtime.InteropServices; + +using Microsoft.Testing.Platform.Acceptance.IntegrationTests.Helpers; +using Microsoft.Testing.Platform.Helpers; + +namespace Microsoft.Testing.Platform.Acceptance.IntegrationTests; + +[TestGroup] +public class MaxFailedTestsExtensionTests : AcceptanceTestBase +{ + private const string AssetName = nameof(MaxFailedTestsExtensionTests); + private readonly TestAssetFixture _testAssetFixture; + + public MaxFailedTestsExtensionTests(ITestExecutionContext testExecutionContext, TestAssetFixture testAssetFixture) + : base(testExecutionContext) => _testAssetFixture = testAssetFixture; + + public async Task TestMaxFailedTestsShouldCallStopTestExecutionAsync() + { + var testHost = TestInfrastructure.TestHost.LocateFrom(_testAssetFixture.TargetAssetPath, AssetName, TargetFrameworks.NetCurrent.Arguments); + TestHostResult testHostResult = await testHost.ExecuteAsync("--maximum-failed-tests 1"); + + testHostResult.AssertExitCodeIs(ExitCodes.TestExecutionStoppedForMaxFailedTests); + + testHostResult.AssertOutputContains("Test session is aborting due to reaching failures ('1') specified by the '--maximum-failed-tests' option."); + testHostResult.AssertOutputContainsSummary(failed: 3, passed: 3, skipped: 0, aborted: true); + } + + [TestFixture(TestFixtureSharingStrategy.PerTestGroup)] + public sealed class TestAssetFixture(AcceptanceFixture acceptanceFixture) : TestAssetFixtureBase(acceptanceFixture.NuGetGlobalPackagesFolder) + { + private const string Sources = """ +#file MaxFailedTestsExtensionTests.csproj + + + $TargetFrameworks$ + Exe + true + enable + preview + + + + + + +#file Program.cs +using System; +using System.Collections.Generic; +using System.Runtime.InteropServices; +using System.Threading.Tasks; +using System.Threading; +using Microsoft.Testing.Platform.Builder; +using Microsoft.Testing.Platform.Capabilities; +using Microsoft.Testing.Platform.Capabilities.TestFramework; +using Microsoft.Testing.Platform.Extensions.Messages; +using Microsoft.Testing.Platform.Extensions.TestFramework; +using Microsoft.Testing.Platform.Helpers; + +internal sealed class Program +{ + public static async Task Main(string[] args) + { + ITestApplicationBuilder builder = await TestApplication.CreateBuilderAsync(args); + var adapter = new DummyAdapter(); + builder.RegisterTestFramework(_ => new Capabilities(), (_, __) => adapter); + +#pragma warning disable TPEXP // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + builder.AddMaximumFailedTestsService(adapter); +#pragma warning restore TPEXP // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + + using ITestApplication app = await builder.BuildAsync(); + return await app.RunAsync(); + } +} + +internal class DummyAdapter : ITestFramework, IDataProducer +{ + public string Uid => nameof(DummyAdapter); + + public string Version => string.Empty; + + public string DisplayName => string.Empty; + + public string Description => string.Empty; + + public Type[] DataTypesProduced => new[] { typeof(TestNodeUpdateMessage) }; + + public Task CloseTestSessionAsync(CloseTestSessionContext context) => Task.FromResult(new CloseTestSessionResult() { IsSuccess = true }); + + public Task CreateTestSessionAsync(CreateTestSessionContext context) => Task.FromResult(new CreateTestSessionResult() { IsSuccess = true }); + + public async Task ExecuteRequestAsync(ExecuteRequestContext context) + { + // First fail. + await context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage(context.Request.Session.SessionUid, + new TestNode() { Uid = "1", DisplayName = "Test1", Properties = new(new FailedTestNodeStateProperty()) })); + + await context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage(context.Request.Session.SessionUid, + new TestNode() { Uid = "2", DisplayName = "Test2", Properties = new(PassedTestNodeStateProperty.CachedInstance) })); + + await context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage(context.Request.Session.SessionUid, + new TestNode() { Uid = "3", DisplayName = "Test3", Properties = new(PassedTestNodeStateProperty.CachedInstance) })); + + await context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage(context.Request.Session.SessionUid, + new TestNode() { Uid = "4", DisplayName = "Test4", Properties = new(PassedTestNodeStateProperty.CachedInstance) })); + + if (GracefulStop.Instance.IsStopRequested) throw new InvalidOperationException("Unexpected stop request!"); + + // Second fail. + await context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage(context.Request.Session.SessionUid, + new TestNode() { Uid = "5", DisplayName = "Test5", Properties = new(new FailedTestNodeStateProperty()) })); + + await GracefulStop.Instance.TCS.Task; + + if (!GracefulStop.Instance.IsStopRequested) throw new InvalidOperationException("Expected stop request!"); + + // Third fail. + await context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage(context.Request.Session.SessionUid, + new TestNode() { Uid = "6", DisplayName = "Test6", Properties = new(new FailedTestNodeStateProperty()) })); + + context.Complete(); + } + + public Task IsEnabledAsync() => Task.FromResult(true); +} + +internal class Capabilities : ITestFrameworkCapabilities +{ + IReadOnlyCollection ICapabilities.Capabilities => [GracefulStop.Instance]; +} + +#pragma warning disable TPEXP // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. +internal sealed class GracefulStop : IGracefulStopTestExecutionCapability +#pragma warning restore TPEXP // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. +{ + private GracefulStop() + { + } + + public static GracefulStop Instance { get; } = new(); + + public TaskCompletionSource TCS { get; } = new(); + + public bool IsStopRequested { get; private set; } + + public Task StopTestExecutionAsync(CancellationToken cancellationToken) + { + IsStopRequested = true; + TCS.SetResult(); + return Task.CompletedTask; + } +} + +"""; + + public string TargetAssetPath => GetAssetPath(AssetName); + + public override IEnumerable<(string ID, string Name, string Code)> GetAssetsToGenerate() + { + // We expect the same semantic for Linux, the test setup is not cross and we're using specific + // Windows API because this gesture is not easy xplat. + if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + yield break; + } + + yield return (AssetName, AssetName, + Sources + .PatchTargetFrameworks(TargetFrameworks.NetCurrent) + .PatchCodeWithReplace("$MicrosoftTestingPlatformVersion$", MicrosoftTestingPlatformVersion)); + } + } +} From f4732db25fc2da34ee519afb1881bfd69853bf11 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Mon, 9 Dec 2024 17:12:38 +0100 Subject: [PATCH 47/64] Progress --- .../Hosts/TestHostBuilder.cs | 6 ++++++ .../OutputDevice/IPlatformOutputDevice.cs | 2 ++ .../OutputDevice/ProxyOutputDevice.cs | 9 +++++++++ .../OutputDevice/TerminalOutputDevice.cs | 19 ++++++++++++++----- .../OutputDevice/TestProcessRole.cs | 11 +++++++++++ .../JsonRpc/ServerModePerCallOutputDevice.cs | 17 ++++++++++++++--- .../Services/StopPoliciesService.cs | 13 ++++++++++++- 7 files changed, 68 insertions(+), 9 deletions(-) create mode 100644 src/Platform/Microsoft.Testing.Platform/OutputDevice/TestProcessRole.cs diff --git a/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs b/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs index ac4f7b6828..bdeb5097d2 100644 --- a/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs +++ b/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs @@ -380,6 +380,8 @@ await LogTestHostCreatedAsync( TestHostOrchestratorConfiguration testHostOrchestratorConfiguration = await TestHostOrchestratorManager.BuildAsync(serviceProvider); if (testHostOrchestratorConfiguration.TestHostOrchestrators.Length > 0 && !commandLineHandler.IsOptionSet(PlatformCommandLineProvider.DiscoverTestsOptionKey)) { + policiesService.ProcessRole = TestProcessRole.TestHostOrchestrator; + await proxyOutputDevice.HandleProcessRoleAsync(TestProcessRole.TestHostOrchestrator); return new TestHostOrchestratorHost(testHostOrchestratorConfiguration, serviceProvider); } @@ -415,6 +417,8 @@ await LogTestHostCreatedAsync( if (testHostControllers.RequireProcessRestart) { testHostControllerInfo.IsCurrentProcessTestHostController = true; + policiesService.ProcessRole = TestProcessRole.TestHostController; + await proxyOutputDevice.HandleProcessRoleAsync(TestProcessRole.TestHostController); TestHostControllersTestHost testHostControllersTestHost = new(testHostControllers, testHostControllersServiceProvider, passiveNode, systemEnvironment, loggerFactory, systemClock); await LogTestHostCreatedAsync( @@ -428,6 +432,8 @@ await LogTestHostCreatedAsync( } // ======= TEST HOST MODE ======== // + policiesService.ProcessRole = TestProcessRole.TestHost; + await proxyOutputDevice.HandleProcessRoleAsync(TestProcessRole.TestHost); // Setup the test host working folder. // Out of the test host controller extension the current working directory is the test host working directory. diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/IPlatformOutputDevice.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/IPlatformOutputDevice.cs index 25d1bf12ae..22705e1968 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/IPlatformOutputDevice.cs +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/IPlatformOutputDevice.cs @@ -15,4 +15,6 @@ internal interface IPlatformOutputDevice : IExtension Task DisplayAfterSessionEndRunAsync(); Task DisplayAsync(IOutputDeviceDataProducer producer, IOutputDeviceData data); + + Task HandleProcessRoleAsync(TestProcessRole processRole); } diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/ProxyOutputDevice.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/ProxyOutputDevice.cs index 6896debd84..b2d0b2ed4c 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/ProxyOutputDevice.cs +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/ProxyOutputDevice.cs @@ -62,4 +62,13 @@ internal async Task InitializeAsync(ServerTestHost serverTestHost) await _serverModeOutputDevice.InitializeAsync(serverTestHost); } } + + internal async Task HandleProcessRoleAsync(TestProcessRole processRole) + { + await OriginalOutputDevice.HandleProcessRoleAsync(processRole); + if (_serverModeOutputDevice is not null) + { + await _serverModeOutputDevice.HandleProcessRoleAsync(processRole); + } + } } diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.cs index a8eef90a12..8ad4b0ea2f 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.cs +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.cs @@ -49,6 +49,7 @@ internal sealed partial class TerminalOutputDevice : IHotReloadPlatformOutputDev private readonly IFileLoggerInformation? _fileLoggerInformation; private readonly ILoggerFactory _loggerFactory; private readonly IClock _clock; + private readonly IStopPoliciesService _policiesService; private readonly string? _longArchitecture; private readonly string? _shortArchitecture; @@ -88,11 +89,7 @@ public TerminalOutputDevice(ITestApplicationCancellationTokenSource testApplicat _fileLoggerInformation = fileLoggerInformation; _loggerFactory = loggerFactory; _clock = clock; - - policiesService.RegisterOnMaxFailedTestsCallback( - async (maxFailedTests, _) => await DisplayAsync( - this, - new TextOutputDeviceData(string.Format(CultureInfo.InvariantCulture, PlatformResources.ReachedMaxFailedTestsMessage, maxFailedTests)))); + _policiesService = policiesService; policiesService.RegisterOnAbortCallback( () => @@ -591,4 +588,16 @@ public async Task ConsumeAsync(IDataProducer dataProducer, IData value, Cancella public void Dispose() => _terminalTestReporter?.Dispose(); + + public Task HandleProcessRoleAsync(TestProcessRole processRole) + { + if (processRole == TestProcessRole.TestHost) + { + _policiesService.RegisterOnMaxFailedTestsCallback( + async (maxFailedTests, _) => await DisplayAsync( + this, new TextOutputDeviceData(string.Format(CultureInfo.InvariantCulture, PlatformResources.ReachedMaxFailedTestsMessage, maxFailedTests)))); + } + + return Task.CompletedTask; + } } diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/TestProcessRole.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/TestProcessRole.cs new file mode 100644 index 0000000000..5c29aae6a0 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/TestProcessRole.cs @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.Testing.Platform; + +internal enum TestProcessRole +{ + TestHost, + TestHostController, + TestHostOrchestrator, +} diff --git a/src/Platform/Microsoft.Testing.Platform/ServerMode/JsonRpc/ServerModePerCallOutputDevice.cs b/src/Platform/Microsoft.Testing.Platform/ServerMode/JsonRpc/ServerModePerCallOutputDevice.cs index d2ff415e6a..9bdf67a495 100644 --- a/src/Platform/Microsoft.Testing.Platform/ServerMode/JsonRpc/ServerModePerCallOutputDevice.cs +++ b/src/Platform/Microsoft.Testing.Platform/ServerMode/JsonRpc/ServerModePerCallOutputDevice.cs @@ -18,6 +18,7 @@ namespace Microsoft.Testing.Platform.ServerMode; internal sealed class ServerModePerCallOutputDevice : IPlatformOutputDevice, IOutputDeviceDataProducer { private readonly FileLoggerProvider? _fileLoggerProvider; + private readonly IStopPoliciesService _policiesService; private readonly ConcurrentBag _messages = new(); private IServerTestHost? _serverTestHost; @@ -27,9 +28,7 @@ internal sealed class ServerModePerCallOutputDevice : IPlatformOutputDevice, IOu public ServerModePerCallOutputDevice(FileLoggerProvider? fileLoggerProvider, IStopPoliciesService policiesService) { _fileLoggerProvider = fileLoggerProvider; - policiesService.RegisterOnMaxFailedTestsCallback( - async (maxFailedTests, _) => await DisplayAsync( - this, new TextOutputDeviceData(string.Format(CultureInfo.InvariantCulture, PlatformResources.ReachedMaxFailedTestsMessage, maxFailedTests)))); + _policiesService = policiesService; } internal async Task InitializeAsync(IServerTestHost serverTestHost) @@ -154,4 +153,16 @@ private static string GetIndentedMessage(string message, int? padding) return builder.ToString(); } + + public Task HandleProcessRoleAsync(TestProcessRole processRole) + { + if (processRole == TestProcessRole.TestHost) + { + _policiesService.RegisterOnMaxFailedTestsCallback( + async (maxFailedTests, _) => await DisplayAsync( + this, new TextOutputDeviceData(string.Format(CultureInfo.InvariantCulture, PlatformResources.ReachedMaxFailedTestsMessage, maxFailedTests)))); + } + + return Task.CompletedTask; + } } diff --git a/src/Platform/Microsoft.Testing.Platform/Services/StopPoliciesService.cs b/src/Platform/Microsoft.Testing.Platform/Services/StopPoliciesService.cs index eaca20bd11..0190bfa509 100644 --- a/src/Platform/Microsoft.Testing.Platform/Services/StopPoliciesService.cs +++ b/src/Platform/Microsoft.Testing.Platform/Services/StopPoliciesService.cs @@ -3,6 +3,8 @@ using System.Collections.Concurrent; +using Microsoft.Testing.Platform.Helpers; + namespace Microsoft.Testing.Platform.Services; internal sealed class StopPoliciesService : IStopPoliciesService @@ -38,6 +40,8 @@ public StopPoliciesService(ITestApplicationCancellationTokenSource testApplicati private BlockingCollection>? _maxFailedTestsCallbacks; private BlockingCollection>? _abortCallbacks; + internal TestProcessRole? ProcessRole { get; set; } + public bool IsMaxFailedTestsTriggered { get; private set; } public bool IsAbortTriggered { get; private set; } @@ -83,7 +87,14 @@ public async Task ExecuteAbortCallbacksAsync() } public void RegisterOnMaxFailedTestsCallback(Func callback) - => RegisterCallback(ref _maxFailedTestsCallbacks, callback); + { + if (ProcessRole != TestProcessRole.TestHost) + { + throw ApplicationStateGuard.Unreachable(); + } + + RegisterCallback(ref _maxFailedTestsCallbacks, callback); + } public void RegisterOnAbortCallback(Func callback) => RegisterCallback(ref _abortCallbacks, callback); From ef5bdb95ddea8ea2be0ff917031c72af9142da44 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Mon, 9 Dec 2024 18:21:42 +0100 Subject: [PATCH 48/64] Doc --- .../OutputDevice/TestProcessRole.cs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/TestProcessRole.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/TestProcessRole.cs index 5c29aae6a0..0955e45246 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/TestProcessRole.cs +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/TestProcessRole.cs @@ -5,7 +5,18 @@ namespace Microsoft.Testing.Platform; internal enum TestProcessRole { + /// + /// Indicates that the currently running process is the test host. + /// TestHost, + + /// + /// Indicates that the currently running process is the test host controller. + /// TestHostController, + + /// + /// Indicates that the currently running process is the test host orchestrator. + /// TestHostOrchestrator, } From 22d846b635a11aefec7d63847811637bf7126d40 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Mon, 9 Dec 2024 19:33:51 +0100 Subject: [PATCH 49/64] Small progress --- .../MaxFailedTestsCommandLineOptionsProvider.cs | 2 +- .../MSTest.Acceptance.IntegrationTests/HelpInfoTests.cs | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Platform/Microsoft.Testing.Platform/CommandLine/MaxFailedTestsCommandLineOptionsProvider.cs b/src/Platform/Microsoft.Testing.Platform/CommandLine/MaxFailedTestsCommandLineOptionsProvider.cs index 84d1ad47b3..e677ef8d03 100644 --- a/src/Platform/Microsoft.Testing.Platform/CommandLine/MaxFailedTestsCommandLineOptionsProvider.cs +++ b/src/Platform/Microsoft.Testing.Platform/CommandLine/MaxFailedTestsCommandLineOptionsProvider.cs @@ -16,7 +16,7 @@ internal sealed class MaxFailedTestsCommandLineOptionsProvider(IExtension extens private static readonly IReadOnlyCollection OptionsCache = [ - new(MaxFailedTestsOptionKey, PlatformResources.PlatformCommandLineMaxFailedTestsOptionDescription, ArgumentArity.ExactlyOne, isHidden: false/*TODO: Should we pass isBuiltIn: true??*/), + new(MaxFailedTestsOptionKey, PlatformResources.PlatformCommandLineMaxFailedTestsOptionDescription, ArgumentArity.ExactlyOne, isHidden: false), ]; public string Uid => extension.Uid; diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/HelpInfoTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/HelpInfoTests.cs index 4e167cded7..e64b0928f5 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/HelpInfoTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/HelpInfoTests.cs @@ -111,14 +111,14 @@ public async Task Info_WhenMSTestExtensionRegistered_OutputInfoContentOfRegister Arity: 1 Hidden: False Description: Filters tests using the given expression. For more information, see the Filter option details section. For more information and examples on how to use selective unit test filtering, see https://learn.microsoft.com/dotnet/core/testing/selective-unit-tests. - --maximum-failed-tests - Arity: 1 - Hidden: False - Description: Specifies a maximum number of test failures that, when exceeded, will abort the test run. --test-parameter Arity: 1..N Hidden: False Description: Specify or override a key-value pair parameter. For more information and examples, see https://learn.microsoft.com/visualstudio/test/configure-unit-tests-by-using-a-dot-runsettings-file#testrunparameters + --maximum-failed-tests + Arity: 1 + Hidden: False + Description: Specifies a maximum number of test failures that, when exceeded, will abort the test run. """; testHostResult.AssertOutputContains(output); From 43d3b21e9352444057ec91e0137c2bdb1511db9b Mon Sep 17 00:00:00 2001 From: Youssef Victor Date: Tue, 10 Dec 2024 05:07:01 +0100 Subject: [PATCH 50/64] Fix tests --- .../MaxFailedTestsExtensionTests.cs | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MaxFailedTestsExtensionTests.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MaxFailedTestsExtensionTests.cs index 02700e1d2a..af141222be 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MaxFailedTestsExtensionTests.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MaxFailedTestsExtensionTests.cs @@ -25,7 +25,7 @@ public async Task TestMaxFailedTestsShouldCallStopTestExecutionAsync() testHostResult.AssertExitCodeIs(ExitCodes.TestExecutionStoppedForMaxFailedTests); testHostResult.AssertOutputContains("Test session is aborting due to reaching failures ('1') specified by the '--maximum-failed-tests' option."); - testHostResult.AssertOutputContainsSummary(failed: 3, passed: 3, skipped: 0, aborted: true); + testHostResult.AssertOutputContainsSummary(failed: 3, passed: 3, skipped: 0); } [TestFixture(TestFixtureSharingStrategy.PerTestGroup)] @@ -160,13 +160,6 @@ public Task StopTestExecutionAsync(CancellationToken cancellationToken) public override IEnumerable<(string ID, string Name, string Code)> GetAssetsToGenerate() { - // We expect the same semantic for Linux, the test setup is not cross and we're using specific - // Windows API because this gesture is not easy xplat. - if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - { - yield break; - } - yield return (AssetName, AssetName, Sources .PatchTargetFrameworks(TargetFrameworks.NetCurrent) From 49422543c804b53e6b38c2818c1ee7a779d6f617 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Tue, 10 Dec 2024 05:14:49 +0100 Subject: [PATCH 51/64] Remove unused using --- .../MaxFailedTestsExtensionTests.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MaxFailedTestsExtensionTests.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MaxFailedTestsExtensionTests.cs index af141222be..932e8049df 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MaxFailedTestsExtensionTests.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MaxFailedTestsExtensionTests.cs @@ -1,8 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using System.Runtime.InteropServices; - using Microsoft.Testing.Platform.Acceptance.IntegrationTests.Helpers; using Microsoft.Testing.Platform.Helpers; From 264eb26b6291da468788eda28fc33457be813ddd Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Tue, 10 Dec 2024 05:30:40 +0100 Subject: [PATCH 52/64] Cleanup --- .../Services/StopPoliciesService.cs | 36 ++----------------- 1 file changed, 3 insertions(+), 33 deletions(-) diff --git a/src/Platform/Microsoft.Testing.Platform/Services/StopPoliciesService.cs b/src/Platform/Microsoft.Testing.Platform/Services/StopPoliciesService.cs index 0190bfa509..206b3a2e58 100644 --- a/src/Platform/Microsoft.Testing.Platform/Services/StopPoliciesService.cs +++ b/src/Platform/Microsoft.Testing.Platform/Services/StopPoliciesService.cs @@ -9,34 +9,12 @@ namespace Microsoft.Testing.Platform.Services; internal sealed class StopPoliciesService : IStopPoliciesService { - private readonly Lock _abortLock = new(); - - public StopPoliciesService(ITestApplicationCancellationTokenSource testApplicationCancellationTokenSource) - { + public StopPoliciesService(ITestApplicationCancellationTokenSource testApplicationCancellationTokenSource) => #pragma warning disable VSTHRD101 // Avoid unsupported async delegates + // Note: If cancellation already requested, Register will still invoke the callback. testApplicationCancellationTokenSource.CancellationToken.Register(async () => await ExecuteAbortCallbacksAsync()); #pragma warning restore VSTHRD101 // Avoid unsupported async delegates - // This happens in mocked tests because the test will do the Cancel on the first CancellationToken property access. - // In theory, cancellation may happen in practice fast enough before StopPoliciesService is created. - if (testApplicationCancellationTokenSource.CancellationToken.IsCancellationRequested) - { - _ = ExecuteAbortCallbacksAsync(); - } - - // NOTE::: Don't move the CancellationToken.Register call to an "else" here. - // If the call is moved to else, then this race may happen: - // 1. Check IsCancellationRequested -> false - // 2. Cancellation happens - // 3. Go to the "else" and do CancellationToken.Register which will never happen because - // cancellation already happened after IsCancellationRequested check and before CancellationToken.Register. - // - // With the current implementation above, we always do the Register first. - // Then, if IsCancellationRequested was false, we are sure the register already happened and we get the callback when cancelled. - // However, if we found IsCancellationRequested to be true, we are not sure if cancellation happened before Register or not. - // So, we may call ExecuteAbortCallbacksAsync twice. This is fine, we handle that with a lock inside ExecuteAbortCallbacksAsync. - } - private BlockingCollection>? _maxFailedTestsCallbacks; private BlockingCollection>? _abortCallbacks; @@ -65,15 +43,7 @@ public async Task ExecuteMaxFailedTestsCallbacksAsync(int maxFailedTests, Cancel public async Task ExecuteAbortCallbacksAsync() { - lock (_abortLock) - { - if (IsAbortTriggered) - { - return; - } - - IsAbortTriggered = true; - } + IsAbortTriggered = true; if (_abortCallbacks is null) { From f2dfe03c020f8573b4792fd7eff2cae0200516da Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Tue, 10 Dec 2024 05:37:53 +0100 Subject: [PATCH 53/64] Add comments --- .../Services/StopPoliciesService.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Platform/Microsoft.Testing.Platform/Services/StopPoliciesService.cs b/src/Platform/Microsoft.Testing.Platform/Services/StopPoliciesService.cs index 206b3a2e58..e639e30f9b 100644 --- a/src/Platform/Microsoft.Testing.Platform/Services/StopPoliciesService.cs +++ b/src/Platform/Microsoft.Testing.Platform/Services/StopPoliciesService.cs @@ -37,6 +37,8 @@ public async Task ExecuteMaxFailedTestsCallbacksAsync(int maxFailedTests, Cancel foreach (Func callback in _maxFailedTestsCallbacks) { + // For now, we are fine if the callback crashed us. It shouldn't happen for our + // current usage anyway and the APIs around this are all internal for now. await callback.Invoke(maxFailedTests, cancellationToken); } } @@ -52,6 +54,8 @@ public async Task ExecuteAbortCallbacksAsync() foreach (Func callback in _abortCallbacks) { + // For now, we are fine if the callback crashed us. It shouldn't happen for our + // current usage anyway and the APIs around this are all internal for now. await callback.Invoke(); } } From e7318df65f0475f17a742fddf36d172b782e8779 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Tue, 10 Dec 2024 11:10:59 +0100 Subject: [PATCH 54/64] Add test for MSTest --- .../MaxFailedTestsExtensionTests.cs | 149 ++++++++++++++++++ 1 file changed, 149 insertions(+) create mode 100644 test/IntegrationTests/MSTest.Acceptance.IntegrationTests/MaxFailedTestsExtensionTests.cs diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/MaxFailedTestsExtensionTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/MaxFailedTestsExtensionTests.cs new file mode 100644 index 0000000000..b89e09bad4 --- /dev/null +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/MaxFailedTestsExtensionTests.cs @@ -0,0 +1,149 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Globalization; +using System.Runtime.InteropServices; +using System.Text.RegularExpressions; + +using Microsoft.Testing.Platform.Acceptance.IntegrationTests; +using Microsoft.Testing.Platform.Acceptance.IntegrationTests.Helpers; +using Microsoft.Testing.Platform.Helpers; + +using Polly.Caching; + +namespace MSTest.Acceptance.IntegrationTests; + +[TestGroup] +public sealed class MaxFailedTestsExtensionTests : AcceptanceTestBase +{ + private const string AssetName = nameof(MaxFailedTestsExtensionTests); + private readonly TestAssetFixture _testAssetFixture; + + public MaxFailedTestsExtensionTests(ITestExecutionContext testExecutionContext, TestAssetFixture testAssetFixture) + : base(testExecutionContext) => _testAssetFixture = testAssetFixture; + + [ArgumentsProvider(nameof(TargetFrameworks.All), typeof(TargetFrameworks))] + public async Task SimpleMaxFailedTestsScenario(string tfm) + { + var testHost = TestHost.LocateFrom(_testAssetFixture.TargetAssetPath, AssetName, tfm); + + TestHostResult testHostResult = await testHost.ExecuteAsync("--maximum-failed-tests 2"); + testHostResult.AssertExitCodeIs(ExitCodes.TestExecutionStoppedForMaxFailedTests); + + int total = int.Parse(Regex.Match(testHostResult.StandardOutput, @"total: (\d+)").Groups[1].Value, CultureInfo.InvariantCulture); + + // We can't know the number of tests that will be executed exactly due to the async + // nature of publish/consume on the platform side. But we expect the cancellation to + // happen "fast" enough that we don't execute all tests. + Assert.IsTrue(total < 12); + Assert.IsTrue(total >= 5); + + testHostResult = await testHost.ExecuteAsync(); + testHostResult.AssertExitCodeIs(ExitCodes.AtLeastOneTestFailed); + + total = int.Parse(Regex.Match(testHostResult.StandardOutput, @"total: (\d+)").Groups[1].Value, CultureInfo.InvariantCulture); + Assert.AreEqual(12, total); + } + + [TestFixture(TestFixtureSharingStrategy.PerTestGroup)] + public sealed class TestAssetFixture(AcceptanceFixture acceptanceFixture) : TestAssetFixtureBase(acceptanceFixture.NuGetGlobalPackagesFolder) + { + private const string Sources = """ +#file MaxFailedTestsExtensionTests.csproj + + + $TargetFrameworks$ + true + Exe + enable + preview + + + + + + + +#file UnitTest1.cs +using Microsoft.VisualStudio.TestTools.UnitTesting; + +[TestClass] +public class UnitTest1 +{ + [TestMethod] + public void Test1() + { + Assert.Fail(); + } + + [TestMethod] + public void Test2() + { + } + + [TestMethod] + public void Test3() + { + } + + [TestMethod] + public void Test4() + { + Assert.Fail(); + } + + [TestMethod] + public void Test5() + { + Assert.Fail(); + } + + [TestMethod] + public void Test6() + { + } + + [TestMethod] + public void Test7() + { + } + + [TestMethod] + public void Test8() + { + } + + [TestMethod] + public void Test9() + { + } + + [TestMethod] + public void Test10() + { + } + + [TestMethod] + public void Test11() + { + } + + [TestMethod] + public void Test12() + { + } +} +"""; + + public string TargetAssetPath => GetAssetPath(AssetName); + + public override IEnumerable<(string ID, string Name, string Code)> GetAssetsToGenerate() + { + yield return (AssetName, AssetName, + Sources + .PatchTargetFrameworks(TargetFrameworks.All) + .PatchCodeWithReplace("$MicrosoftTestingPlatformVersion$", MicrosoftTestingPlatformVersion) + .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion)); + } + } +} From f134a13a9638f9bf357d4dc377cbcee4bda3bb78 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Tue, 10 Dec 2024 15:13:56 +0100 Subject: [PATCH 55/64] Address feedback --- ...axFailedTestsCommandLineOptionsProvider.cs | 8 ++--- .../AbortForMaxFailedTestsExtension.cs | 4 +-- .../OutputDevice/TerminalOutputDevice.cs | 24 ++++++------- .../Services/IStopPoliciesService.cs | 4 +-- .../Services/StopPoliciesService.cs | 34 +++++++++++++++---- 5 files changed, 45 insertions(+), 29 deletions(-) diff --git a/src/Platform/Microsoft.Testing.Platform/CommandLine/MaxFailedTestsCommandLineOptionsProvider.cs b/src/Platform/Microsoft.Testing.Platform/CommandLine/MaxFailedTestsCommandLineOptionsProvider.cs index e677ef8d03..e81cc907b7 100644 --- a/src/Platform/Microsoft.Testing.Platform/CommandLine/MaxFailedTestsCommandLineOptionsProvider.cs +++ b/src/Platform/Microsoft.Testing.Platform/CommandLine/MaxFailedTestsCommandLineOptionsProvider.cs @@ -40,10 +40,10 @@ public Task ValidateOptionArgumentsAsync(CommandLineOption com if (commandOption.Name == MaxFailedTestsOptionKey) { string arg = arguments[0]; - // We consider --maximum-failed-tests 0 as valid. - // The idea is that we stop the execution when we *exceed* the max failed tests, not when *reach*. - // So zero means, stop execution on the first failure. - return int.TryParse(arg, out int maxFailedTestsResult) && maxFailedTestsResult >= 0 + // We consider --maximum-failed-tests 0 as invalid. + // The idea is that we stop the execution when we *reach* the max failed tests, not when *exceed*. + // So the value 1 means, stop execution on the first failure. + return int.TryParse(arg, out int maxFailedTestsResult) && maxFailedTestsResult > 0 ? ValidationResult.ValidTask : ValidationResult.InvalidTask(string.Format(CultureInfo.InvariantCulture, PlatformResources.MaxFailedTestsMustBePositive, arg)); } diff --git a/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs b/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs index 3b0765e082..78e4999b3c 100644 --- a/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs +++ b/src/Platform/Microsoft.Testing.Platform/Extensions/AbortForMaxFailedTestsExtension.cs @@ -28,7 +28,7 @@ public AbortForMaxFailedTestsExtension( { if (commandLineOptions.TryGetOptionArgumentList(MaxFailedTestsCommandLineOptionsProvider.MaxFailedTestsOptionKey, out string[]? args) && int.TryParse(args[0], out int maxFailedTests) && - maxFailedTests >= 0) + maxFailedTests > 0) { _maxFailedTests = maxFailedTests; } @@ -65,7 +65,7 @@ public async Task ConsumeAsync(IDataProducer dataProducer, IData value, Cancella TestNodeStateProperty testNodeStateProperty = node.TestNode.Properties.Single(); if (TestNodePropertiesCategories.WellKnownTestNodeTestRunOutcomeFailedProperties.Any(t => t == testNodeStateProperty.GetType()) && - ++_failCount > _maxFailedTests.Value && + ++_failCount >= _maxFailedTests.Value && // If already triggered, don't do it again. !_policiesService.IsMaxFailedTestsTriggered) { diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.cs index 8ad4b0ea2f..b50f9aad31 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.cs +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.cs @@ -91,13 +91,6 @@ public TerminalOutputDevice(ITestApplicationCancellationTokenSource testApplicat _clock = clock; _policiesService = policiesService; - policiesService.RegisterOnAbortCallback( - () => - { - _terminalTestReporter?.StartCancelling(); - return Task.CompletedTask; - }); - if (_runtimeFeature.IsDynamicCodeSupported) { #if !NETCOREAPP @@ -120,8 +113,15 @@ public TerminalOutputDevice(ITestApplicationCancellationTokenSource testApplicat } } - public Task InitializeAsync() + public async Task InitializeAsync() { + await _policiesService.RegisterOnAbortCallbackAsync( + () => + { + _terminalTestReporter?.StartCancelling(); + return Task.CompletedTask; + }); + if (_fileLoggerInformation is not null) { _logger = _loggerFactory.CreateLogger(GetType().ToString()); @@ -172,8 +172,6 @@ public Task InitializeAsync() UseAnsi = !noAnsi, ShowProgress = shouldShowProgress, }); - - return Task.CompletedTask; } private string GetShortArchitecture(string runtimeIdentifier) @@ -589,15 +587,13 @@ public async Task ConsumeAsync(IDataProducer dataProducer, IData value, Cancella public void Dispose() => _terminalTestReporter?.Dispose(); - public Task HandleProcessRoleAsync(TestProcessRole processRole) + public async Task HandleProcessRoleAsync(TestProcessRole processRole) { if (processRole == TestProcessRole.TestHost) { - _policiesService.RegisterOnMaxFailedTestsCallback( + await _policiesService.RegisterOnMaxFailedTestsCallbackAsync( async (maxFailedTests, _) => await DisplayAsync( this, new TextOutputDeviceData(string.Format(CultureInfo.InvariantCulture, PlatformResources.ReachedMaxFailedTestsMessage, maxFailedTests)))); } - - return Task.CompletedTask; } } diff --git a/src/Platform/Microsoft.Testing.Platform/Services/IStopPoliciesService.cs b/src/Platform/Microsoft.Testing.Platform/Services/IStopPoliciesService.cs index d2b49d99d6..d82def8818 100644 --- a/src/Platform/Microsoft.Testing.Platform/Services/IStopPoliciesService.cs +++ b/src/Platform/Microsoft.Testing.Platform/Services/IStopPoliciesService.cs @@ -9,9 +9,9 @@ internal interface IStopPoliciesService bool IsAbortTriggered { get; } - void RegisterOnMaxFailedTestsCallback(Func callback); + Task RegisterOnMaxFailedTestsCallbackAsync(Func callback); - void RegisterOnAbortCallback(Func callback); + Task RegisterOnAbortCallbackAsync(Func callback); Task ExecuteMaxFailedTestsCallbacksAsync(int maxFailedTests, CancellationToken cancellationToken); diff --git a/src/Platform/Microsoft.Testing.Platform/Services/StopPoliciesService.cs b/src/Platform/Microsoft.Testing.Platform/Services/StopPoliciesService.cs index e639e30f9b..fdd0b5f032 100644 --- a/src/Platform/Microsoft.Testing.Platform/Services/StopPoliciesService.cs +++ b/src/Platform/Microsoft.Testing.Platform/Services/StopPoliciesService.cs @@ -9,14 +9,21 @@ namespace Microsoft.Testing.Platform.Services; internal sealed class StopPoliciesService : IStopPoliciesService { - public StopPoliciesService(ITestApplicationCancellationTokenSource testApplicationCancellationTokenSource) => + private readonly ITestApplicationCancellationTokenSource _testApplicationCancellationTokenSource; + + private BlockingCollection>? _maxFailedTestsCallbacks; + private BlockingCollection>? _abortCallbacks; + private int _lastMaxFailedTests; + + public StopPoliciesService(ITestApplicationCancellationTokenSource testApplicationCancellationTokenSource) + { + _testApplicationCancellationTokenSource = testApplicationCancellationTokenSource; + #pragma warning disable VSTHRD101 // Avoid unsupported async delegates // Note: If cancellation already requested, Register will still invoke the callback. testApplicationCancellationTokenSource.CancellationToken.Register(async () => await ExecuteAbortCallbacksAsync()); #pragma warning restore VSTHRD101 // Avoid unsupported async delegates - - private BlockingCollection>? _maxFailedTestsCallbacks; - private BlockingCollection>? _abortCallbacks; + } internal TestProcessRole? ProcessRole { get; set; } @@ -29,6 +36,7 @@ private static void RegisterCallback(ref BlockingCollection? callbacks, T public async Task ExecuteMaxFailedTestsCallbacksAsync(int maxFailedTests, CancellationToken cancellationToken) { + _lastMaxFailedTests = maxFailedTests; IsMaxFailedTestsTriggered = true; if (_maxFailedTestsCallbacks is null) { @@ -60,16 +68,28 @@ public async Task ExecuteAbortCallbacksAsync() } } - public void RegisterOnMaxFailedTestsCallback(Func callback) + public async Task RegisterOnMaxFailedTestsCallbackAsync(Func callback) { if (ProcessRole != TestProcessRole.TestHost) { throw ApplicationStateGuard.Unreachable(); } + if (IsMaxFailedTestsTriggered) + { + await callback(_lastMaxFailedTests, _testApplicationCancellationTokenSource.CancellationToken); + } + RegisterCallback(ref _maxFailedTestsCallbacks, callback); } - public void RegisterOnAbortCallback(Func callback) - => RegisterCallback(ref _abortCallbacks, callback); + public async Task RegisterOnAbortCallbackAsync(Func callback) + { + if (IsAbortTriggered) + { + await callback(); + } + + RegisterCallback(ref _abortCallbacks, callback); + } } From 3d35495881aaa660282d0977c432430205d90629 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Tue, 10 Dec 2024 15:18:40 +0100 Subject: [PATCH 56/64] Fix build error --- .../ServerMode/JsonRpc/ServerModePerCallOutputDevice.cs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/Platform/Microsoft.Testing.Platform/ServerMode/JsonRpc/ServerModePerCallOutputDevice.cs b/src/Platform/Microsoft.Testing.Platform/ServerMode/JsonRpc/ServerModePerCallOutputDevice.cs index 9bdf67a495..8c831b8f7f 100644 --- a/src/Platform/Microsoft.Testing.Platform/ServerMode/JsonRpc/ServerModePerCallOutputDevice.cs +++ b/src/Platform/Microsoft.Testing.Platform/ServerMode/JsonRpc/ServerModePerCallOutputDevice.cs @@ -154,15 +154,13 @@ private static string GetIndentedMessage(string message, int? padding) return builder.ToString(); } - public Task HandleProcessRoleAsync(TestProcessRole processRole) + public async Task HandleProcessRoleAsync(TestProcessRole processRole) { if (processRole == TestProcessRole.TestHost) { - _policiesService.RegisterOnMaxFailedTestsCallback( + await _policiesService.RegisterOnMaxFailedTestsCallbackAsync( async (maxFailedTests, _) => await DisplayAsync( this, new TextOutputDeviceData(string.Format(CultureInfo.InvariantCulture, PlatformResources.ReachedMaxFailedTestsMessage, maxFailedTests)))); } - - return Task.CompletedTask; } } From 529c5216a0af8c667af233b0a9fcc7d50433fad2 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Tue, 10 Dec 2024 16:12:26 +0100 Subject: [PATCH 57/64] Progress --- .../MSTest.TestAdapter/Execution/TestClassInfo.cs | 9 ++++++--- .../MaxFailedTestsExtensionTests.cs | 3 --- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Adapter/MSTest.TestAdapter/Execution/TestClassInfo.cs b/src/Adapter/MSTest.TestAdapter/Execution/TestClassInfo.cs index 07423c3608..c3c8db9d38 100644 --- a/src/Adapter/MSTest.TestAdapter/Execution/TestClassInfo.cs +++ b/src/Adapter/MSTest.TestAdapter/Execution/TestClassInfo.cs @@ -252,6 +252,7 @@ public void RunClassInitialize(TestContext testContext) // If no class initialize and no base class initialize, return if (ClassInitializeMethod is null && BaseClassInitMethods.Count == 0) { + IsClassInitializeExecuted = true; return; } @@ -547,7 +548,7 @@ UnitTestResult DoRun() /// internal void ExecuteClassCleanup() { - if ((ClassCleanupMethod is null && BaseClassCleanupMethods.Count == 0) + if ((ClassCleanupMethod is null && BaseClassCleanupMethods.Count == 0) || IsClassCleanupExecuted) { return; @@ -558,8 +559,10 @@ internal void ExecuteClassCleanup() lock (_testClassExecuteSyncObject) { if (IsClassCleanupExecuted - // If there is a ClassInitialize method and it has not been executed, then we should not execute ClassCleanup - || (!IsClassInitializeExecuted && ClassInitializeMethod is not null)) + // If ClassInitialize method has not been executed, then we should not execute ClassCleanup + // Note that if there is no ClassInitialze method at all, we will still set + // IsClassInitializeExecuted to true in RunClassInitialize + || !IsClassInitializeExecuted) { return; } diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/MaxFailedTestsExtensionTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/MaxFailedTestsExtensionTests.cs index b89e09bad4..77f673c343 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/MaxFailedTestsExtensionTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/MaxFailedTestsExtensionTests.cs @@ -2,15 +2,12 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System.Globalization; -using System.Runtime.InteropServices; using System.Text.RegularExpressions; using Microsoft.Testing.Platform.Acceptance.IntegrationTests; using Microsoft.Testing.Platform.Acceptance.IntegrationTests.Helpers; using Microsoft.Testing.Platform.Helpers; -using Polly.Caching; - namespace MSTest.Acceptance.IntegrationTests; [TestGroup] From 7e56af8b8f75bb159280de867cb6c64780fb8e90 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Tue, 10 Dec 2024 16:13:58 +0100 Subject: [PATCH 58/64] Fix test --- .../MaxFailedTestsExtensionTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MaxFailedTestsExtensionTests.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MaxFailedTestsExtensionTests.cs index 932e8049df..1f9883dbec 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MaxFailedTestsExtensionTests.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MaxFailedTestsExtensionTests.cs @@ -18,11 +18,11 @@ public MaxFailedTestsExtensionTests(ITestExecutionContext testExecutionContext, public async Task TestMaxFailedTestsShouldCallStopTestExecutionAsync() { var testHost = TestInfrastructure.TestHost.LocateFrom(_testAssetFixture.TargetAssetPath, AssetName, TargetFrameworks.NetCurrent.Arguments); - TestHostResult testHostResult = await testHost.ExecuteAsync("--maximum-failed-tests 1"); + TestHostResult testHostResult = await testHost.ExecuteAsync("--maximum-failed-tests 2"); testHostResult.AssertExitCodeIs(ExitCodes.TestExecutionStoppedForMaxFailedTests); - testHostResult.AssertOutputContains("Test session is aborting due to reaching failures ('1') specified by the '--maximum-failed-tests' option."); + testHostResult.AssertOutputContains("Test session is aborting due to reaching failures ('2') specified by the '--maximum-failed-tests' option."); testHostResult.AssertOutputContainsSummary(failed: 3, passed: 3, skipped: 0); } From 2c1b69af6c67023f3c55c4070a796b680ad23482 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Tue, 10 Dec 2024 16:14:36 +0100 Subject: [PATCH 59/64] Fix test --- .../MaxFailedTestsExtensionTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/MaxFailedTestsExtensionTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/MaxFailedTestsExtensionTests.cs index 77f673c343..bd4d4a57d3 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/MaxFailedTestsExtensionTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/MaxFailedTestsExtensionTests.cs @@ -24,7 +24,7 @@ public async Task SimpleMaxFailedTestsScenario(string tfm) { var testHost = TestHost.LocateFrom(_testAssetFixture.TargetAssetPath, AssetName, tfm); - TestHostResult testHostResult = await testHost.ExecuteAsync("--maximum-failed-tests 2"); + TestHostResult testHostResult = await testHost.ExecuteAsync("--maximum-failed-tests 3"); testHostResult.AssertExitCodeIs(ExitCodes.TestExecutionStoppedForMaxFailedTests); int total = int.Parse(Regex.Match(testHostResult.StandardOutput, @"total: (\d+)").Groups[1].Value, CultureInfo.InvariantCulture); From 612a94c174a12a0135abe5a006510a021e45b622 Mon Sep 17 00:00:00 2001 From: Youssef Victor Date: Tue, 10 Dec 2024 16:33:26 +0100 Subject: [PATCH 60/64] Fix formatting --- src/Adapter/MSTest.TestAdapter/Execution/TestClassInfo.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Adapter/MSTest.TestAdapter/Execution/TestClassInfo.cs b/src/Adapter/MSTest.TestAdapter/Execution/TestClassInfo.cs index c3c8db9d38..fe29c5998e 100644 --- a/src/Adapter/MSTest.TestAdapter/Execution/TestClassInfo.cs +++ b/src/Adapter/MSTest.TestAdapter/Execution/TestClassInfo.cs @@ -548,7 +548,7 @@ UnitTestResult DoRun() /// internal void ExecuteClassCleanup() { - if ((ClassCleanupMethod is null && BaseClassCleanupMethods.Count == 0) + if ((ClassCleanupMethod is null && BaseClassCleanupMethods.Count == 0) || IsClassCleanupExecuted) { return; From 4f3e7d06612edd12ed45b1ad72ff0639910d28a5 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Tue, 10 Dec 2024 19:43:16 +0100 Subject: [PATCH 61/64] Adjust unit tests per product change --- .../Execution/TestClassInfoTests.cs | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/test/UnitTests/MSTestAdapter.UnitTests/Execution/TestClassInfoTests.cs b/test/UnitTests/MSTestAdapter.UnitTests/Execution/TestClassInfoTests.cs index c4840468df..9fc878852d 100644 --- a/test/UnitTests/MSTestAdapter.UnitTests/Execution/TestClassInfoTests.cs +++ b/test/UnitTests/MSTestAdapter.UnitTests/Execution/TestClassInfoTests.cs @@ -419,6 +419,7 @@ public void RunClassCleanupShouldInvokeIfClassCleanupMethod() _testClassInfo.ClassCleanupMethod = typeof(DummyTestClass).GetMethod(nameof(DummyTestClass.ClassCleanupMethod)); // Act + _testClassInfo.RunClassInitialize(null); _testClassInfo.ExecuteClassCleanup(); // Assert @@ -446,6 +447,7 @@ public void RunClassCleanupShouldReturnAssertFailureExceptionDetails() _testClassInfo.ClassCleanupMethod = typeof(DummyTestClass).GetMethod(nameof(DummyTestClass.ClassCleanupMethod)); // Act + _testClassInfo.RunClassInitialize(null); Exception classCleanupException = VerifyThrows(_testClassInfo.ExecuteClassCleanup); // Assert @@ -461,6 +463,7 @@ public void RunClassCleanupShouldReturnAssertInconclusiveExceptionDetails() _testClassInfo.ClassCleanupMethod = typeof(DummyTestClass).GetMethod(nameof(DummyTestClass.ClassCleanupMethod)); // Act + _testClassInfo.RunClassInitialize(null); Exception classCleanupException = VerifyThrows(_testClassInfo.ExecuteClassCleanup); // Assert @@ -476,6 +479,7 @@ public void RunClassCleanupShouldReturnExceptionDetailsOfNonAssertExceptions() _testClassInfo.ClassCleanupMethod = typeof(DummyTestClass).GetMethod(nameof(DummyTestClass.ClassCleanupMethod)); // Act + _testClassInfo.RunClassInitialize(null); Exception classCleanupException = VerifyThrows(_testClassInfo.ExecuteClassCleanup); // Assert @@ -493,6 +497,7 @@ public void RunBaseClassCleanupWithNoDerivedClassCleanupShouldReturnExceptionDet _testClassInfo.BaseClassCleanupMethods.Add(baseClassCleanupMethod); // Act + _testClassInfo.RunClassInitialize(null); Exception classCleanupException = VerifyThrows(_testClassInfo.ExecuteClassCleanup); // Assert @@ -515,6 +520,22 @@ public void RunBaseClassCleanupEvenIfThereIsNoDerivedClassCleanup() // Assert Verify(_testClassInfo.HasExecutableCleanupMethod); + Verify(classCleanupCallCount == 0, "DummyBaseTestClass.CleanupClassMethod call count"); + + // Act 2 + _testClassInfo.RunClassInitialize(null); + _testClassInfo.ExecuteClassCleanup(); + + // Assert 2 + Verify(_testClassInfo.HasExecutableCleanupMethod); + Verify(_testClassInfo.IsClassInitializeExecuted); + Verify(classCleanupCallCount == 1, "DummyBaseTestClass.CleanupClassMethod call count"); + + // Act 3 + _testClassInfo.ExecuteClassCleanup(); + + // Assert 3 + Verify(_testClassInfo.HasExecutableCleanupMethod); Verify(classCleanupCallCount == 1, "DummyBaseTestClass.CleanupClassMethod call count"); } @@ -526,6 +547,7 @@ public void RunClassCleanupShouldThrowTheInnerMostExceptionWhenThereAreMultipleN DummyTestClass.ClassCleanupMethodBody = FailingStaticHelper.DoWork; _testClassInfo.ClassCleanupMethod = typeof(DummyTestClass).GetMethod("ClassCleanupMethod"); + _testClassInfo.RunClassInitialize(null); Exception classCleanupException = VerifyThrows(_testClassInfo.ExecuteClassCleanup); Verify(classCleanupException.Message.StartsWith("Class Cleanup method DummyTestClass.ClassCleanupMethod failed. Error Message: System.InvalidOperationException: I fail..", StringComparison.Ordinal)); From 17fb777d1da3a1bb239fb1f79e064f1de026f481 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Tue, 10 Dec 2024 20:10:40 +0100 Subject: [PATCH 62/64] Try to stabilize test --- .../MaxFailedTestsExtensionTests.cs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/MaxFailedTestsExtensionTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/MaxFailedTestsExtensionTests.cs index bd4d4a57d3..7da507b1ce 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/MaxFailedTestsExtensionTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/MaxFailedTestsExtensionTests.cs @@ -62,6 +62,7 @@ public sealed class TestAssetFixture(AcceptanceFixture acceptanceFixture) : Test #file UnitTest1.cs +using System.Threading.Tasks; using Microsoft.VisualStudio.TestTools.UnitTesting; [TestClass] @@ -98,36 +99,43 @@ public void Test5() [TestMethod] public void Test6() { + await Task.Delay(10); } [TestMethod] public void Test7() { + await Task.Delay(10); } [TestMethod] public void Test8() { + await Task.Delay(10); } [TestMethod] public void Test9() { + await Task.Delay(10); } [TestMethod] public void Test10() { + await Task.Delay(10); } [TestMethod] public void Test11() { + await Task.Delay(10); } [TestMethod] - public void Test12() + public async Task Test12() { + await Task.Delay(10); } } """; From ffad719a6bc15010bba25350b3e780f4fe9714fa Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Tue, 10 Dec 2024 21:03:07 +0100 Subject: [PATCH 63/64] Small fix --- src/Adapter/MSTest.TestAdapter/Execution/TestClassInfo.cs | 1 + src/Adapter/MSTest.TestAdapter/Execution/UnitTestRunner.cs | 1 + 2 files changed, 2 insertions(+) diff --git a/src/Adapter/MSTest.TestAdapter/Execution/TestClassInfo.cs b/src/Adapter/MSTest.TestAdapter/Execution/TestClassInfo.cs index 6c0500ce26..2e23564482 100644 --- a/src/Adapter/MSTest.TestAdapter/Execution/TestClassInfo.cs +++ b/src/Adapter/MSTest.TestAdapter/Execution/TestClassInfo.cs @@ -358,6 +358,7 @@ internal UnitTestResult GetResultOrRunClassInitialize(ITestContext testContext, // a thread for nothing. if (ClassInitializeMethod is null && BaseClassInitMethods.Count == 0) { + IsClassInitializeExecuted = true; return new() { Outcome = ObjectModelUnitTestOutcome.Passed }; } diff --git a/src/Adapter/MSTest.TestAdapter/Execution/UnitTestRunner.cs b/src/Adapter/MSTest.TestAdapter/Execution/UnitTestRunner.cs index e836ab4af4..6d6e854add 100644 --- a/src/Adapter/MSTest.TestAdapter/Execution/UnitTestRunner.cs +++ b/src/Adapter/MSTest.TestAdapter/Execution/UnitTestRunner.cs @@ -169,6 +169,7 @@ internal UnitTestResult[] RunSingleTest(TestMethod testMethod, IDictionary Date: Wed, 11 Dec 2024 05:03:26 +0100 Subject: [PATCH 64/64] Fix test --- .../MaxFailedTestsExtensionTests.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/MaxFailedTestsExtensionTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/MaxFailedTestsExtensionTests.cs index 7da507b1ce..9c5f2fe087 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/MaxFailedTestsExtensionTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/MaxFailedTestsExtensionTests.cs @@ -97,37 +97,37 @@ public void Test5() } [TestMethod] - public void Test6() + public async Task Test6() { await Task.Delay(10); } [TestMethod] - public void Test7() + public async Task Test7() { await Task.Delay(10); } [TestMethod] - public void Test8() + public async Task Test8() { await Task.Delay(10); } [TestMethod] - public void Test9() + public async Task Test9() { await Task.Delay(10); } [TestMethod] - public void Test10() + public async Task Test10() { await Task.Delay(10); } [TestMethod] - public void Test11() + public async Task Test11() { await Task.Delay(10); }