Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Plug and test EnableMSTestRunner #1937

Merged
merged 8 commits into from
Dec 14, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions src/Adapter/Build/Common/MSTest.TestAdapter.props
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
<Project>

<PropertyGroup>
<EnableMSTestV2CopyResources Condition=" '$(EnableMSTestV2CopyResources)' == '' ">true</EnableMSTestV2CopyResources>
<EnableMSTestRunner Condition=" '$(EnableMSTestRunner)' == '' ">false</EnableMSTestRunner>
<IsTestingPlatformApplication>$(EnableMSTestRunner)</IsTestingPlatformApplication>
</PropertyGroup>

<ItemGroup>
Expand All @@ -11,7 +12,7 @@
DO NOT CHANGE THE GUID, IT'S A WELL KNOWN EXTENSION POINT AND THIS EXTENSION NEEDS TO BE REGISTERED AT THE END
WE HAVE CODE INSIDE THE TASK 'TestingPlatformEntryPoint' TO ENSURE THE ORDER OF THE REGISTRATION BASED ON THIS GUID
-->
<TestingPlatformBuilderHook Include="031F8871-2660-4208-8F6B-FC142B40ABFF" Condition=" $(GenerateTestingPlatformEntryPoint) == 'True' " >
<TestingPlatformBuilderHook Include="031F8871-2660-4208-8F6B-FC142B40ABFF" Condition=" $(GenerateTestingPlatformEntryPoint) == 'true' " >
<DisplayName>MSTest</DisplayName>
<TypeFullName>Microsoft.Testing.Platform.Extensions.MSTest.TestingPlatformBuilderHook</TypeFullName>
</TestingPlatformBuilderHook>
Expand Down
6 changes: 6 additions & 0 deletions src/Adapter/Build/Common/MSTest.TestAdapter.targets
Original file line number Diff line number Diff line change
@@ -1,5 +1,11 @@
<Project>

<!-- Handle the coexistance between testing platform and Microsoft.NET.Test.Sdk -->
<PropertyGroup>
<GenerateTestingPlatformEntryPoint>$(EnableMSTestRunner)</GenerateTestingPlatformEntryPoint>
<GenerateProgramFile Condition=" '$(EnableMSTestRunner)' == 'true' ">false</GenerateProgramFile>
</PropertyGroup>

<ItemGroup>
<TestAdapterContent Include="$(MSBuildThisFileDirectory)Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.dll" Condition=" '$(EnableMSTestRunner)' != 'true' ">
<Link>Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.dll</Link>
Expand Down
15 changes: 11 additions & 4 deletions src/Adapter/Build/NetWithWinUI/MSTest.TestAdapter.targets
Original file line number Diff line number Diff line change
@@ -1,4 +1,11 @@
<Project>

<!-- Handle the coexistance between testing platform and Microsoft.NET.Test.Sdk -->
<PropertyGroup>
<GenerateTestingPlatformEntryPoint>$(EnableMSTestRunner)</GenerateTestingPlatformEntryPoint>
<GenerateProgramFile Condition=" '$(EnableMSTestRunner)' == 'true' ">false</GenerateProgramFile>
</PropertyGroup>

<PropertyGroup Condition=" '$(UseWinUI)' == 'true' ">
<__TestAdapterPlatformServicesRoot>$(MSBuildThisFileDirectory)winui/</__TestAdapterPlatformServicesRoot>
</PropertyGroup>
Expand Down Expand Up @@ -64,10 +71,10 @@
<ItemGroup>
<Content Include="@(MSTestV2Files->'%(RootDir)%(Directory)$(CurrentUICultureHierarchy)\%(FileName).resources.dll')"
Condition="Exists('%(RootDir)%(Directory)$(CurrentUICultureHierarchy)\%(FileName).resources.dll')">
<Link>$(CurrentUICultureHierarchy)\%(FileName).resources.dll</Link>
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
<BaseAssemblyFullPath>%(FullPath)</BaseAssemblyFullPath>
<Visible>False</Visible>
<Link>$(CurrentUICultureHierarchy)\%(FileName).resources.dll</Link>
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
<BaseAssemblyFullPath>%(FullPath)</BaseAssemblyFullPath>
<Visible>False</Visible>
</Content>
</ItemGroup>
</Target>
Expand Down
14 changes: 9 additions & 5 deletions src/Adapter/Build/Universal/MSTest.TestAdapter.targets
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
<Project>

<PropertyGroup>
<EnableMSTestV2CopyResources Condition=" '$(EnableMSTestV2CopyResources)' == '' ">true</EnableMSTestV2CopyResources>
<!-- Handle the coexistance between testing platform and Microsoft.NET.Test.Sdk -->
<GenerateTestingPlatformEntryPoint>$(EnableMSTestRunner)</GenerateTestingPlatformEntryPoint>
<GenerateProgramFile Condition=" '$(EnableMSTestRunner)' == 'true' ">false</GenerateProgramFile>
</PropertyGroup>

<Target Name="GetMSTestV2CultureHierarchy">
Expand Down Expand Up @@ -32,13 +36,13 @@
<MSTestV2Files Include="$(MSBuildThisFileDirectory)*.dll" />
</ItemGroup>

<ItemGroup>
<ItemGroup>
<Content Include="@(MSTestV2Files->'%(RootDir)%(Directory)$(CurrentUICultureHierarchy)\%(FileName).resources.dll')"
Condition="Exists('%(RootDir)%(Directory)$(CurrentUICultureHierarchy)\%(FileName).resources.dll')">
<Link>$(CurrentUICultureHierarchy)\%(FileName).resources.dll</Link>
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
<BaseAssemblyFullPath>%(FullPath)</BaseAssemblyFullPath>
<Visible>False</Visible>
<Link>$(CurrentUICultureHierarchy)\%(FileName).resources.dll</Link>
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
<BaseAssemblyFullPath>%(FullPath)</BaseAssemblyFullPath>
<Visible>False</Visible>
</Content>
</ItemGroup>
</Target>
Expand Down
Original file line number Diff line number Diff line change
@@ -1,20 +1,53 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System.Runtime.InteropServices;
using System.Text.RegularExpressions;

namespace Microsoft.Testing.Platform.Acceptance.IntegrationTests;

/// <summary>
/// All the properties of this class should be non static.
/// At the moment are static because we need to share them between perclass/id fixtures and
/// it's not supported at the moment.
/// </summary>
public abstract class BaseAcceptanceTests : TestBase
public abstract partial class BaseAcceptanceTests : TestBase
{
[GeneratedRegex("^(.*?)\\.(?=(?:[0-9]+\\.){2,}[0-9]+(?:-[a-z]+)?\\.nupkg)(.*?)\\.nupkg$")]
private static partial Regex ParseNugetPacakgeFileName();

internal static string RID { get; private set; } = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "win-x64" : "linux-x64";

public static string MSTestCurrentVersion { get; private set; }

static BaseAcceptanceTests()
{
var mstestTestFrameworkPackage = Path.GetFileName(Directory.GetFiles(Constants.ArtifactsPackagesShipping, "MSTest.TestFramework*.nupkg", SearchOption.AllDirectories).Single());
Match match = ParseNugetPacakgeFileName().Match(mstestTestFrameworkPackage);
if (!match.Success)
{
throw new InvalidOperationException("Package version not found");
}

MSTestCurrentVersion = match.Groups[2].Value;
}

protected BaseAcceptanceTests(ITestExecutionContext testExecutionContext, AcceptanceFixture acceptanceFixture)
: base(testExecutionContext)
{
AcceptanceFixture = acceptanceFixture;
}

public AcceptanceFixture AcceptanceFixture { get; }

internal static IEnumerable<TestArgumentsEntry<(string Tfm, BuildConfiguration BuildConfiguration)>> GetBuildMatrixTfmBuildConfiguration()
{
foreach (TestArgumentsEntry<string> tfm in TargetFrameworks.All)
{
foreach (BuildConfiguration compilationMode in Enum.GetValues<BuildConfiguration>())
{
yield return new TestArgumentsEntry<(string Tfm, BuildConfiguration BuildConfiguration)>((tfm.Arguments, compilationMode), $"{tfm.Arguments},{compilationMode}");
}
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using Microsoft.Testing.Platform.Acceptance.IntegrationTests.Helpers;

namespace Microsoft.Testing.Platform.Acceptance.IntegrationTests;

[TestGroup]
public class MSTestRunnerTests : BaseAcceptanceTests
{
private readonly AcceptanceFixture _acceptanceFixture;
private const string AssetName = "MSTestProject";

public MSTestRunnerTests(ITestExecutionContext testExecutionContext, AcceptanceFixture acceptanceFixture)
: base(testExecutionContext, acceptanceFixture)
{
_acceptanceFixture = acceptanceFixture;
}

[ArgumentsProvider(nameof(GetBuildMatrixTfmBuildConfiguration))]
public async Task EnableMSTestRunner_True_Will_Run_Standalone(string tfm, BuildConfiguration buildConfiguration)
{
using TestAsset generator = await TestAsset.GenerateAssetAsync(
AssetName,
CurrentTemplateSourceCode
.PatchCodeWithRegularExpression("tfm", tfm)
.PatchCodeWithRegularExpression("mstestversion", MSTestCurrentVersion)
.PatchCodeWithRegularExpression("enablemstestrunner", "<EnableMSTestRunner>true</EnableMSTestRunner>")
.PatchCodeWithRegularExpression("outputtype", "<OutputType>Exe</OutputType>"),
addPublicFeeds: true);
string binlogFile = Path.Combine(generator.TargetAssetPath, "msbuild.binlog");
var compilationResult = await DotnetCli.RunAsync($"restore -nodeReuse:false {generator.TargetAssetPath} -r {RID}", _acceptanceFixture.NuGetGlobalPackagesFolder);
compilationResult = await DotnetCli.RunAsync(
$"build -nodeReuse:false {generator.TargetAssetPath} -c {buildConfiguration} -bl:{binlogFile} -r {RID}",
_acceptanceFixture.NuGetGlobalPackagesFolder, failIfReturnValueIsNotZero: false);
var testHost = TestInfrastructure.TestHost.LocateFrom(generator.TargetAssetPath, AssetName, tfm, buildConfiguration: buildConfiguration);
var testHostResult = await testHost.ExecuteAsync(environmentVariables: new Dictionary<string, string>() { { "TESTINGPLATFORM_TELEMETRY_OPTOUT", "1" } });
testHostResult.AssertOutputContains("Passed! - Failed: 0, Passed: 1, Skipped: 0, Total: 1");
}

[ArgumentsProvider(nameof(GetBuildMatrixTfmBuildConfiguration))]
public async Task EnableMSTestRunner_False_Will_Run_Empty_Program_EntryPoint_From_Tpv2_SDK(string tfm, BuildConfiguration buildConfiguration)
{
using TestAsset generator = await TestAsset.GenerateAssetAsync(
AssetName,
CurrentTemplateSourceCode
.PatchCodeWithRegularExpression("tfm", tfm)
.PatchCodeWithRegularExpression("mstestversion", MSTestCurrentVersion)
.PatchCodeWithRegularExpression("enablemstestrunner", "<EnableMSTestRunner>false</EnableMSTestRunner>")
.PatchCodeWithRegularExpression("outputtype", "<OutputType>Exe</OutputType>"),
addPublicFeeds: true);

string binlogFile = Path.Combine(generator.TargetAssetPath, "msbuild.binlog");
var compilationResult = await DotnetCli.RunAsync($"restore -nodeReuse:false {generator.TargetAssetPath} -r {RID}", _acceptanceFixture.NuGetGlobalPackagesFolder);
try
{
compilationResult = await DotnetCli.RunAsync($"build -nodeReuse:false {generator.TargetAssetPath} -c {buildConfiguration} -bl:{binlogFile} -r {RID}", _acceptanceFixture.NuGetGlobalPackagesFolder);
var testHost = TestInfrastructure.TestHost.LocateFrom(generator.TargetAssetPath, AssetName, tfm, buildConfiguration: buildConfiguration);
var testHostResult = await testHost.ExecuteAsync(environmentVariables: new Dictionary<string, string>() { { "TESTINGPLATFORM_TELEMETRY_OPTOUT", "1" } });
Assert.AreEqual(string.Empty, testHostResult.StandardOutput);
}
catch (Exception ex)
{
if (TargetFrameworks.NetFramework.Any(x => x.Arguments == tfm))
{
Assert.IsTrue(ex.Message.Contains("Program does not contain a static 'Main' method suitable for an entry point"), ex.Message);

// .NET Framework does not insert the entry point for empty program.
return;
}
}
}

private const string CurrentTemplateSourceCode = """
#file MSTestProject.csproj
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<TargetFramework>tfm</TargetFramework>
<IsPackable>false</IsPackable>
<IsTestProject>true</IsTestProject>
outputtype
enablemstestrunner
</PropertyGroup>

<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="*" />
<PackageReference Include="MSTest.TestAdapter" Version="mstestversion" />
<PackageReference Include="MSTest.TestFramework" Version="mstestversion" />
<PackageReference Include="coverlet.collector" Version="6.0.0" />
</ItemGroup>

</Project>

#file UnitTest1.cs
using Microsoft.VisualStudio.TestTools.UnitTesting;

[TestClass]
public class UnitTest1
{
[TestMethod]
public void TestMethod1()
{
}
}
""";
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
"profiles": {
"Microsoft.Testing.Platform.Acceptance.IntegrationTests": {
"commandName": "Project",
"commandLineArgs": "--treenode-filter /*/*/*/**",
// "commandLineArgs": "--treenode-filter /*/*/[TestClassName]/**",
"environmentVariables": {
//"TESTINGPLATFORM_HOTRELOAD_ENABLED": "1"
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
# Avoid automatic attach(AeDebug)

Create a file called `Crash.reg` with the following contents:

```txt
Windows Registry Editor Version 5.00

[HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows NT\CurrentVersion\AeDebug\AutoExclusionList]
"UnhandledExceptionPolicyTests.exe"=dword:00000001


```

Install the registry file by double clicking on it.
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,22 @@ Microsoft.Testing.Platform.Acceptance.IntegrationTests.Microsoft.Testing.Platfor
Microsoft.Testing.Platform.Acceptance.IntegrationTests.Microsoft.Testing.Platform.Acceptance.IntegrationTests.NoBannerTests.WithoutUsingNoBanner_TheBannerAppears(string) (net6.0)
Microsoft.Testing.Platform.Acceptance.IntegrationTests.Microsoft.Testing.Platform.Acceptance.IntegrationTests.NoBannerTests.WithoutUsingNoBanner_TheBannerAppears(string) (net7.0)
Microsoft.Testing.Platform.Acceptance.IntegrationTests.Microsoft.Testing.Platform.Acceptance.IntegrationTests.NoBannerTests.WithoutUsingNoBanner_TheBannerAppears(string) (net8.0)
Microsoft.Testing.Platform.Acceptance.IntegrationTests.Microsoft.Testing.Platform.Acceptance.IntegrationTests.MSTestRunnerTests.EnableMSTestRunner_False_Will_Run_Empty_Program_EntryPoint_From_Tpv2_SDK(string, Microsoft.Testing.TestInfrastructure.BuildConfiguration) (net462,Debug)
Microsoft.Testing.Platform.Acceptance.IntegrationTests.Microsoft.Testing.Platform.Acceptance.IntegrationTests.MSTestRunnerTests.EnableMSTestRunner_False_Will_Run_Empty_Program_EntryPoint_From_Tpv2_SDK(string, Microsoft.Testing.TestInfrastructure.BuildConfiguration) (net462,Release)
Microsoft.Testing.Platform.Acceptance.IntegrationTests.Microsoft.Testing.Platform.Acceptance.IntegrationTests.MSTestRunnerTests.EnableMSTestRunner_False_Will_Run_Empty_Program_EntryPoint_From_Tpv2_SDK(string, Microsoft.Testing.TestInfrastructure.BuildConfiguration) (net6.0,Debug)
Microsoft.Testing.Platform.Acceptance.IntegrationTests.Microsoft.Testing.Platform.Acceptance.IntegrationTests.MSTestRunnerTests.EnableMSTestRunner_False_Will_Run_Empty_Program_EntryPoint_From_Tpv2_SDK(string, Microsoft.Testing.TestInfrastructure.BuildConfiguration) (net6.0,Release)
Microsoft.Testing.Platform.Acceptance.IntegrationTests.Microsoft.Testing.Platform.Acceptance.IntegrationTests.MSTestRunnerTests.EnableMSTestRunner_False_Will_Run_Empty_Program_EntryPoint_From_Tpv2_SDK(string, Microsoft.Testing.TestInfrastructure.BuildConfiguration) (net7.0,Debug)
Microsoft.Testing.Platform.Acceptance.IntegrationTests.Microsoft.Testing.Platform.Acceptance.IntegrationTests.MSTestRunnerTests.EnableMSTestRunner_False_Will_Run_Empty_Program_EntryPoint_From_Tpv2_SDK(string, Microsoft.Testing.TestInfrastructure.BuildConfiguration) (net7.0,Release)
Microsoft.Testing.Platform.Acceptance.IntegrationTests.Microsoft.Testing.Platform.Acceptance.IntegrationTests.MSTestRunnerTests.EnableMSTestRunner_False_Will_Run_Empty_Program_EntryPoint_From_Tpv2_SDK(string, Microsoft.Testing.TestInfrastructure.BuildConfiguration) (net8.0,Debug)
Microsoft.Testing.Platform.Acceptance.IntegrationTests.Microsoft.Testing.Platform.Acceptance.IntegrationTests.MSTestRunnerTests.EnableMSTestRunner_False_Will_Run_Empty_Program_EntryPoint_From_Tpv2_SDK(string, Microsoft.Testing.TestInfrastructure.BuildConfiguration) (net8.0,Release)
Microsoft.Testing.Platform.Acceptance.IntegrationTests.Microsoft.Testing.Platform.Acceptance.IntegrationTests.MSTestRunnerTests.EnableMSTestRunner_True_Will_Run_Standalone(string, Microsoft.Testing.TestInfrastructure.BuildConfiguration) (net462,Debug)
Microsoft.Testing.Platform.Acceptance.IntegrationTests.Microsoft.Testing.Platform.Acceptance.IntegrationTests.MSTestRunnerTests.EnableMSTestRunner_True_Will_Run_Standalone(string, Microsoft.Testing.TestInfrastructure.BuildConfiguration) (net462,Release)
Microsoft.Testing.Platform.Acceptance.IntegrationTests.Microsoft.Testing.Platform.Acceptance.IntegrationTests.MSTestRunnerTests.EnableMSTestRunner_True_Will_Run_Standalone(string, Microsoft.Testing.TestInfrastructure.BuildConfiguration) (net6.0,Debug)
Microsoft.Testing.Platform.Acceptance.IntegrationTests.Microsoft.Testing.Platform.Acceptance.IntegrationTests.MSTestRunnerTests.EnableMSTestRunner_True_Will_Run_Standalone(string, Microsoft.Testing.TestInfrastructure.BuildConfiguration) (net6.0,Release)
Microsoft.Testing.Platform.Acceptance.IntegrationTests.Microsoft.Testing.Platform.Acceptance.IntegrationTests.MSTestRunnerTests.EnableMSTestRunner_True_Will_Run_Standalone(string, Microsoft.Testing.TestInfrastructure.BuildConfiguration) (net7.0,Debug)
Microsoft.Testing.Platform.Acceptance.IntegrationTests.Microsoft.Testing.Platform.Acceptance.IntegrationTests.MSTestRunnerTests.EnableMSTestRunner_True_Will_Run_Standalone(string, Microsoft.Testing.TestInfrastructure.BuildConfiguration) (net7.0,Release)
Microsoft.Testing.Platform.Acceptance.IntegrationTests.Microsoft.Testing.Platform.Acceptance.IntegrationTests.MSTestRunnerTests.EnableMSTestRunner_True_Will_Run_Standalone(string, Microsoft.Testing.TestInfrastructure.BuildConfiguration) (net8.0,Debug)
Microsoft.Testing.Platform.Acceptance.IntegrationTests.Microsoft.Testing.Platform.Acceptance.IntegrationTests.MSTestRunnerTests.EnableMSTestRunner_True_Will_Run_Standalone(string, Microsoft.Testing.TestInfrastructure.BuildConfiguration) (net8.0,Release)
Microsoft.Testing.Platform.Acceptance.IntegrationTests.Microsoft.Testing.Platform.Acceptance.IntegrationTests.TelemetryTests.Telemetry_ByDefault_TelemetryIsEnabled(string) (net462)
Microsoft.Testing.Platform.Acceptance.IntegrationTests.Microsoft.Testing.Platform.Acceptance.IntegrationTests.TelemetryTests.Telemetry_ByDefault_TelemetryIsEnabled(string) (net6.0)
Microsoft.Testing.Platform.Acceptance.IntegrationTests.Microsoft.Testing.Platform.Acceptance.IntegrationTests.TelemetryTests.Telemetry_ByDefault_TelemetryIsEnabled(string) (net7.0)
Expand Down
10 changes: 8 additions & 2 deletions test/Utilities/Microsoft.Testing.TestInfrastructure/TestAsset.cs
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,13 @@ private static (string Name, string Content) ParseFile(string fileContent)
return (name, content);
}

public static async Task<TestAsset> GenerateAssetAsync(string assetName, string code, bool addDefaultNuGetConfigFile = true)
public static async Task<TestAsset> GenerateAssetAsync(string assetName, string code, bool addDefaultNuGetConfigFile = true, bool addPublicFeeds = false)
{
string publicFeedsFragment = addPublicFeeds ? """
<add key="nuget.org" value="https://api.nuget.org/v3/index.json" />
<add key="test-tools" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/test-tools/nuget/v3/index.json" />
""" : string.Empty;

string defaultNuGetConfig = $"""

#file NuGet.config
Expand All @@ -64,10 +69,11 @@ public static async Task<TestAsset> GenerateAssetAsync(string assetName, string
<configuration>
<packageSources>
<clear/>
{publicFeedsFragment}
<add key="local-nonshipping" value="{Constants.ArtifactsPackagesNonShipping}" />
<add key="local-shipping" value="{Constants.ArtifactsPackagesShipping}" />
<add key="local-tmp-packages" value="{Constants.ArtifactsTmpPackages}" />
<add key="dotnet-public" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/index.json" />
<add key="dotnet-public" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/index.json" />
</packageSources>
<config>
<add key="globalPackagesFolder" value=".packages" />
Expand Down