Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add flat flow test functionality to the ReproTool #4559

Open
wants to merge 7 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions arcade-services.sln
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ProductConstructionService.
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FlatFlowMigrationCli", "tools\FlatFlowMigrationCli\FlatFlowMigrationCli.csproj", "{89DF188B-B1FC-D3C4-A76E-019144ABA9CB}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Tools.Common", "tools\Tools.Common\Tools.Common.csproj", "{6D710AEF-A872-4F96-97F1-A8B9845839AB}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Expand Down Expand Up @@ -514,6 +516,18 @@ Global
{89DF188B-B1FC-D3C4-A76E-019144ABA9CB}.Release|x64.Build.0 = Release|Any CPU
{89DF188B-B1FC-D3C4-A76E-019144ABA9CB}.Release|x86.ActiveCfg = Release|Any CPU
{89DF188B-B1FC-D3C4-A76E-019144ABA9CB}.Release|x86.Build.0 = Release|Any CPU
{6D710AEF-A872-4F96-97F1-A8B9845839AB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{6D710AEF-A872-4F96-97F1-A8B9845839AB}.Debug|Any CPU.Build.0 = Debug|Any CPU
{6D710AEF-A872-4F96-97F1-A8B9845839AB}.Debug|x64.ActiveCfg = Debug|Any CPU
{6D710AEF-A872-4F96-97F1-A8B9845839AB}.Debug|x64.Build.0 = Debug|Any CPU
{6D710AEF-A872-4F96-97F1-A8B9845839AB}.Debug|x86.ActiveCfg = Debug|Any CPU
{6D710AEF-A872-4F96-97F1-A8B9845839AB}.Debug|x86.Build.0 = Debug|Any CPU
{6D710AEF-A872-4F96-97F1-A8B9845839AB}.Release|Any CPU.ActiveCfg = Release|Any CPU
{6D710AEF-A872-4F96-97F1-A8B9845839AB}.Release|Any CPU.Build.0 = Release|Any CPU
{6D710AEF-A872-4F96-97F1-A8B9845839AB}.Release|x64.ActiveCfg = Release|Any CPU
{6D710AEF-A872-4F96-97F1-A8B9845839AB}.Release|x64.Build.0 = Release|Any CPU
{6D710AEF-A872-4F96-97F1-A8B9845839AB}.Release|x86.ActiveCfg = Release|Any CPU
{6D710AEF-A872-4F96-97F1-A8B9845839AB}.Release|x86.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Expand Down Expand Up @@ -556,6 +570,7 @@ Global
{B95B12A8-FCE1-618A-CA77-134B59A5C050} = {02EA681E-C7D8-13C7-8484-4AC65E1B71E8}
{261CB211-6023-8025-48E1-D11953F4C61C} = {02EA681E-C7D8-13C7-8484-4AC65E1B71E8}
{89DF188B-B1FC-D3C4-A76E-019144ABA9CB} = {02EA681E-C7D8-13C7-8484-4AC65E1B71E8}
{6D710AEF-A872-4F96-97F1-A8B9845839AB} = {02EA681E-C7D8-13C7-8484-4AC65E1B71E8}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {32B9C883-432E-4FC8-A1BF-090EB033DD5B}
Expand Down
1 change: 1 addition & 0 deletions tools/FlatFlowMigrationCli/FlatFlowMigrationCli.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -20,5 +20,6 @@
<ProjectReference Include="..\..\src\Microsoft.DotNet.Darc\DarcLib\Microsoft.DotNet.DarcLib.csproj" />
<ProjectReference Include="..\..\src\ProductConstructionService\Microsoft.DotNet.ProductConstructionService.Client\Microsoft.DotNet.ProductConstructionService.Client.csproj" />
<ProjectReference Include="..\..\src\ProductConstructionService\ProductConstructionService.Common\ProductConstructionService.Common.csproj" />
<ProjectReference Include="..\Tools.Common\Tools.Common.csproj" />
</ItemGroup>
</Project>
1 change: 1 addition & 0 deletions tools/FlatFlowMigrationCli/MigrationLogger.cs
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
using System.Text.Json.Serialization;
using Microsoft.DotNet.ProductConstructionService.Client.Models;
using Microsoft.Extensions.Logging;
using Tools.Common;

namespace FlatFlowMigrationCli;

Expand Down
2 changes: 2 additions & 0 deletions tools/FlatFlowMigrationCli/Operations/MigrateOperation.cs
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@
using Microsoft.DotNet.ProductConstructionService.Client;
using Microsoft.DotNet.ProductConstructionService.Client.Models;
using Microsoft.Extensions.Logging;
using Tools.Common;
using Constants = Tools.Common.Constants;

namespace FlatFlowMigrationCli.Operations;

Expand Down
1 change: 1 addition & 0 deletions tools/FlatFlowMigrationCli/Options/Options.cs
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Console;
using ProductConstructionService.Common;
using Tools.Common;

namespace FlatFlowMigrationCli.Options;

Expand Down
1 change: 1 addition & 0 deletions tools/FlatFlowMigrationCli/SubscriptionMigrator.cs
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
using Microsoft.DotNet.ProductConstructionService.Client;
using Microsoft.DotNet.ProductConstructionService.Client.Models;
using Microsoft.Extensions.Logging;
using Tools.Common;

namespace FlatFlowMigrationCli;

Expand Down
15 changes: 11 additions & 4 deletions tools/ProductConstructionService.ReproTool/DarcProcessManager.cs
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ internal async Task<ProcessExecutionResult> ExecuteAsync(IEnumerable<string> arg
_darcExePath,
[
.. args,
"--bar-uri", ReproToolConfiguration.PcsLocalUri
"--bar-uri", Options.Options.PcsLocalUri
]);
}

Expand Down Expand Up @@ -135,14 +135,19 @@ public async Task<AsyncDisposableValue<string>> CreateSubscriptionAsync(
string targetBranch,
string? sourceDirectory,
string? targetDirectory,
bool skipCleanup)
bool skipCleanup,
List<string>? excludedAssets = null)
{
logger.LogInformation("Creating a test subscription");

string[] directoryArg = !string.IsNullOrEmpty(sourceDirectory) ?
["--source-directory", sourceDirectory] :
["--target-directory", targetDirectory!];

string[] excludedAssetsParameter = excludedAssets != null ?
[ "--excluded-assets", string.Join(';', excludedAssets) ] :
[];

var res = await ExecuteAsync([
"add-subscription",
"--channel", channel,
Expand All @@ -153,7 +158,8 @@ public async Task<AsyncDisposableValue<string>> CreateSubscriptionAsync(
"--no-trigger",
"--source-enabled", "true",
"--update-frequency", "none",
.. directoryArg
.. directoryArg,
.. excludedAssetsParameter
]);

Match match = Regex.Match(res.StandardOutput, "Successfully created new subscription with id '([a-f0-9-]+)'");
Expand Down Expand Up @@ -187,7 +193,8 @@ public async Task<ProcessExecutionResult> TriggerSubscriptionAsync(string subscr
return await ExecuteAsync(
[
"trigger-subscriptions",
"--ids", subscriptionId
"--ids", subscriptionId,
"-q"
]);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using Maestro.Data;
using Microsoft.DotNet.DarcLib;
using Microsoft.DotNet.ProductConstructionService.Client;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Tools.Common;
using GitHubClient = Octokit.GitHubClient;

namespace ProductConstructionService.ReproTool.Operations;

internal class FlatFlowTestOperation(
VmrDependencyResolver vmrDependencyResolver,
ILogger<FlatFlowTestOperation> logger,
GitHubClient ghClient,
BuildAssetRegistryContext context,
DarcProcessManager darcProcessManager,
IBarApiClient prodBarClient,
[FromKeyedServices("local")] IProductConstructionServiceApi localPcsApi) : Operation(logger, ghClient, context, localPcsApi)
{
internal override async Task RunAsync()
{
await darcProcessManager.InitializeAsync();

var vmrRepos = await vmrDependencyResolver.GetVmrRepositoriesAsync(
"https://github.com/dotnet/dotnet",
"https://github.com/dotnet/sdk",
"main");
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This should probably come from the Options.VmrBranch?


vmrRepos = vmrRepos.Where(d => d.Mapping.Name == "runtime").ToList();
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This probably stayed behind


var vmrTestBranch = await PrepareVmrForkAsync("main", skipCleanup: true);

var channelName = $"repro-{Guid.NewGuid()}";
await using var channel = await darcProcessManager.CreateTestChannelAsync(channelName, true);

foreach (var vmrRepo in vmrRepos)
{
var productRepoForkUri = $"{ProductRepoFormat}{vmrRepo.Mapping.Name}";
if (vmrRepo.Mapping.Name == "nuget-client")
{
productRepoForkUri = $"{ProductRepoFormat}nuget.client";
}
Comment on lines +41 to +45
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The URI is in vmrRepo.Channel.Repository

var latestBuild = await prodBarClient.GetLatestBuildAsync(vmrRepo.Mapping.DefaultRemote, vmrRepo.Channel.Channel.Id);

var productRepoTmpBranch = await PrepareProductRepoForkAsync(vmrRepo.Mapping.DefaultRemote, productRepoForkUri, latestBuild.GetBranch(), false);

var testBuild = await CreateBuildAsync(
productRepoForkUri,
productRepoTmpBranch.Value,
latestBuild.Commit,
[]);

await UpdateVmrSourceFiles(
vmrTestBranch.Value,
vmrRepo.Mapping.DefaultRemote,
productRepoForkUri);

await using var testSubscription = await darcProcessManager.CreateSubscriptionAsync(
channel: channelName,
sourceRepo: productRepoForkUri,
targetRepo: VmrForkUri,
targetBranch: vmrTestBranch.Value,
sourceDirectory: null,
targetDirectory: vmrRepo.Mapping.Name,
skipCleanup: true);

await darcProcessManager.AddBuildToChannelAsync(testBuild.Id, channelName, skipCleanup: true);

await TriggerSubscriptionAsync(testSubscription.Value);
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using Maestro.Data;
using Microsoft.DotNet.DarcLib;
using Microsoft.DotNet.ProductConstructionService.Client;
using Microsoft.DotNet.ProductConstructionService.Client.Models;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using ProductConstructionService.ReproTool.Options;
using Tools.Common;
using GitHubClient = Octokit.GitHubClient;

namespace ProductConstructionService.ReproTool.Operations;
internal class FullBackflowTestOperation : Operation
{
private readonly IBarApiClient _prodBarClient;
private readonly FullBackflowTestOptions _options;
private readonly DarcProcessManager _darcProcessManager;
private readonly VmrDependencyResolver _vmrDependencyResolver;

public FullBackflowTestOperation(
ILogger<Operation> logger,
GitHubClient ghClient,
BuildAssetRegistryContext context,
[FromKeyedServices("local")] IProductConstructionServiceApi localPcsApi,
IBarApiClient prodBarClient,
FullBackflowTestOptions options,
DarcProcessManager darcProcessManager,
VmrDependencyResolver vmrDependencyResolver)
: base(logger, ghClient, context, localPcsApi)
{
_prodBarClient = prodBarClient;
_options = options;
_darcProcessManager = darcProcessManager;
_vmrDependencyResolver = vmrDependencyResolver;
}

internal override async Task RunAsync()
{
await _darcProcessManager.InitializeAsync();
Build vmrBuild = await _prodBarClient.GetBuildAsync(_options.BuildId);

Build testBuild = await CreateBuildAsync(
VmrForkUri,
_options.VmrBranch,
_options.Commit,
[ ..CreateAssetDataFromBuild(vmrBuild).Take(1000)]);

var channelName = $"repro-{Guid.NewGuid()}";
await using var channel = await _darcProcessManager.CreateTestChannelAsync(channelName, skipCleanup: true);
await _darcProcessManager.AddBuildToChannelAsync(testBuild.Id, channelName, skipCleanup: true);

var vmrRepos = (await _vmrDependencyResolver.GetVmrRepositoriesAsync(
"https://github.com/dotnet/dotnet",
"https://github.com/dotnet/sdk",
"main"));

foreach (var vmrRepo in vmrRepos)
{
var productRepoForkUri = $"{ProductRepoFormat}{vmrRepo.Mapping.Name}";
if (vmrRepo.Mapping.Name == "nuget-client")
{
productRepoForkUri = $"{ProductRepoFormat}nuget.client";
}
Comment on lines +61 to +65
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same as before


var subscription = await _darcProcessManager.CreateSubscriptionAsync(
channel: channelName,
sourceRepo: VmrForkUri,
targetRepo: productRepoForkUri,
targetBranch: _options.TargetBranch,
sourceDirectory: vmrRepo.Mapping.Name,
targetDirectory: null,
skipCleanup: true);

await _darcProcessManager.TriggerSubscriptionAsync(subscription.Value);
}
}
}
Loading