Skip to content

Commit fc7f00a

Browse files
premunpremun
authored
[Rollout] Production rollout 2024-10-25 (#4101)
2 parents 4bb6117 + dc0a773 commit fc7f00a

File tree

36 files changed

+305
-147
lines changed

36 files changed

+305
-147
lines changed

‎.vault-config/maestroprod.yaml

+1-9
Original file line numberDiff line numberDiff line change
@@ -24,12 +24,4 @@ keys:
2424

2525
importSecretsFrom: shared/maestro-secrets.yaml
2626

27-
secrets:
28-
# Needed during Maestro rollouts to create GitHub releases in arcade-services
29-
BotAccount-dotnet-bot-repo-PAT:
30-
type: github-access-token
31-
parameters:
32-
gitHubBotAccountSecret:
33-
location: engkeyvault
34-
name: BotAccount-dotnet-bot
35-
gitHubBotAccountName: dotnet-bot
27+
secrets: {}

‎.vault-config/product-construction-dev.yaml

-7
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,6 @@ references:
1818
name: engkeyvault
1919

2020
secrets:
21-
BotAccount-dotnet-bot-repo-PAT:
22-
type: github-access-token
23-
parameters:
24-
gitHubBotAccountSecret:
25-
location: engkeyvault
26-
name: BotAccount-dotnet-bot
27-
gitHubBotAccountName: dotnet-bot
2821

2922
github:
3023
type: github-app-secret

‎.vault-config/product-construction-prod.yaml

+1-8
Original file line numberDiff line numberDiff line change
@@ -12,14 +12,7 @@ references:
1212
name: engkeyvault
1313

1414
secrets:
15-
BotAccount-dotnet-bot-repo-PAT:
16-
type: github-access-token
17-
parameters:
18-
gitHubBotAccountSecret:
19-
location: engkeyvault
20-
name: BotAccount-dotnet-bot
21-
gitHubBotAccountName: dotnet-bot
22-
15+
2316
github:
2417
type: github-app-secret
2518
parameters:

‎.vault-config/vmr-synchronization.1.yaml

-6
Original file line numberDiff line numberDiff line change
@@ -42,9 +42,3 @@ secrets:
4242
name: dn-bot-dnceng-build
4343
organizations: dnceng
4444
scopes: build_execute code_write
45-
46-
BotAccount-dotnet-bot-repo-PAT:
47-
type: github-access-token
48-
parameters:
49-
gitHubBotAccountSecret: BotAccount-dotnet-bot
50-
gitHubBotAccountName: dotnet-bot

‎eng/service-templates/ProductConstructionService/storage-account.bicep

+5
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,11 @@ resource storageAccountQueue 'Microsoft.Storage/storageAccounts/queueServices/qu
4343
parent: storageAccountQueueService
4444
}
4545

46+
resource codeFlowQueue 'Microsoft.Storage/storageAccounts/queueServices/queues@2022-09-01' = {
47+
name: 'pcs-codeflow-workitems'
48+
parent: storageAccountQueueService
49+
}
50+
4651
// allow storage queue access to the identity used for the aca's
4752
resource pcsStorageQueueAccess 'Microsoft.Authorization/roleAssignments@2022-04-01' = {
4853
scope: storageAccount

‎src/Maestro/FeedCleanerService/.config/settings.Production.json

-3
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,6 @@
44
"TableName": "healthreport"
55
},
66
"KeyVaultUri": "https://maestroprod.vault.azure.net/",
7-
"FeedCleaner": {
8-
"Enabled": true
9-
},
107
"BuildAssetRegistry": {
118
"ConnectionString": "Data Source=tcp:maestro-prod.database.windows.net,1433; Initial Catalog=BuildAssetRegistry; Authentication=Active Directory Managed Identity; Persist Security Info=False; MultipleActiveResultSets=True; Connect Timeout=30; Encrypt=True; TrustServerCertificate=False;"
129
},

‎src/Maestro/SubscriptionActorService/PullRequestActor.cs

+47-1
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
using Microsoft.Extensions.Logging;
1919
using Microsoft.ServiceFabric.Actors;
2020
using Microsoft.ServiceFabric.Actors.Runtime;
21+
using Microsoft.VisualStudio.Services.Common;
2122
using ProductConstructionService.Client;
2223
using ProductConstructionService.Client.Models;
2324
using SubscriptionActorService.StateModel;
@@ -872,8 +873,11 @@ await AddDependencyFlowEventsAsync(
872873
MergePolicyCheckResult.PendingPolicies,
873874
pr.Url);
874875

876+
var requiredDescriptionUpdates =
877+
await CalculateOriginalDependencies(darcRemote, targetRepository, targetBranch, targetRepositoryUpdates);
878+
875879
pullRequest.Description = await _pullRequestBuilder.CalculatePRDescriptionAndCommitUpdatesAsync(
876-
targetRepositoryUpdates.RequiredUpdates,
880+
requiredDescriptionUpdates,
877881
pullRequest.Description,
878882
targetRepository,
879883
pullRequest.HeadBranch);
@@ -1075,6 +1079,48 @@ private async Task<RepositoryBranchUpdate> GetRepositoryBranchUpdate()
10751079

10761080
private static string GetNewBranchName(string targetBranch) => $"darc-{targetBranch}-{Guid.NewGuid()}";
10771081

1082+
/// <summary>
1083+
/// Given a set of updates, replace the `from` version of every dependency update with the corresponding version
1084+
/// from the target branch
1085+
/// </summary>
1086+
/// <param name="darcRemote">Darc client used to fetch target branch dependencies.</param>
1087+
/// <param name="targetRepository">Target repository to fetch the dependencies from.</param>
1088+
/// <param name="targetBranch">Target branch to fetch the dependencies from.</param>
1089+
/// <param name="targetRepositoryUpdates">Incoming updates to the repository</param>
1090+
/// <returns>
1091+
/// Asset update and the corresponding list of altered dependencies
1092+
/// </returns>
1093+
/// <remarks>
1094+
/// This method is intended for use in situations where we want to keep the information about the original dependency
1095+
/// version, such as when updating PR descriptions.
1096+
/// </remarks>
1097+
private static async Task<List<(UpdateAssetsParameters update, List<DependencyUpdate> deps)>> CalculateOriginalDependencies(
1098+
IRemote darcRemote,
1099+
string targetRepository,
1100+
string targetBranch,
1101+
TargetRepoDependencyUpdate targetRepositoryUpdates)
1102+
{
1103+
List<DependencyDetail> targetBranchDeps = [..await darcRemote.GetDependenciesAsync(targetRepository, targetBranch)];
1104+
1105+
List<(UpdateAssetsParameters update, List<DependencyUpdate> deps)> alteredUpdates = [];
1106+
foreach (var requiredUpdate in targetRepositoryUpdates.RequiredUpdates)
1107+
{
1108+
var updatedDependencies = requiredUpdate.deps
1109+
.Select(dependency => new DependencyUpdate()
1110+
{
1111+
From = targetBranchDeps
1112+
.Where(replace => dependency.From.Name == replace.Name)
1113+
.FirstOrDefault(dependency.From),
1114+
To = dependency.To,
1115+
})
1116+
.ToList();
1117+
1118+
alteredUpdates.Add((requiredUpdate.update, updatedDependencies));
1119+
}
1120+
1121+
return alteredUpdates;
1122+
}
1123+
10781124
#region Code flow subscriptions
10791125

10801126
/// <summary>

‎src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/CloneManager.cs

+13-1
Original file line numberDiff line numberDiff line change
@@ -151,7 +151,19 @@ protected async Task<NativePath> PrepareCloneInternal(string remoteUri, string d
151151
else
152152
{
153153
_logger.LogDebug("Clone of {repo} found in {clonePath}", remoteUri, clonePath);
154-
var remote = await _localGitRepo.AddRemoteIfMissingAsync(clonePath, remoteUri, cancellationToken);
154+
155+
string remote;
156+
157+
try
158+
{
159+
remote = await _localGitRepo.AddRemoteIfMissingAsync(clonePath, remoteUri, cancellationToken);
160+
}
161+
catch (Exception e) when (e.Message.Contains("fatal: not a git repository"))
162+
{
163+
_logger.LogWarning("Clone at {clonePath} is not a git repository, re-cloning", clonePath);
164+
_fileSystem.DeleteDirectory(clonePath, recursive: true);
165+
return await PrepareCloneInternal(remoteUri, dirName, cancellationToken);
166+
}
155167

156168
// We cannot do `fetch --all` as tokens might be needed but fetch +refs/heads/*:+refs/remotes/origin/* doesn't fetch new refs
157169
// So we need to call `remote update origin` to fetch everything

‎src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -187,7 +187,7 @@ where sub.Enabled
187187

188188
if (subscriptionToUpdate != null)
189189
{
190-
await _workItemProducerFactory.CreateProducer<SubscriptionTriggerWorkItem>().ProduceWorkItemAsync(new()
190+
await _workItemProducerFactory.CreateProducer<SubscriptionTriggerWorkItem>(subscriptionToUpdate.SourceEnabled).ProduceWorkItemAsync(new()
191191
{
192192
SubscriptionId = subscriptionToUpdate.Id,
193193
BuildId = buildId

‎src/ProductConstructionService/ProductConstructionService.Api/Controllers/StatusController.cs

+14-2
Original file line numberDiff line numberDiff line change
@@ -45,8 +45,20 @@ public async Task<IActionResult> StopPcsWorkItemProcessors()
4545
{
4646
return Ok(await PerformActionOnAllProcessors(async stateCache =>
4747
{
48-
await stateCache.SetStateAsync(WorkItemProcessorState.Stopping);
49-
return (stateCache.ReplicaName, WorkItemProcessorState.Stopping);
48+
var state = await stateCache.GetStateAsync();
49+
switch (state)
50+
{
51+
case WorkItemProcessorState.Stopping:
52+
case WorkItemProcessorState.Working:
53+
await stateCache.SetStateAsync(WorkItemProcessorState.Stopping);
54+
return (stateCache.ReplicaName, WorkItemProcessorState.Stopping);
55+
case WorkItemProcessorState.Initializing:
56+
throw new BadHttpRequestException("Can't stop the service while initializing, try again later");
57+
case WorkItemProcessorState.Stopped:
58+
return (stateCache.ReplicaName, WorkItemProcessorState.Stopped);
59+
default:
60+
throw new Exception("PCS is in an unsupported state");
61+
}
5062
}));
5163
}
5264

‎src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs

+2-1
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,7 @@ static PcsStartup()
8181
{
8282
var context = (BuildAssetRegistryContext)entry.Context;
8383
ILogger<BuildAssetRegistryContext> logger = context.GetService<ILogger<BuildAssetRegistryContext>>();
84-
var workItemProducer = context.GetService<IWorkItemProducerFactory>().CreateProducer<SubscriptionTriggerWorkItem>();
84+
var workItemProducerFactory = context.GetService<IWorkItemProducerFactory>();
8585
var subscriptionIdGenerator = context.GetService<SubscriptionIdGenerator>();
8686
BuildChannel entity = entry.Entity;
8787

@@ -118,6 +118,7 @@ static PcsStartup()
118118

119119
foreach (Subscription subscription in subscriptionsToUpdate)
120120
{
121+
var workItemProducer = workItemProducerFactory.CreateProducer<SubscriptionTriggerWorkItem>(subscription.SourceEnabled);
121122
workItemProducer.ProduceWorkItemAsync(new()
122123
{
123124
BuildId = entity.BuildId,

‎src/ProductConstructionService/ProductConstructionService.Api/Program.cs

+3-2
Original file line numberDiff line numberDiff line change
@@ -48,8 +48,9 @@ await builder.ConfigurePcs(
4848
if (isDevelopment)
4949
{
5050
app.UseDeveloperExceptionPage();
51-
await app.Services.UseLocalWorkItemQueues(
52-
app.Configuration.GetRequiredValue(WorkItemConfiguration.WorkItemQueueNameConfigurationKey));
51+
await app.Services.UseLocalWorkItemQueues([
52+
app.Configuration.GetRequiredValue(WorkItemConfiguration.DefaultWorkItemQueueNameConfigurationKey),
53+
app.Configuration.GetRequiredValue(WorkItemConfiguration.CodeFlowWorkItemQueueNameConfigurationKey)]);
5354

5455
if (useSwagger)
5556
{

‎src/ProductConstructionService/ProductConstructionService.Api/appsettings.json

+3-2
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,9 @@
1212
}
1313
},
1414
"AllowedHosts": "*",
15-
"WorkItemQueueName": "pcs-workitems",
16-
"WorkItemConsumerCount": 5,
15+
"DefaultWorkItemQueueName": "pcs-workitems",
16+
"DefaultWorkItemConsumerCount": 4,
17+
"CodeFlowWorkItemQueueName": "pcs-codeflow-workitems",
1718
"WorkItemConsumerOptions": {
1819
"QueuePollTimeout": "00:01:00",
1920
"MaxWorkItemRetries": 3,

‎src/ProductConstructionService/ProductConstructionService.DependencyFlow/NonBatchedPullRequestUpdater.cs

+9-4
Original file line numberDiff line numberDiff line change
@@ -66,8 +66,12 @@ public NonBatchedPullRequestUpdater(
6666
Subscription? subscription = await _context.Subscriptions.FindAsync(SubscriptionId);
6767
if (subscription == null)
6868
{
69-
await _pullRequestCheckReminders.UnsetReminderAsync();
70-
await _pullRequestUpdateReminders.UnsetReminderAsync();
69+
// We don't know if the subscription was a code flow one, so just unset both
70+
await _pullRequestCheckReminders.UnsetReminderAsync(isCodeFlow: true);
71+
await _pullRequestCheckReminders.UnsetReminderAsync(isCodeFlow: false);
72+
await _pullRequestUpdateReminders.UnsetReminderAsync(isCodeFlow: true);
73+
await _pullRequestUpdateReminders.UnsetReminderAsync(isCodeFlow: false);
74+
7175
return null;
7276
}
7377

@@ -98,14 +102,15 @@ protected override async Task<IReadOnlyList<MergePolicyDefinition>> GetMergePoli
98102
}
99103

100104
protected override async Task<bool> CheckInProgressPullRequestAsync(
101-
InProgressPullRequest pullRequestCheck)
105+
InProgressPullRequest pullRequestCheck,
106+
bool isCodeFlow)
102107
{
103108
Subscription? subscription = await GetSubscription();
104109
if (subscription == null)
105110
{
106111
return false;
107112
}
108113

109-
return await base.CheckInProgressPullRequestAsync(pullRequestCheck);
114+
return await base.CheckInProgressPullRequestAsync(pullRequestCheck, isCodeFlow);
110115
}
111116
}

0 commit comments

Comments
 (0)