Skip to content

Commit b7cd45a

Browse files
freddydkfreddydk
authored
Increase Get-BcArtifactUrl Performance (#3739)
When looking for the latest version of the artifacts for a specific country, we need to fetch all artifacts in the blob storage in order to figure out what is latest. This fix uses an approximation of latest version number (actually the one shipping in a month) and trying to download that (or 2 prior versions) Additional improvements we are looking at for future optimizations: - Use OCI artifacts for artifacts storage instead of traditional blob storage - Restructure artifacts in more layers to allow for partial download of what you need in various scenarios - Create index if artifacts to allow for querying artifacts without reading tags/annotations --------- Co-authored-by: freddydk <[email protected]>
1 parent aaf98da commit b7cd45a

File tree

4 files changed

+175
-137
lines changed

4 files changed

+175
-137
lines changed

‎Artifacts/Get-BCArtifactUrl.ps1

+170-136
Original file line numberDiff line numberDiff line change
@@ -141,172 +141,206 @@ try {
141141
if ($storageAccount -eq 'bcinsider.blob.core.windows.net' -and !$accept_insiderEULA) {
142142
throw "You need to accept the insider EULA (https://go.microsoft.com/fwlink/?linkid=2245051) by specifying -accept_insiderEula or by providing a SAS token to get access to insider builds"
143143
}
144-
$GetListUrl = "https://$storageAccount/$($Type.ToLowerInvariant())/?comp=list&restype=container"
145-
146-
$upMajorFilter = ''
147-
$upVersionFilter = ''
148-
if ($select -eq 'SecondToLastMajor') {
149-
if ($version) {
150-
throw "You cannot specify a version when asking for the Second To Last Major version"
151-
}
152-
}
153-
elseif ($select -eq 'Closest') {
154-
if (!($version)) {
155-
throw "You must specify a version number when you want to get the closest artifact Url"
144+
145+
if ($type -eq 'sandbox' -and $storageAccount -eq 'bcartifacts.blob.core.windows.net' -and $select -eq 'latest' -and $version -eq '' -and $bcContainerHelperConfig.useApproximateVersion) {
146+
# Temp fix / hack for Get-BcArtifact performance
147+
# If Microsoft changes versioning schema, this needs to change (or useApproximateVersion should be set to false)
148+
$now = ([DateTime]::Now).AddDays(15)
149+
$approximateMajor = 23+2*($now.Year-2024)+($now.Month -ge 4)+($now.Month -ge 10)
150+
$approximateMinor = ($now.Month + 2)%6
151+
$artifactUrl = Get-BCArtifactUrl -country $country -version "$approximateMajor.$approximateMinor" -select Latest -doNotCheckPlatform:$doNotCheckPlatform
152+
if ($artifactUrl) {
153+
# We found an artifact - check if it is the latest
154+
while ($artifactUrl) {
155+
$lastGoodArtifact = $artifactUrl
156+
if ($approximateMinor -eq 5) {
157+
$approximateMajor += 1
158+
$approximateMinor = 0
159+
}
160+
else {
161+
$approximateMinor += 1
162+
}
163+
$artifactUrl = Get-BCArtifactUrl -country $country -version "$approximateMajor.$approximateMinor" -select Latest -doNotCheckPlatform:$doNotCheckPlatform
164+
}
165+
$artifactUrl = $lastGoodArtifact
156166
}
157-
$dots = ($version.ToCharArray() -eq '.').Count
158-
$closestToVersion = [Version]"0.0.0.0"
159-
if ($dots -ne 3 -or !([Version]::TryParse($version, [ref] $closestToVersion))) {
160-
throw "Version number must be in the format 1.2.3.4 when you want to get the closest artifact Url"
167+
else {
168+
# No artifact found - try previous 3 versions (else give up - maybe country is unavailable)
169+
$tryVersions = 3
170+
while (-not $artifactUrl -and $tryVersions-- -gt 0) {
171+
if ($approximateMinor -eq 0) {
172+
$approximateMajor -= 1
173+
$approximateMinor = 5
174+
}
175+
else {
176+
$approximateMinor -= 1
177+
}
178+
$artifactUrl = Get-BCArtifactUrl -country $country -version "$approximateMajor.$approximateMinor" -select Latest -doNotCheckPlatform:$doNotCheckPlatform
179+
}
161180
}
162-
$GetListUrl += "&prefix=$($closestToVersion.Major).$($closestToVersion.Minor)."
163-
$upMajorFilter = "$($closestToVersion.Major)"
164-
$upVersionFilter = "$($closestToVersion.Minor)."
181+
$artifactUrl
165182
}
166-
elseif (!([string]::IsNullOrEmpty($version))) {
167-
$dots = ($version.ToCharArray() -eq '.').Count
168-
if ($dots -lt 3) {
169-
# avoid 14.1 returning 14.10, 14.11 etc.
170-
$version = "$($version.TrimEnd('.'))."
183+
else {
184+
$GetListUrl = "https://$storageAccount/$($Type.ToLowerInvariant())/?comp=list&restype=container"
185+
if ($select -eq 'SecondToLastMajor') {
186+
if ($version) {
187+
throw "You cannot specify a version when asking for the Second To Last Major version"
188+
}
171189
}
172-
$GetListUrl += "&prefix=$($Version)"
173-
$upMajorFilter = $version.Split('.')[0]
174-
$upVersionFilter = $version.Substring($version.Length).TrimStart('.')
175-
}
176-
177-
$Artifacts = @()
178-
$nextMarker = ''
179-
$currentMarker = ''
180-
$downloadAttempt = 1
181-
$downloadRetryAttempts = 10
182-
do {
183-
if ($currentMarker -ne $nextMarker)
184-
{
185-
$currentMarker = $nextMarker
186-
$downloadAttempt = 1
190+
elseif ($select -eq 'Closest') {
191+
if (!($version)) {
192+
throw "You must specify a version number when you want to get the closest artifact Url"
193+
}
194+
$dots = ($version.ToCharArray() -eq '.').Count
195+
$closestToVersion = [Version]"0.0.0.0"
196+
if ($dots -ne 3 -or !([Version]::TryParse($version, [ref] $closestToVersion))) {
197+
throw "Version number must be in the format 1.2.3.4 when you want to get the closest artifact Url"
198+
}
199+
$GetListUrl += "&prefix=$($closestToVersion.Major).$($closestToVersion.Minor)."
187200
}
188-
Write-Verbose "Download String $GetListUrl$nextMarker"
189-
try
190-
{
191-
$Response = Invoke-RestMethod -UseBasicParsing -ContentType "application/json; charset=UTF8" -Uri "$GetListUrl$nextMarker"
192-
if (([int]$Response[0]) -eq 239 -and ([int]$Response[1]) -eq 187 -and ([int]$Response[2]) -eq 191) {
193-
# Remove UTF8 BOM
194-
$response = $response.Substring(3)
201+
elseif (!([string]::IsNullOrEmpty($version))) {
202+
$dots = ($version.ToCharArray() -eq '.').Count
203+
if ($dots -lt 3) {
204+
# avoid 14.1 returning 14.10, 14.11 etc.
205+
$version = "$($version.TrimEnd('.'))."
195206
}
196-
if (([int]$Response[0]) -eq 65279) {
197-
# Remove Unicode BOM (PowerShell 7.4)
198-
$response = $response.Substring(1)
207+
$GetListUrl += "&prefix=$($Version)"
208+
}
209+
210+
$Artifacts = @()
211+
$nextMarker = ''
212+
$currentMarker = ''
213+
$downloadAttempt = 1
214+
$downloadRetryAttempts = 10
215+
do {
216+
if ($currentMarker -ne $nextMarker)
217+
{
218+
$currentMarker = $nextMarker
219+
$downloadAttempt = 1
199220
}
200-
$enumerationResults = ([xml]$Response).EnumerationResults
201-
202-
if ($enumerationResults.Blobs) {
203-
if (($After) -or ($Before)) {
204-
$artifacts += $enumerationResults.Blobs.Blob | % {
205-
if ($after) {
206-
$blobModifiedDate = [DateTime]::Parse($_.Properties."Last-Modified")
207-
if ($before) {
208-
if ($blobModifiedDate -lt $before -and $blobModifiedDate -gt $after) {
221+
Write-Verbose "Download String $GetListUrl$nextMarker"
222+
try
223+
{
224+
$Response = Invoke-RestMethod -UseBasicParsing -ContentType "application/json; charset=UTF8" -Uri "$GetListUrl$nextMarker"
225+
if (([int]$Response[0]) -eq 239 -and ([int]$Response[1]) -eq 187 -and ([int]$Response[2]) -eq 191) {
226+
# Remove UTF8 BOM
227+
$response = $response.Substring(3)
228+
}
229+
if (([int]$Response[0]) -eq 65279) {
230+
# Remove Unicode BOM (PowerShell 7.4)
231+
$response = $response.Substring(1)
232+
}
233+
$enumerationResults = ([xml]$Response).EnumerationResults
234+
235+
if ($enumerationResults.Blobs) {
236+
if (($After) -or ($Before)) {
237+
$artifacts += $enumerationResults.Blobs.Blob | % {
238+
if ($after) {
239+
$blobModifiedDate = [DateTime]::Parse($_.Properties."Last-Modified")
240+
if ($before) {
241+
if ($blobModifiedDate -lt $before -and $blobModifiedDate -gt $after) {
242+
$_.Name
243+
}
244+
}
245+
elseif ($blobModifiedDate -gt $after) {
209246
$_.Name
210247
}
211248
}
212-
elseif ($blobModifiedDate -gt $after) {
213-
$_.Name
214-
}
215-
}
216-
else {
217-
$blobModifiedDate = [DateTime]::Parse($_.Properties."Last-Modified")
218-
if ($blobModifiedDate -lt $before) {
219-
$_.Name
249+
else {
250+
$blobModifiedDate = [DateTime]::Parse($_.Properties."Last-Modified")
251+
if ($blobModifiedDate -lt $before) {
252+
$_.Name
253+
}
220254
}
221255
}
222256
}
257+
else {
258+
$artifacts += $enumerationResults.Blobs.Blob.Name
259+
}
223260
}
224-
else {
225-
$artifacts += $enumerationResults.Blobs.Blob.Name
261+
$nextMarker = $enumerationResults.NextMarker
262+
if ($nextMarker) {
263+
$nextMarker = "&marker=$nextMarker"
226264
}
227265
}
228-
$nextMarker = $enumerationResults.NextMarker
229-
if ($nextMarker) {
230-
$nextMarker = "&marker=$nextMarker"
231-
}
232-
}
233-
catch
234-
{
235-
$downloadAttempt += 1
236-
Write-Host "Error querying artifacts. Error message was $($_.Exception.Message)"
237-
Write-Host
238-
239-
if ($downloadAttempt -le $downloadRetryAttempts)
266+
catch
240267
{
241-
Write-Host "Repeating download attempt (" $downloadAttempt.ToString() " of " $downloadRetryAttempts.ToString() ")..."
268+
$downloadAttempt += 1
269+
Write-Host "Error querying artifacts. Error message was $($_.Exception.Message)"
242270
Write-Host
271+
272+
if ($downloadAttempt -le $downloadRetryAttempts)
273+
{
274+
Write-Host "Repeating download attempt (" $downloadAttempt.ToString() " of " $downloadRetryAttempts.ToString() ")..."
275+
Write-Host
276+
}
277+
else
278+
{
279+
throw
280+
}
243281
}
244-
else
245-
{
246-
throw
247-
}
248-
}
249-
} while ($nextMarker)
282+
} while ($nextMarker)
250283

251-
if (!([string]::IsNullOrEmpty($country))) {
252-
# avoid confusion between base and se
253-
$countryArtifacts = $Artifacts | Where-Object { $_.EndsWith("/$country", [System.StringComparison]::InvariantCultureIgnoreCase) -and ($doNotCheckPlatform -or ($Artifacts.Contains("$($_.Split('/')[0])/platform"))) }
254-
if (!$countryArtifacts) {
255-
if (($type -eq "sandbox") -and ($bcContainerHelperConfig.mapCountryCode.PSObject.Properties.Name -eq $country)) {
256-
$country = $bcContainerHelperConfig.mapCountryCode."$country"
257-
$countryArtifacts = $Artifacts | Where-Object { $_.EndsWith("/$country", [System.StringComparison]::InvariantCultureIgnoreCase) -and ($doNotCheckPlatform -or ($Artifacts.Contains("$($_.Split('/')[0])/platform"))) }
284+
if (!([string]::IsNullOrEmpty($country))) {
285+
# avoid confusion between base and se
286+
$countryArtifacts = $Artifacts | Where-Object { $_.EndsWith("/$country", [System.StringComparison]::InvariantCultureIgnoreCase) -and ($doNotCheckPlatform -or ($Artifacts.Contains("$($_.Split('/')[0])/platform"))) }
287+
if (!$countryArtifacts) {
288+
if (($type -eq "sandbox") -and ($bcContainerHelperConfig.mapCountryCode.PSObject.Properties.Name -eq $country)) {
289+
$country = $bcContainerHelperConfig.mapCountryCode."$country"
290+
$countryArtifacts = $Artifacts | Where-Object { $_.EndsWith("/$country", [System.StringComparison]::InvariantCultureIgnoreCase) -and ($doNotCheckPlatform -or ($Artifacts.Contains("$($_.Split('/')[0])/platform"))) }
291+
}
258292
}
293+
$Artifacts = $countryArtifacts
259294
}
260-
$Artifacts = $countryArtifacts
261-
}
262-
else {
263-
$Artifacts = $Artifacts | Where-Object { !($_.EndsWith("/platform", [System.StringComparison]::InvariantCultureIgnoreCase)) }
264-
}
265-
266-
switch ($Select) {
267-
'All' {
268-
$Artifacts = $Artifacts |
269-
Sort-Object { [Version]($_.Split('/')[0]) }
270-
}
271-
'Latest' {
272-
$Artifacts = $Artifacts |
273-
Sort-Object { [Version]($_.Split('/')[0]) } |
274-
Select-Object -Last 1
275-
}
276-
'First' {
277-
$Artifacts = $Artifacts |
278-
Sort-Object { [Version]($_.Split('/')[0]) } |
279-
Select-Object -First 1
295+
else {
296+
$Artifacts = $Artifacts | Where-Object { !($_.EndsWith("/platform", [System.StringComparison]::InvariantCultureIgnoreCase)) }
280297
}
281-
'SecondToLastMajor' {
282-
$Artifacts = $Artifacts |
283-
Sort-Object -Descending { [Version]($_.Split('/')[0]) }
284-
$latest = $Artifacts | Select-Object -First 1
285-
if ($latest) {
286-
$latestversion = [Version]($latest.Split('/')[0])
298+
299+
switch ($Select) {
300+
'All' {
301+
$Artifacts = $Artifacts |
302+
Sort-Object { [Version]($_.Split('/')[0]) }
303+
}
304+
'Latest' {
305+
$Artifacts = $Artifacts |
306+
Sort-Object { [Version]($_.Split('/')[0]) } |
307+
Select-Object -Last 1
308+
}
309+
'First' {
287310
$Artifacts = $Artifacts |
288-
Where-Object { ([Version]($_.Split('/')[0])).Major -ne $latestversion.Major } |
311+
Sort-Object { [Version]($_.Split('/')[0]) } |
289312
Select-Object -First 1
290313
}
291-
else {
292-
$Artifacts = @()
314+
'SecondToLastMajor' {
315+
$Artifacts = $Artifacts |
316+
Sort-Object -Descending { [Version]($_.Split('/')[0]) }
317+
$latest = $Artifacts | Select-Object -First 1
318+
if ($latest) {
319+
$latestversion = [Version]($latest.Split('/')[0])
320+
$Artifacts = $Artifacts |
321+
Where-Object { ([Version]($_.Split('/')[0])).Major -ne $latestversion.Major } |
322+
Select-Object -First 1
323+
}
324+
else {
325+
$Artifacts = @()
326+
}
293327
}
294-
}
295-
'Closest' {
296-
$Artifacts = $Artifacts |
297-
Sort-Object { [Version]($_.Split('/')[0]) }
298-
$closest = $Artifacts |
299-
Where-Object { [Version]($_.Split('/')[0]) -ge $closestToVersion } |
300-
Select-Object -First 1
301-
if (-not $closest) {
302-
$closest = $Artifacts | Select-Object -Last 1
328+
'Closest' {
329+
$Artifacts = $Artifacts |
330+
Sort-Object { [Version]($_.Split('/')[0]) }
331+
$closest = $Artifacts |
332+
Where-Object { [Version]($_.Split('/')[0]) -ge $closestToVersion } |
333+
Select-Object -First 1
334+
if (-not $closest) {
335+
$closest = $Artifacts | Select-Object -Last 1
336+
}
337+
$Artifacts = $closest
303338
}
304-
$Artifacts = $closest
305339
}
306-
}
307340

308-
foreach ($Artifact in $Artifacts) {
309-
"$BaseUrl$($Artifact)$sasToken"
341+
foreach ($Artifact in $Artifacts) {
342+
"$BaseUrl$($Artifact)$sasToken"
343+
}
310344
}
311345
}
312346
}

‎BC.HelperFunctions.ps1

+1
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,7 @@ function Get-ContainerHelperConfig {
118118
"IsGitHubActions" = ($env:GITHUB_ACTIONS -eq "true")
119119
"IsAzureDevOps" = ($env:TF_BUILD -eq "true")
120120
"IsGitLab" = ($env:GITLAB_CI -eq "true")
121+
"useApproximateVersion" = $false
121122
}
122123

123124
if ($isInsider) {

‎ReleaseNotes.txt

+3
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
1+
6.0.28
2+
Increase performance of Get-BcArtifactUrl when selecting latest artifact, by using an approximate filtering (set useApproximateVersion to false in settings to disable)
3+
14
6.0.27
25
Issue 3538 Compile-AppWithBcCompilerFolder fails when dependency does propagateDependencies
36
Issue 3727 Regression - Release pipelines failing with SaaS environments due BcAuthContext

‎Version.txt

+1-1
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
6.0.27-dev
1+
6.0.28-dev

0 commit comments

Comments
 (0)