useEarliestArtifact + a few small bugs (microsoft#1029)

Fix regression: DownloadRelease escapes the '*' sign in the project name

Fixes microsoft#1018
Fixes microsoft#970
Fixes microsoft#818
Fixes microsoft#918

Adding new setting:
- `useEarliestArtifact`: set this to true in order for AL-Go for GitHub
to build a project using the earliest compatible Business Central
artifact, determined by the highest application dependency from app.json
in all apps in the project and the `applicationDependency` in project
settings.

Small bugs:
- Only show a warning about a specific version of BcContainerHelper if
this is specified in Settings

---------

Co-authored-by: freddydk <[email protected]>
Co-authored-by: Alexander Holstrup <[email protected]>

Author: freddydk

Actions/AL-Go-Helper.ps1

@@ -400,6 +400,9 @@ function DownloadAndImportBcContainerHelper([string] $baseFolder = $ENV:GITHUB_W
             if ($bcContainerHelperVersion -like "https://*") {
                 throw "Setting BcContainerHelperVersion to a URL in settings is not allowed. Fork the AL-Go repository and use direct AL-Go development instead."
             }
+            if ($bcContainerHelperVersion -ne 'latest' -and $bcContainerHelperVersion -ne 'preview') {
+                Write-Host "::Warning::Using a specific version of BcContainerHelper is not recommended and will lead to build failures in the future. Consider removing the setting."
+            }
         }
         $params += @{ "bcContainerHelperConfigFile" = $repoSettingsPath }
     }
@@ -412,10 +415,6 @@ function DownloadAndImportBcContainerHelper([string] $baseFolder = $ENV:GITHUB_W
         throw "ContainerHelperVersion private is no longer supported. Use direct AL-Go development and a direct download url instead."
     }
 
-    if ($bcContainerHelperVersion -ne 'latest' -and $bcContainerHelperVersion -ne 'preview') {
-        Write-Host "::Warning::Using a specific version of BcContainerHelper is not recommended and will lead to build failures in the future. Consider removing the setting."
-    }
-
     $bcContainerHelperPath = GetBcContainerHelperPath -bcContainerHelperVersion $bcContainerHelperVersion
 
     Write-Host "Import from $bcContainerHelperPath"
@@ -1712,12 +1711,6 @@ function CreateDevEnv {
             Write-Host "Repository is empty"
         }
 
-        if ($kind -eq "local" -and $settings.type -eq "AppSource App" ) {
-            if ($licenseFileUrl -eq "") {
-                OutputWarning -message "When building an AppSource App, you should create a secret called LicenseFileUrl, containing a secure URL to your license file with permission to the objects used in the app."
-            }
-        }
-
         $installApps = $settings.installApps
         $installTestApps = $settings.installTestApps
 
@@ -2208,9 +2201,28 @@ function DetermineArtifactUrl {
         $version = $segments[2]
         $country = $segments[3]; if ($country -eq "") { $country = $projectSettings.country }
         $select = $segments[4]; if ($select -eq "") { $select = "latest" }
-        $artifactUrl = Get-BCArtifactUrl -storageAccount $storageAccount -type $artifactType -version $version -country $country -select $select -accept_insiderEula | Select-Object -First 1
-        if (-not $artifactUrl) {
-            throw "No artifacts found for the artifact setting ($artifact) in $ALGoSettingsFile"
+        if ($version -eq '*') {
+            $version = "$(([Version]$projectSettings.applicationDependency).Major).$(([Version]$projectSettings.applicationDependency).Minor)"
+            $allArtifactUrls = @(Get-BCArtifactUrl -storageAccount $storageAccount -type $artifactType -version $version -country $country -select all -accept_insiderEula | Where-Object { [Version]$_.Split('/')[4] -ge [Version]$projectSettings.applicationDependency })
+            if ($select -eq 'latest') {
+                $artifactUrl = $allArtifactUrls | Select-Object -Last 1
+            }
+            elseif ($select -eq 'first') {
+                $artifactUrl = $allArtifactUrls | Select-Object -First 1
+            }
+            else {
+                throw "Invalid artifact setting ($artifact) in $ALGoSettingsFile. Version can only be '*' if select is first or latest."
+            }
+            Write-Host "Found $($allArtifactUrls.Count) artifacts for version $version matching application dependency $($projectSettings.applicationDependency), selecting $select."
+            if (-not $artifactUrl) {
+                throw "No artifacts found for the artifact setting ($artifact) in $ALGoSettingsFile, when application dependency is $($projectSettings.applicationDependency)"
+            }
+        }
+        else {
+            $artifactUrl = Get-BCArtifactUrl -storageAccount $storageAccount -type $artifactType -version $version -country $country -select $select -accept_insiderEula | Select-Object -First 1
+            if (-not $artifactUrl) {
+                throw "No artifacts found for the artifact setting ($artifact) in $ALGoSettingsFile"
+            }
         }
         $version = $artifactUrl.Split('/')[4]
         $storageAccount = $artifactUrl.Split('/')[2]

Actions/RunPipeline/RunPipeline.ps1

@@ -124,12 +124,6 @@ try {
         exit
     }
 
-    if ($settings.type -eq "AppSource App" ) {
-        if ($licenseFileUrl -eq "") {
-            OutputWarning -message "When building an AppSource App, you should create a secret called LicenseFileUrl, containing a secure URL to your license file with permission to the objects used in the app."
-        }
-    }
-
     $installApps = $settings.installApps
     $installTestApps = $settings.installTestApps
 

RELEASENOTES.md

@@ -13,10 +13,18 @@ Note that when using the preview version of AL-Go for GitHub, we recommend you U
 - Issue 997 'Deliver to AppSource' action fails for projects containing a space
 - Issue 987 Resource not accessible by integration when creating release from specific version
 - Issue 979 Publish to AppSource Documentation
+- Issue 1018 Artifact setting - possiblity to read version from app.json
 - Issue 1008 Allow PullRequestHandler to use ubuntu or self hosted runners for all jobs except for pregateCheck
 - Issue 962 Finer control of "shell"-property
 
+### Better artifact selection
+
+The artifact setting in your project settings file can now contain a `*` instead of the version number. This means that AL-Go for GitHub will determine the application dependency for your projects together with the `applicationDependency` setting and determine which Business Central version is needed for the project.
+- `"artifact": "//*//latest"` will give you the latest Business Central version, higher than your application dependency and with the same major.minor as your application dependency.
+- `"artifact": "//*//first"` will give you the first Business Central version, higher than your application dependency and with the same major.minor as your application dependency.
+
 ### New Settings
+
 - `deliverToAppSource`: a JSON object containing the following properties
   - **productId** must be the product Id from partner Center.
   - **mainAppFolder** specifies the appFolder of the main app if you have multiple apps in the same project.
@@ -25,6 +33,7 @@ Note that when using the preview version of AL-Go for GitHub, we recommend you U
 - Add `shell` as a property under `DeployTo` structure
 
 ### Deprecated Settings
+
 - `appSourceContinuousDelivery` is moved to the `deliverToAppSource` structure
 - `appSourceMainAppFolder` is moved to the `deliverToAppSource` structure
 - `appSourceProductId` is moved to the `deliverToAppSource` structure

Scenarios/SetupCiCdForExistingAppSourceApp.md

@@ -7,7 +7,7 @@
 ![Create Zip Url](https://github.com/microsoft/AL-Go/assets/10775043/fa287e2e-d2e9-4e62-a5e8-641a8e2d4ab3)
 1. Back on github.com, under **Actions**, select the **Add existing app or test app** workflow and choose **Run workflow**. Paste in the **Secure Download URL** and choose **Run Workflow**. When the workflow finishes, complete the pull request created.
 1. A CI workflow is kicked off by the pull request, this will fail with this error: *For AppSource Apps with AppSourceCop enabled, you need to specify AppSourceCopMandatoryAffixes in .AL-Go\settings.json.*
-1. If you fix this and re-run, you will get a warning: *When building an AppSource App, you should create a secret called LicenseFileUrl, containing a secure URL to your license file with permission to the objects used in the app*. If you are building your AppSource app for Business Central versions prior to 22, the license file is a requirement. In 22, the CRONUS license has sufficient rights to be used as a DevOps license.
+1. Note that if you are building your AppSource app for Business Central versions prior to 22, you also need to create a secret called LicenseFileUrl, with permissions to your objects. for version 22 and later, the CRONUS license has sufficient rights to be used as a DevOps license.
 1. I will use my **KeyVault from [Scenario 7](UseAzureKeyVault.md)**, by adding a secret called **AZURE_CREDENTIALS** to my GitHub repo. And then add or modify the following 3 properties in the **.AL-Go\settings.json** file:
 ```json
 "LicenseFileUrlSecretName": "LicenseFile",

Scenarios/UseAzureKeyVault.md

@@ -1,5 +1,5 @@
 # #7 Use Azure KeyVault for secrets with AL-Go
-*Prerequisites: A completed [scenario 6](UpdateAlGoSystemFiles.md), an Azure KeyVault and you will need to follow the guidelines on how to connect to an Azure KeyVault as specified [here](https://go.microsoft.com/fwlink/?linkid=2217417&clcid=0x409). Add your KeyVault name to the the JSON construct from this walkthrough (using **“keyVaultName” : “{your keyvault name}”**) and add this JSON construct as a repository secret called AZURE_CREDENTIALS. You can also specify the KeyVault name in the AL-Go settings file if you do not wait to mess with the JSON construct.*
+*Prerequisites: A completed [scenario 6](UpdateAlGoSystemFiles.md), an Azure KeyVault and you will need to follow the guidelines on how to connect to an Azure KeyVault as specified [here](https://learn.microsoft.com/azure/developer/github/connect-from-azure?tabs=azure-portal%2Cwindows#use-the-azure-login-action-with-a-service-principal-secret). Add your KeyVault name to the the JSON construct from this walkthrough (using **“keyVaultName” : “{your keyvault name}”**) and add this JSON construct as a repository secret called AZURE_CREDENTIALS. You can also specify the KeyVault name in the AL-Go settings file if you do not wait to mess with the JSON construct.*
 
 *If you need to use Hardware Security Modules, you'll need to use a Premium SKU key vault. For more information on this, see [learn.microsoft.com](https://learn.microsoft.com/en-us/azure/key-vault/keys/about-keys)*
 

Scenarios/settings.md

@@ -72,7 +72,7 @@ The repository settings are only read from the repository settings file (.github
 
 | Name | Description | Default value |
 | :-- | :-- | :-- |
-| <a id="artifact"></a>artifact | Determines the artifacts used for building and testing the app.<br />This setting can either be an absolute pointer to Business Central artifacts (https://... - rarely used) or it can be a search specification for artifacts (\<storageaccount\>/\<type\>/\<version\>/\<country\>/\<select\>).<br />If not specified, the artifacts used will be the latest sandbox artifacts from the country specified in the country setting. | |
+| <a id="artifact"></a>artifact | Determines the artifacts used for building and testing the app.<br />This setting can either be an absolute pointer to Business Central artifacts (https://... - rarely used) or it can be a search specification for artifacts (\<storageaccount\>/\<type\>/\<version\>/\<country\>/\<select\>).<br />If not specified, the artifacts used will be the latest sandbox artifacts from the country specified in the country setting.<br />**Note:** if version is set to `*`, then the application dependency from the apps in your project will determine which artifacts to use. If select is *first*, then you will get the first artifacts matching your application dependency. If select is *latest* then you will get the latest artifacts with the same major.minor as your application dependency. | |
 | <a id="updateDependencies"></a>updateDependencies | Setting updateDependencies to true causes AL-Go to build your app against the first compatible Business Central build and set the dependency version numbers in the app.json accordingly during build. All version numbers in the built app will be set to the version number used during compilation. | false |
 | <a id="generateDependencyArtifact"></a>generateDependencyArtifact | When this repository setting is true, CI/CD pipeline generates an artifact with the external dependencies used for building the apps in this repo. | false |
 | <a id="companyName"></a>companyName | Company name selected in the database, used for running the CI/CD workflow. Default is to use the default company in the selected Business Central localization. | |
@@ -95,7 +95,7 @@ The repository settings are only read from the repository settings file (.github
 | <a id="rulesetFile"></a>rulesetFile | Filename of the custom ruleset file | |
 | <a id="enableExternalRulesets"></a>enableExternalRulesets | If enableExternalRulesets is set to true, then you can have external rule references in the ruleset | false |
 | <a id="vsixFile"></a>vsixFile | Determines which version of the AL Language Extension to use for building the apps. This can be:<br />**default** to use the AL Language Extension which ships with the Business Central version you are building for<br />**latest** to always download the latest AL Language Extension from the marketplace<br />**preview** to always download the preview AL Language Extension from the marketplace.<br/>or a **direct download URL** pointing to the AL Language VSIX file to use for building the apps.<br />By default, AL-Go uses the AL Language extension, which is shipped with the artifacts used for the build. | default |
-| <a id="codeSignCertificateUrlSecretName"></a>codeSignCertificateUrlSecretName<br />codeSignCertificatePasswordSecretName | When developing AppSource Apps, your app needs to be code signed and you need to add secrets to GitHub secrets or Azure KeyVault, specifying the secure URL from which your codesigning certificate pfx file can be downloaded and the password for this certificate. These settings specifies the names (**NOT the secrets**) of the code signing certificate url and password. Default is to look for secrets called CodeSignCertificateUrl and CodeSignCertificatePassword. Read [this](SetupCiCdForExistingAppSourceApp.md) for more information. | CodeSignCertificateUrl<br />CodeSignCertificatePassword |
+| <a id="codeSignCertificateUrlSecretName"></a>codeSignCertificateUrlSecretName<br />codeSignCertificatePasswordSecretName | **Note** there is a new way of signing apps, which is described [here](../Scenarios/Codesigning.md).<br />Using the old mechanism, you need a certificate .pfx file and password and you need to add secrets to GitHub secrets or Azure KeyVault, specifying the secure URL from which your codesigning certificate pfx file can be downloaded and the password for this certificate. These settings specifies the names (**NOT the secrets**) of the code signing certificate url and password. Default is to look for secrets called CodeSignCertificateUrl and CodeSignCertificatePassword. Read [this](SetupCiCdForExistingAppSourceApp.md) for more information. | CodeSignCertificateUrl<br />CodeSignCertificatePassword |
 | <a id="keyVaultCodesignCertificateName"></a>keyVaultCodesignCertificateName | Name of a certificate stored in your KeyVault that can be used to codesigning. To use this setting you will also need enable Azure KeyVault in your AL-GO project. Read [this](UseAzureKeyVault.md) for more information | |
 | <a id="applicationInsightsConnectionStringSecretName"></a>applicationInsightsConnectionStringSecretName | This setting specifies the name (**NOT the secret**) of a secret containing the application insights connection string for the apps. | applicationInsightsConnectionString |
 | <a id="storageContextSecretName"></a>storageContextSecretName | This setting specifies the name (**NOT the secret**) of a secret containing a json string with StorageAccountName, ContainerName, BlobName and StorageAccountKey or SAS Token. If this secret exists, AL-Go will upload builds to this storage account for every successful build.<br />The BcContainerHelper function New-ALGoStorageContext can create a .json structure with this content. | StorageContext |
@@ -106,7 +106,7 @@ The repository settings are only read from the repository settings file (.github
 | <a id="cacheKeepDays"></a>cacheKeepDays | When using self-hosted runners, cacheKeepDays specifies the number of days docker image are cached before cleaned up when running the next pipeline.<br />Note that setting cacheKeepDays to 0 will flush the cache before every build and will cause all other running builds using agents on the same host to fail. | 3 |
 | <a id="assignPremiumPlan"></a>assignPremiumPlan | Setting assignPremiumPlan to true in your project setting file, causes the build container to be created with the AssignPremiumPlan set. This causes the auto-created user to have Premium Plan enabled. This setting is needed if your tests require premium plan enabled. | false |
 | <a id="enableTaskScheduler"></a>enableTaskScheduler | Setting enableTaskScheduler to true in your project setting file, causes the build container to be created with the Task Scheduler running. | false |
-| <a id="useCompilerFolder"></a>useCompilerFolder | Setting useCompilerFolder to true causes your pipelines to use containerless compiling. Unless you also set **doNotPublishApps** to true, setting useCompilerFolder to true won't give you any performance advantage, since AL-Go for GitHub will still need to create a container in order to publish and test the apps. In the future, publishing and testing will be split from building and there will be other options for getting an instance of Business Central for publishing and testing. | false |
+| <a id="useCompilerFolder"></a>useCompilerFolder | Setting useCompilerFolder to true causes your pipelines to use containerless compiling. Unless you also set **doNotPublishApps** to true, setting useCompilerFolder to true won't give you any performance advantage, since AL-Go for GitHub will still need to create a container in order to publish and test the apps. In the future, publishing and testing will be split from building and there will be other options for getting an instance of Business Central for publishing and testing. **Note** when using UseCompilerFolder you need to sign apps using the new signing mechanism described [here](../Scenarios/Codesigning.md). | false |
 | <a id="excludeEnvironments"></a>excludeEnvironments | excludeEnvironments can be an array of GitHub Environments, which should be excluded from the list of environments considered for deployment. github-pages is automatically added to this array and cannot be used as environment for deployment of AL-Go for GitHub projects. | [ ] |
 
 ## AppSource specific advanced settings

Templates/AppSource App/.AL-Go/localDevEnv.ps1

@@ -5,6 +5,7 @@
 #
 Param(
     [string] $containerName = "",
+    [ValidateSet("UserPassword", "Windows")]
     [string] $auth = "",
     [pscredential] $credential = $null,
     [string] $licenseFileUrl = "",

Templates/AppSource App/.github/workflows/CICD.yaml

@@ -211,6 +211,9 @@ jobs:
     strategy: ${{ fromJson(needs.Initialization.outputs.environmentsMatrixJson) }}
     runs-on: ${{ fromJson(matrix.os) }}
     name: Deploy to ${{ matrix.environment }}
+    defaults:
+      run:
+        shell: ${{ matrix.shell }}
     environment:
       name: ${{ matrix.environment }}
       url: ${{ steps.Deploy.outputs.environmentUrl }}
@@ -230,7 +233,6 @@ jobs:
 
       - name: EnvName
         id: envName
-        shell: ${{ matrix.shell }}
         run: |
           $errorActionPreference = "Stop"; $ProgressPreference = "SilentlyContinue"; Set-StrictMode -Version 2.0
           $envName = '${{ matrix.environment }}'.split(' ')[0]

Templates/AppSource App/.github/workflows/PublishToEnvironment.yaml

@@ -121,6 +121,9 @@ jobs:
     strategy: ${{ fromJson(needs.Initialization.outputs.environmentsMatrixJson) }}
     runs-on: ${{ fromJson(matrix.os) }}
     name: Deploy to ${{ matrix.environment }}
+    defaults:
+      run:
+        shell: ${{ matrix.shell }}
     environment:
       name: ${{ matrix.environment }}
       url: ${{ steps.Deploy.outputs.environmentUrl }}
@@ -132,7 +135,6 @@ jobs:
 
       - name: EnvName
         id: envName
-        shell: ${{ matrix.shell }}
         run: |
           $errorActionPreference = "Stop"; $ProgressPreference = "SilentlyContinue"; Set-StrictMode -Version 2.0
           $envName = '${{ matrix.environment }}'.split(' ')[0]

Templates/Per Tenant Extension/.AL-Go/localDevEnv.ps1

@@ -5,6 +5,7 @@
 #
 Param(
     [string] $containerName = "",
+    [ValidateSet("UserPassword", "Windows")]
     [string] $auth = "",
     [pscredential] $credential = $null,
     [string] $licenseFileUrl = "",