github · github-actions · Oct 3, 2024 · Oct 3, 2024 · Oct 3, 2024 · Oct 3, 2024
diff --git a/.github/workflows/upgrade_codeql_dependencies.yml b/.github/workflows/upgrade_codeql_dependencies.yml
@@ -20,6 +20,16 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v2
 
+      - name: Fetch CodeQL
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+          RUNNER_TEMP: ${{ runner.temp }}
+        run: |
+          cd $RUNNER_TEMP
+          gh release download "v${CODEQL_CLI_VERSION}" --repo https://github.com/github/codeql-cli-binaries --pattern codeql-linux64.zip
+          unzip -q codeql-linux64.zip
+          echo "$RUNNER_TEMP/codeql/" >> $GITHUB_PATH
+
       - name: Install Python
         uses: actions/setup-python@v4
         with:
@@ -35,27 +45,27 @@ jobs:
         run: |
           python3 scripts/upgrade-codeql-dependencies/upgrade-codeql-dependencies.py --cli-version "$CODEQL_CLI_VERSION"
 
-      - name: Fetch CodeQL
-        env:
-          GITHUB_TOKEN: ${{ github.token }}
-          RUNNER_TEMP: ${{ runner.temp }}
-        run: |
-          cd $RUNNER_TEMP
-          gh release download "v${CODEQL_CLI_VERSION}" --repo https://github.com/github/codeql-cli-binaries --pattern codeql-linux64.zip
-          unzip -q codeql-linux64.zip
-
       - name: Update CodeQL formatting based on new CLI version
         env:
           RUNNER_TEMP: ${{ runner.temp }}
         run: |
-          find cpp \( -name '*.ql' -or -name '*.qll' \) -print0 | xargs -0 --max-procs "$XARGS_MAX_PROCS" $RUNNER_TEMP/codeql/codeql query format --in-place
-          find c \( -name '*.ql' -or -name '*.qll' \) -print0 | xargs -0 --max-procs "$XARGS_MAX_PROCS" $RUNNER_TEMP/codeql/codeql query format --in-place
+          find cpp \( -name '*.ql' -or -name '*.qll' \) -print0 | xargs -0 --max-procs "$XARGS_MAX_PROCS" codeql query format --in-place
+          find c \( -name '*.ql' -or -name '*.qll' \) -print0 | xargs -0 --max-procs "$XARGS_MAX_PROCS" codeql query format --in-place
 
       - name: Create Pull Request
         uses: peter-evans/create-pull-request@v3
         with:
-          title: "Upgrading `github/codeql` dependency to ${{ github.event.inputs.codeql_cli_version }}"
-          body: "This PR upgrades the CodeQL CLI version to ${{ github.event.inputs.codeql_cli_version }}."
+          title: "Upgrade `github/codeql` dependency to ${{ github.event.inputs.codeql_cli_version }}"
+          body: |
+            This PR upgrades the CodeQL CLI version to ${{ github.event.inputs.codeql_cli_version }}.
+
+            ## CodeQL dependency upgrade checklist:
+
+            - [ ] Confirm the code has been correctly reformatted according to the new CodeQL CLI.
+            - [ ] Identify any CodeQL compiler warnings and errors, and update queries as required.
+            - [ ] Validate that the `github/codeql` test cases succeed.
+            - [ ] Address any CodeQL test failures in the `github/codeql-coding-standards` repository.
+            - [ ] Validate performance vs pre-upgrade, using /test-performance
           commit-message: "Upgrading `github/codeql` dependency to ${{ github.event.inputs.codeql_cli_version }}"
           delete-branch: true
           branch: "codeql/upgrade-to-${{ github.event.inputs.codeql_cli_version }}"
diff --git a/c/cert/src/codeql-pack.lock.yml b/c/cert/src/codeql-pack.lock.yml
@@ -2,13 +2,17 @@
 lockVersion: 1.0.0
 dependencies:
   codeql/cpp-all:
-    version: 0.9.3
+    version: 0.12.2
   codeql/dataflow:
+    version: 0.1.5
+  codeql/rangeanalysis:
     version: 0.0.4
   codeql/ssa:
-    version: 0.1.5
+    version: 0.2.5
   codeql/tutorial:
-    version: 0.1.5
+    version: 0.2.5
+  codeql/typetracking:
+    version: 0.2.5
   codeql/util:
-    version: 0.1.5
+    version: 0.2.5
 compiled: false
diff --git a/c/cert/src/qlpack.yml b/c/cert/src/qlpack.yml
@@ -5,4 +5,4 @@ suites: codeql-suites
 license: MIT
 dependencies:
   codeql/common-c-coding-standards: '*'
-  codeql/cpp-all: 0.9.3
+  codeql/cpp-all: 0.12.2
diff --git a/c/common/src/codeql-pack.lock.yml b/c/common/src/codeql-pack.lock.yml
@@ -2,13 +2,17 @@
 lockVersion: 1.0.0
 dependencies:
   codeql/cpp-all:
-    version: 0.9.3
+    version: 0.12.2
   codeql/dataflow:
+    version: 0.1.5
+  codeql/rangeanalysis:
     version: 0.0.4
   codeql/ssa:
-    version: 0.1.5
+    version: 0.2.5
   codeql/tutorial:
-    version: 0.1.5
+    version: 0.2.5
+  codeql/typetracking:
+    version: 0.2.5
   codeql/util:
-    version: 0.1.5
+    version: 0.2.5
 compiled: false
diff --git a/c/common/src/qlpack.yml b/c/common/src/qlpack.yml
@@ -3,4 +3,4 @@ version: 2.36.0-dev
 license: MIT
 dependencies:
   codeql/common-cpp-coding-standards: '*'
-  codeql/cpp-all: 0.9.3
+  codeql/cpp-all: 0.12.2
diff --git a/c/misra/src/codeql-pack.lock.yml b/c/misra/src/codeql-pack.lock.yml
@@ -2,13 +2,17 @@
 lockVersion: 1.0.0
 dependencies:
   codeql/cpp-all:
-    version: 0.9.3
+    version: 0.12.2
   codeql/dataflow:
+    version: 0.1.5
+  codeql/rangeanalysis:
     version: 0.0.4
   codeql/ssa:
-    version: 0.1.5
+    version: 0.2.5
   codeql/tutorial:
-    version: 0.1.5
+    version: 0.2.5
+  codeql/typetracking:
+    version: 0.2.5
   codeql/util:
-    version: 0.1.5
+    version: 0.2.5
 compiled: false
diff --git a/c/misra/src/qlpack.yml b/c/misra/src/qlpack.yml
@@ -5,4 +5,4 @@ suites: codeql-suites
 license: MIT
 dependencies:
   codeql/common-c-coding-standards: '*'
-  codeql/cpp-all: 0.9.3
+  codeql/cpp-all: 0.12.2
diff --git a/cpp/autosar/src/codeql-pack.lock.yml b/cpp/autosar/src/codeql-pack.lock.yml
@@ -2,13 +2,17 @@
 lockVersion: 1.0.0
 dependencies:
   codeql/cpp-all:
-    version: 0.9.3
+    version: 0.12.2
   codeql/dataflow:
+    version: 0.1.5
+  codeql/rangeanalysis:
     version: 0.0.4
   codeql/ssa:
-    version: 0.1.5
+    version: 0.2.5
   codeql/tutorial:
-    version: 0.1.5
+    version: 0.2.5
+  codeql/typetracking:
+    version: 0.2.5
   codeql/util:
-    version: 0.1.5
+    version: 0.2.5
 compiled: false
diff --git a/cpp/autosar/src/qlpack.yml b/cpp/autosar/src/qlpack.yml
@@ -5,4 +5,4 @@ suites: codeql-suites
 license: MIT
 dependencies:
   codeql/common-cpp-coding-standards: '*'
-  codeql/cpp-all: 0.9.3
+  codeql/cpp-all: 0.12.2
diff --git a/cpp/cert/src/codeql-pack.lock.yml b/cpp/cert/src/codeql-pack.lock.yml
@@ -2,13 +2,17 @@
 lockVersion: 1.0.0
 dependencies:
   codeql/cpp-all:
-    version: 0.9.3
+    version: 0.12.2
   codeql/dataflow:
+    version: 0.1.5
+  codeql/rangeanalysis:
     version: 0.0.4
   codeql/ssa:
-    version: 0.1.5
+    version: 0.2.5
   codeql/tutorial:
-    version: 0.1.5
+    version: 0.2.5
+  codeql/typetracking:
+    version: 0.2.5
   codeql/util:
-    version: 0.1.5
+    version: 0.2.5
 compiled: false
diff --git a/cpp/cert/src/qlpack.yml b/cpp/cert/src/qlpack.yml
@@ -4,5 +4,5 @@ description: CERT C++ 2016
 suites: codeql-suites
 license: MIT
 dependencies:
-  codeql/cpp-all: 0.9.3
+  codeql/cpp-all: 0.12.2
   codeql/common-cpp-coding-standards: '*'
diff --git a/cpp/common/src/codeql-pack.lock.yml b/cpp/common/src/codeql-pack.lock.yml
@@ -2,13 +2,17 @@
 lockVersion: 1.0.0
 dependencies:
   codeql/cpp-all:
-    version: 0.9.3
+    version: 0.12.2
   codeql/dataflow:
+    version: 0.1.5
+  codeql/rangeanalysis:
     version: 0.0.4
   codeql/ssa:
-    version: 0.1.5
+    version: 0.2.5
   codeql/tutorial:
-    version: 0.1.5
+    version: 0.2.5
+  codeql/typetracking:
+    version: 0.2.5
   codeql/util:
-    version: 0.1.5
+    version: 0.2.5
 compiled: false
diff --git a/cpp/common/src/qlpack.yml b/cpp/common/src/qlpack.yml
@@ -2,6 +2,6 @@ name: codeql/common-cpp-coding-standards
 version: 2.36.0-dev
 license: MIT
 dependencies:
-  codeql/cpp-all: 0.9.3
+  codeql/cpp-all: 0.12.2
 dataExtensions:
-  - ext/*.model.yml
+- ext/*.model.yml
diff --git a/cpp/misra/src/codeql-pack.lock.yml b/cpp/misra/src/codeql-pack.lock.yml
@@ -2,13 +2,17 @@
 lockVersion: 1.0.0
 dependencies:
   codeql/cpp-all:
-    version: 0.9.3
+    version: 0.12.2
   codeql/dataflow:
+    version: 0.1.5
+  codeql/rangeanalysis:
     version: 0.0.4
   codeql/ssa:
-    version: 0.1.5
+    version: 0.2.5
   codeql/tutorial:
-    version: 0.1.5
+    version: 0.2.5
+  codeql/typetracking:
+    version: 0.2.5
   codeql/util:
-    version: 0.1.5
+    version: 0.2.5
 compiled: false
diff --git a/cpp/misra/src/qlpack.yml b/cpp/misra/src/qlpack.yml
@@ -5,4 +5,4 @@ suites: codeql-suites
 license: MIT
 dependencies:
   codeql/common-cpp-coding-standards: '*'
-  codeql/cpp-all: 0.9.3
+  codeql/cpp-all: 0.12.2
diff --git a/cpp/report/src/codeql-pack.lock.yml b/cpp/report/src/codeql-pack.lock.yml
@@ -2,13 +2,17 @@
 lockVersion: 1.0.0
 dependencies:
   codeql/cpp-all:
-    version: 0.9.3
+    version: 0.12.2
   codeql/dataflow:
+    version: 0.1.5
+  codeql/rangeanalysis:
     version: 0.0.4
   codeql/ssa:
-    version: 0.1.5
+    version: 0.2.5
   codeql/tutorial:
-    version: 0.1.5
+    version: 0.2.5
+  codeql/typetracking:
+    version: 0.2.5
   codeql/util:
-    version: 0.1.5
+    version: 0.2.5
 compiled: false
diff --git a/cpp/report/src/qlpack.yml b/cpp/report/src/qlpack.yml
@@ -2,4 +2,4 @@ name: codeql/report-cpp-coding-standards
 version: 2.36.0-dev
 license: MIT
 dependencies:
-  codeql/cpp-all: 0.9.3
+  codeql/cpp-all: 0.12.2
diff --git a/docs/development_handbook.md b/docs/development_handbook.md
@@ -496,46 +496,53 @@ There are two external dependencies required for running the coding standards qu
 
 For the purpose of this repository, and any tool qualification, we consider these external dependencies to be "black boxes" which require verification when upgrading.
 
-To (a) clearly specify the supported versions of these external dependencies and to (b) enable automation around them, the repository contains a `supported_codeql_configs.json` which lists the sets of supported configurations. There are four fields:
+To (a) clearly specify the supported versions of these external dependencies and to (b) enable automation around them, the repository contains a `supported_codeql_configs.json` which lists the sets of supported configurations under the `supported_environments` property. There are three fields:
 
 - `codeql_cli` - this is the plain version number of the supported CodeQL CLI, e.g. `2.6.3`.
 - `codeql_standard_library` - this is the name of a tag on the `github.com/github/codeql` repository. The tag should be compatible with the CodeQL CLI given above. This would typically use the `codeql-cli/v<version-number>` tag for the release, although any tag which is compatible is allowed.
 - `codeql_cli_bundle` - (optional) - if present, describes the CodeQL CLI bundle version that is compatible. The bundle should include precisely the CodeQL CLI version and CodeQL Standard Library versions specified in the two mandatory fields.
-- `ghes` - (optional) - if present describes the GitHub Enterprise Server release whose integrated copy of the CodeQL Action points to the CodeQL CLI bundle specified in the `codeql_cli_bundle` field.
 
 #### Upgrading external dependencies
 
 To upgrade the CodeQL external dependencies:
 
 1. Determine appropriate versions of the CodeQL CLI and `github/codeql` repository, according to the release schedule and customer demands.
 2. Determine if there is a compatible CodeQL CLI bundle version by looking at the releases specified at [CodeQL Action releases](https://github.com/github/codeql-action/releases). The bundle always includes the standard library at the version specified by the `codeql-cli/v<version-number>` tag in the `github/codeql` repository.
-3. If you find a compatible CodeQL CLI bundle, determine whether that bundle was released in a GitHub Enterprise server release, by inspecting the `defaults.json` file at https://github.com/github/codeql-action/blob/main/lib/defaults.json#L2 for the CodeQL Action submitted with
-4. Populated the `supported_codeql_configs.json` file with the given values, ensuring to delete the optional fields if they are not populated.
-5. Submit a Pull Request to the `github/codeql-coding-standards` repository with the title `Upgrade `github/codeql` dependency to <insert codeql_standard_library value>`. Use this template for the description, filling :
 
-    ```md
-    This PR updates the `supported_codeql_configs.json` file to target:
+If all components are being upgraded to a consistent veresion (e.g. CodeQL CLI v2.15.5, with `github/codeql` tag `codeql-cli/v2.15.5` and bundle `codeql-cli-bundle-v2.15.5`) then the following process can be used:
+
+1. Run the [upgrade_codeql_dependencies.yml](./github/workflows/upgrade_codeql_dependencies.yml) workflow, with the plain version number, e.g. `2.15.5`. This will:
+  - Download the specified version of the CodeQL CLI
+  - Run the [upgrade-codeql-dependencies.py](scripts/release/upgrade-codeql-dependencies.py) script, which
+    - Validates the version selected exists in all relevant places
+    - Updates the `supported_codeql_configs.json` file.
+    - Updates each `qlpack.yml` in the repository with an appropriate value for the `codeql/cpp-all` pack, consistent with the selected CodeQL CLI version.
+    - Updates each `codeql-lock.yml` file to upgrade to the new version.
+2. Follow the dependency upgrade checklist, confirming each step. The `.github/workflows/standard_library_upgrade_tests.yml` will trigger automation for running the `github/codeql` unit tests with the appropriate CLI version.
+3. Once all the automate tests have passed, and the checklist is complete, the PR can be merged.
+4. An internal notification should be shared with the development team.
 
-    - CodeQL CLI <codeql_cli>
-    - CodeQL Standard Library <codeql_standard_library>
-    - GHES <ghes>
-    - CodeQL CLI Bundle <date_of_bundle>
+If the upgrade is of mismatched versions you will need to manually create the upgrade following this process:
 
-    <EITHER:This should match the versions of CodeQL deployed with GitHub Enterprise Server <ghes>>
-    <OR: This does not match any released version of GitHub Enterprise Server.>
+1. Populate the `supported_codeql_configs.json` file with the given values, ensuring to delete the optional fields if they are not populated.
+2. Submit a Pull Request to the `github/codeql-coding-standards` repository with the title `Upgrade `github/codeql` dependency to <insert codeql_standard_library value>`. Use this template for the description, filling:
+
+    ```md
+    This PR updates the `supported_codeql_configs.json` file to target CodeQL CLI <codeql_cli>.
 
     ## CodeQL dependency upgrade checklist:
 
-    - [ ] Reformat our CodeQL using the latest version (if required)
+    - [ ] Confirm the code has been correctly reformatted according to the new CodeQL CLI.
     - [ ] Identify any CodeQL compiler warnings and errors, and update queries as required.
     - [ ] Validate that the `github/codeql` test cases succeed.
     - [ ] Address any CodeQL test failures in the `github/codeql-coding-standards` repository.
-    - [ ] Validate performance vs pre-upgrade
+    - [ ] Validate performance vs pre-upgrade, using /test-performance
     ```
 
-6. Follow the dependency upgrade checklist, confirming each step. The `.github/workflows/standard_library_upgrade_tests.yml` will trigger automation for running the `github/codeql` unit tests with the appropriate CLI version.
-7. Once all the automate tests have passed, and the checklist is complete, the PR can be merged.
-8. An internal notification should be shared with the development team.
+3. Follow the dependency upgrade checklist, confirming each step. The `.github/workflows/standard_library_upgrade_tests.yml` will trigger automation for running the `github/codeql` unit tests with the appropriate CLI version.
+4. Once all the automate tests have passed, and the checklist is complete, the PR can be merged.
+5. An internal notification should be shared with the development team.
+
 
 ### Release process
 

diff --git a/scripts/generate_modules/queries/codeql-pack.lock.yml b/scripts/generate_modules/queries/codeql-pack.lock.yml
@@ -2,13 +2,17 @@
 lockVersion: 1.0.0
 dependencies:
   codeql/cpp-all:
-    version: 0.9.3
+    version: 0.12.2
   codeql/dataflow:
+    version: 0.1.5
+  codeql/rangeanalysis:
     version: 0.0.4
   codeql/ssa:
-    version: 0.1.5
+    version: 0.2.5
   codeql/tutorial:
-    version: 0.1.5
+    version: 0.2.5
+  codeql/typetracking:
+    version: 0.2.5
   codeql/util:
-    version: 0.1.5
+    version: 0.2.5
 compiled: false
diff --git a/scripts/generate_modules/queries/qlpack.yml b/scripts/generate_modules/queries/qlpack.yml
@@ -2,4 +2,4 @@ name: codeql/standard-library-extraction-cpp-coding-standards
 version: 0.0.0
 license: MIT
 dependencies:
-  codeql/cpp-all: 0.9.3
+  codeql/cpp-all: 0.12.2
diff --git a/scripts/upgrade-codeql-dependencies/requirements.txt b/scripts/upgrade-codeql-dependencies/requirements.txt
@@ -4,3 +4,4 @@ idna==3.4
 requests==2.31.0
 semantic-version==2.10.0
 urllib3==1.26.18
+pyyaml==6.0.1