Compare commits

...

88 Commits

Author SHA1 Message Date
Henry Mercer
ce729e4d35 Merge pull request #3315 from github/henrymercer/dead-code-elimination
Delete unused exports
2025-11-19 15:24:22 +00:00
Henry Mercer
ac359aad20 Add return type 2025-11-19 14:59:16 +00:00
Henry Mercer
112cd075bd Merge branch 'main' into henrymercer/dead-code-elimination 2025-11-19 14:56:28 +00:00
Michael B. Gale
0b4317954f Merge pull request #3306 from github/dependabot/npm_and_yarn/types/sinon-21.0.0
Bump @types/sinon from 17.0.4 to 21.0.0
2025-11-19 14:13:16 +00:00
Michael B. Gale
e818008b54 Merge pull request #3305 from github/dependabot/npm_and_yarn/eslint/compat-2.0.0
Bump @eslint/compat from 1.4.1 to 2.0.0
2025-11-19 13:41:43 +00:00
Michael B. Gale
90871e185b Merge pull request #3304 from github/dependabot/npm_and_yarn/npm-minor-7439af33e4
Bump the npm-minor group with 2 updates
2025-11-19 13:18:38 +00:00
Kasper Svendsen
a102014397 Merge pull request #3317 from github/kaspersv/bump-minimum-overlay-version
Overlay: Increase minimum CLI version required for overlay analysis
2025-11-19 14:18:24 +01:00
Kasper Svendsen
de74d762a3 Overlay: Increase minimum CLI version 2025-11-19 13:04:23 +01:00
Kasper Svendsen
ce07e7d196 Merge pull request #3310 from github/kaspersv/overlay-disk-available-limit
Overlay: Fall back to full analysis if runner disk space is low
2025-11-19 12:57:53 +01:00
Henry Mercer
86d2aa55c0 Merge pull request #3316 from github/henrymercer/upload-overlay-to-api
Upload overlay base DBs to GitHub API behind FF
2025-11-19 10:29:28 +00:00
Kasper Svendsen
4eccb3798e Overlay: Round available disk space in MB 2025-11-19 08:40:56 +01:00
Kasper Svendsen
ed80d6e5e9 Overlay: Reorder available disk space check 2025-11-19 07:54:05 +01:00
Henry Mercer
378219ced2 Merge pull request #3313 from github/mergeback/v4.31.4-to-main-e12f0178
Mergeback v4.31.4 refs/heads/releases/v4 into main
2025-11-18 18:46:24 +00:00
Henry Mercer
c649c5993d Upload overlay base DB to API behind FF 2025-11-18 18:43:19 +00:00
Henry Mercer
31042e9879 Rename function calls to make destructive operation clearer 2025-11-18 18:42:15 +00:00
Henry Mercer
5da2098551 Add feature flag for uploading overlay DBs to API 2025-11-18 18:40:51 +00:00
Henry Mercer
cac5926de5 Delete unused exports 2025-11-18 18:16:54 +00:00
Henry Mercer
e24190a70c Remove unused dependencies 2025-11-18 18:14:49 +00:00
github-actions[bot]
ce9b526448 Rebuild 2025-11-18 16:17:35 +00:00
github-actions[bot]
28f4a61417 Merge remote-tracking branch 'origin/main' into mergeback/v4.31.4-to-main-e12f0178 2025-11-18 16:16:46 +00:00
github-actions[bot]
fea250010c Update changelog and version after v4.31.4 2025-11-18 16:14:11 +00:00
Michael B. Gale
e12f017898 Merge pull request #3312 from github/update-v4.31.4-70434f6dd
Merge main into releases/v4
2025-11-18 16:12:25 +00:00
Michael B. Gale
249458aab2 Merge pull request #3296 from github/mbg/dependency-caching/skip-uploads-for-exact-matches
Skip uploading dependency caches if we know they exist
2025-11-18 15:44:06 +00:00
github-actions[bot]
c9cb6f9c13 Update changelog for v4.31.4 2025-11-18 15:18:43 +00:00
Kasper Svendsen
726a2a01b8 Overlay: Increase disk storage threshold to 20GB 2025-11-18 15:37:27 +01:00
Michael B. Gale
70434f6dd2 Merge pull request #3311 from github/mbg/deps/bump-glob
Bump `glob` to at least `11.1.0`
2025-11-18 12:39:21 +00:00
Michael B. Gale
528362a7c1 Bump glob to at least 11.1.0 2025-11-18 12:20:00 +00:00
Michael B. Gale
de12435376 Merge pull request #3308 from github/mbg/pr-template/nov25
Add additional options to PR template and clarify some
2025-11-18 11:52:08 +00:00
Kasper Svendsen
4f746e4a60 Overlay: Fall back to full analysis if runner disk space is low 2025-11-18 08:19:13 +01:00
Michael B. Gale
ffa63f0dac Merge pull request #3307 from github/dependabot/github_actions/dot-github/workflows/actions-minor-761b22fa12
Bump ruby/setup-ruby from 1.267.0 to 1.268.0 in /.github/workflows in the actions-minor group across 1 directory
2025-11-17 18:06:59 +00:00
Michael B. Gale
7bcdb4bc66 Add additional options to PR template and clarify some 2025-11-17 17:48:39 +00:00
Mario Campos
07eae6420a Merge pull request #3303 from github/mario-campos/v3-core-warning
Change v3 deprecation message to warning.
2025-11-17 11:35:30 -06:00
github-actions[bot]
e546fff076 Rebuild 2025-11-17 17:18:36 +00:00
dependabot[bot]
c418a0fc93 Bump ruby/setup-ruby
Bumps the actions-minor group with 1 update in the /.github/workflows directory: [ruby/setup-ruby](https://github.com/ruby/setup-ruby).


Updates `ruby/setup-ruby` from 1.267.0 to 1.268.0
- [Release notes](https://github.com/ruby/setup-ruby/releases)
- [Changelog](https://github.com/ruby/setup-ruby/blob/master/release.rb)
- [Commits](d5126b9b35...8aeb6ff803)

---
updated-dependencies:
- dependency-name: ruby/setup-ruby
  dependency-version: 1.268.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: actions-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-11-17 17:17:07 +00:00
Mario Campos
fc329e3bb5 Revert "Add CHANGELOG.md entry for "v3 deprecation" to warning change."
This reverts commit 023fd08cc9.
2025-11-17 11:08:58 -06:00
github-actions[bot]
b595847fa5 Rebuild 2025-11-17 17:04:50 +00:00
github-actions[bot]
4f39cef4c6 Rebuild 2025-11-17 17:03:39 +00:00
github-actions[bot]
d4a7ccd1f0 Rebuild 2025-11-17 17:03:22 +00:00
dependabot[bot]
cd808e1260 Bump @types/sinon from 17.0.4 to 21.0.0
Bumps [@types/sinon](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/HEAD/types/sinon) from 17.0.4 to 21.0.0.
- [Release notes](https://github.com/DefinitelyTyped/DefinitelyTyped/releases)
- [Commits](https://github.com/DefinitelyTyped/DefinitelyTyped/commits/HEAD/types/sinon)

---
updated-dependencies:
- dependency-name: "@types/sinon"
  dependency-version: 21.0.0
  dependency-type: direct:development
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-11-17 17:02:13 +00:00
dependabot[bot]
01577d4797 Bump @eslint/compat from 1.4.1 to 2.0.0
Bumps [@eslint/compat](https://github.com/eslint/rewrite/tree/HEAD/packages/compat) from 1.4.1 to 2.0.0.
- [Release notes](https://github.com/eslint/rewrite/releases)
- [Changelog](https://github.com/eslint/rewrite/blob/main/packages/compat/CHANGELOG.md)
- [Commits](https://github.com/eslint/rewrite/commits/compat-v2.0.0/packages/compat)

---
updated-dependencies:
- dependency-name: "@eslint/compat"
  dependency-version: 2.0.0
  dependency-type: direct:development
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-11-17 17:01:53 +00:00
dependabot[bot]
3b635815d6 Bump the npm-minor group with 2 updates
Bumps the npm-minor group with 2 updates: [@octokit/request-error](https://github.com/octokit/request-error.js) and [eslint-plugin-jsdoc](https://github.com/gajus/eslint-plugin-jsdoc).


Updates `@octokit/request-error` from 7.0.2 to 7.1.0
- [Release notes](https://github.com/octokit/request-error.js/releases)
- [Commits](https://github.com/octokit/request-error.js/compare/v7.0.2...v7.1.0)

Updates `eslint-plugin-jsdoc` from 61.1.12 to 61.2.1
- [Release notes](https://github.com/gajus/eslint-plugin-jsdoc/releases)
- [Changelog](https://github.com/gajus/eslint-plugin-jsdoc/blob/main/.releaserc)
- [Commits](https://github.com/gajus/eslint-plugin-jsdoc/compare/v61.1.12...v61.2.1)

---
updated-dependencies:
- dependency-name: "@octokit/request-error"
  dependency-version: 7.1.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: npm-minor
- dependency-name: eslint-plugin-jsdoc
  dependency-version: 61.2.1
  dependency-type: direct:development
  update-type: version-update:semver-minor
  dependency-group: npm-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-11-17 17:01:47 +00:00
Mario Campos
023fd08cc9 Add CHANGELOG.md entry for "v3 deprecation" to warning change. 2025-11-17 09:04:58 -06:00
Mario Campos
ed3a01336f Change v3 deprecation message to warning. 2025-11-17 08:59:44 -06:00
Michael B. Gale
c1a2b73420 Merge pull request #3301 from github/dependabot/npm_and_yarn/js-yaml-4.1.1
Bump js-yaml from 4.1.0 to 4.1.1
2025-11-16 17:54:05 +00:00
github-actions[bot]
8c254d05f3 Rebuild 2025-11-15 10:57:22 +00:00
dependabot[bot]
b9620e1249 Bump js-yaml from 4.1.0 to 4.1.1
Bumps [js-yaml](https://github.com/nodeca/js-yaml) from 4.1.0 to 4.1.1.
- [Changelog](https://github.com/nodeca/js-yaml/blob/master/CHANGELOG.md)
- [Commits](https://github.com/nodeca/js-yaml/compare/4.1.0...4.1.1)

---
updated-dependencies:
- dependency-name: js-yaml
  dependency-version: 4.1.1
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-11-15 10:55:57 +00:00
Michael B. Gale
1ed85b4501 Add test coverage for uploadDependencyCaches 2025-11-14 14:30:54 +00:00
Michael B. Gale
51c9af3a3b Don't try to upload cache if we have restored a cache with the same key 2025-11-14 14:30:54 +00:00
Michael B. Gale
594c0cc369 Store restored keys in action state 2025-11-14 14:30:54 +00:00
Michael B. Gale
11889c27fd Return keys of restored caches from downloadDependencyCaches 2025-11-14 14:30:54 +00:00
Kasper Svendsen
85f1517bb4 Merge pull request #3285 from github/kaspersv/remove-overlay-org-restriction
Overlay: Remove repository owner restriction
2025-11-14 08:28:09 +01:00
Michael B. Gale
86b7d4fc36 Merge pull request #3294 from github/mergeback/v4.31.3-to-main-014f16e7
Mergeback v4.31.3 refs/heads/releases/v4 into main
2025-11-13 22:22:18 +00:00
github-actions[bot]
246edb9b1d Rebuild 2025-11-13 21:59:57 +00:00
github-actions[bot]
497c7f627a Update changelog and version after v4.31.3 2025-11-13 21:54:56 +00:00
Michael B. Gale
014f16e7ab Merge pull request #3293 from github/update-v4.31.3-8c10e89c7
Merge main into releases/v4
2025-11-13 21:53:12 +00:00
github-actions[bot]
14d898ef09 Update changelog for v4.31.3 2025-11-13 21:18:01 +00:00
Michael B. Gale
8c10e89c78 Merge pull request #3288 from github/update-bundle/codeql-bundle-v2.23.5
Update default bundle to 2.23.5
2025-11-13 20:50:51 +00:00
Michael B. Gale
9777b01a49 Merge branch 'main' into update-bundle/codeql-bundle-v2.23.5 2025-11-13 20:11:59 +00:00
Henry Mercer
456a74a6fa Merge pull request #3289 from github/mbg/ci/setup-dotnet
Add support for adding `setup-dotnet` steps to `sync.sh`
2025-11-13 20:11:33 +00:00
Michael B. Gale
3fac49c140 Update remaining workflows 2025-11-13 19:53:24 +00:00
Michael B. Gale
38a3a7258f Enable installDotNet in all workflows that analyse C# 2025-11-13 19:48:37 +00:00
Michael B. Gale
58c9eb6c03 Add global.json 2025-11-13 19:48:37 +00:00
Michael B. Gale
f20e02164a Add support for adding setup-dotnet steps to sync.sh 2025-11-13 18:58:54 +00:00
github-actions[bot]
8d3d4001e3 Add changelog note 2025-11-13 18:40:00 +00:00
github-actions[bot]
362f8d1d2d Update default bundle to codeql-bundle-v2.23.5 2025-11-13 18:39:52 +00:00
Kasper Svendsen
5091e42a03 Overlay: Remove repository owner restriction 2025-11-13 10:48:25 +01:00
Michael B. Gale
ba454b8ab4 Merge pull request #3284 from github/mbg/ci/fix-enterprise-workflow
Fix `update-supported-enterprise-server-versions.yml` workflow
2025-11-12 15:35:56 +00:00
Michael B. Gale
7a7cd8565c Don't push for PR event 2025-11-12 15:09:25 +00:00
Michael B. Gale
fd830db27b Trigger on PR for relevant changes 2025-11-12 15:05:11 +00:00
Michael B. Gale
a7e52b690b Perform sparse checkout 2025-11-12 15:04:21 +00:00
Michael B. Gale
71c3720f43 Run npm ci in update-supported-enterprise-server-versions.yml 2025-11-12 14:57:05 +00:00
Michael B. Gale
534824ea1b Merge pull request #3117 from github/mbg/csharp/new-cache-key-calculation
Support non-lock files for C# cache key computation
2025-11-12 11:03:09 +00:00
Michael B. Gale
48a56f6b93 Add some tests for downloadDependencyCaches related to feature prefixes 2025-11-09 12:03:18 +00:00
Michael B. Gale
4885eb2ad9 Insert new featurePrefix after general cache key prefix 2025-11-09 11:17:02 +00:00
Michael B. Gale
a47d5507cf Restore earlier log messages for checkHashPatterns 2025-11-09 11:11:10 +00:00
Michael B. Gale
b0e9dfce55 Restore missing status.push resulting from a bad merge 2025-11-09 11:10:27 +00:00
Michael B. Gale
35c91ef0af Add tests for getCsharpHashPatterns
- Make the function more easily testable by allowing `makePatternCheck` to be stubbed.
- Use `makePatternCheck` for base patterns as well.
2025-11-09 11:03:15 +00:00
Michael B. Gale
71abac76d2 Fix comment in getCsharpHashPatterns 2025-11-09 10:32:10 +00:00
Michael B. Gale
46e03b48bc Fix JSDoc param name 2025-11-09 10:05:18 +00:00
Michael B. Gale
26804552e4 Use undefined instead of NoMatchingFilesError
Add tests for `makePatternCheck` and `checkHashPatterns`
2025-11-05 17:23:22 +00:00
Michael B. Gale
03b2dc2a3f Add and use getFeaturePrefix for dependency caching 2025-11-05 16:33:21 +00:00
Michael B. Gale
0cbd930deb Move createCacheKeyHash to caching-utils 2025-11-05 16:15:26 +00:00
Michael B. Gale
0324490286 Use additional files for C# key hashes if Feature.CsharpNewCacheKey is enabled 2025-11-05 16:03:41 +00:00
Michael B. Gale
6b48207907 Move check whether there are files for hashing into getHashPatterns 2025-11-05 16:03:39 +00:00
Michael B. Gale
ab1c84236a Change hash to be a function that can use Features 2025-11-05 15:57:57 +00:00
Michael B. Gale
2a7680fca6 Change getDefaultCacheConfig to be a const by turning paths into a function
Changing `paths` to be a function is necessary to allow `getTemporaryDirectory` to be called
2025-11-05 15:57:55 +00:00
Michael B. Gale
2aa1f55f3d Propagate features into cachePrefix function 2025-11-05 15:54:28 +00:00
Michael B. Gale
1ca20ab026 Add CsharpNewCacheKey FF 2025-11-05 15:48:04 +00:00
92 changed files with 8580 additions and 7404 deletions

View File

@@ -18,14 +18,25 @@ For internal use only. Please select the risk level of this change:
#### Which use cases does this change impact?
<!-- Delete options that don't apply. -->
<!-- Delete options that don't apply. If in doubt, do not delete an option. -->
- **Advanced setup** - Impacts users who have custom workflows.
- **Default setup** - Impacts users who use default setup.
- **Code Scanning** - Impacts Code Scanning (i.e. `analysis-kinds: code-scanning`).
- **Code Quality** - Impacts Code Quality (i.e. `analysis-kinds: code-quality`).
- **Third-party analyses** - Impacts third-party analyses (i.e. `upload-sarif`).
- **GHES** - Impacts GitHub Enterprise Server.
Workflow types:
- **Advanced setup** - Impacts users who have custom CodeQL workflows.
- **Managed** - Impacts users with `dynamic` workflows (Default Setup, CCR, ...).
Products:
- **Code Scanning** - The changes impact analyses when `analysis-kinds: code-scanning`.
- **Code Quality** - The changes impact analyses when `analysis-kinds: code-quality`.
- **CCR** - The changes impact analyses for Copilot Code Reviews.
- **Third-party analyses** - The changes affect the `upload-sarif` action.
Environments:
- **Dotcom** - Impacts CodeQL workflows on `github.com`.
- **GHES** - Impacts CodeQL workflows on GitHub Enterprise Server.
- **Testing/None** - This change does not impact any CodeQL workflows in production.
#### How did/will you validate this change?
@@ -54,6 +65,15 @@ For internal use only. Please select the risk level of this change:
- **Alerts** - New or existing monitors will trip if something goes wrong with this change.
- **Other** - Please provide details.
#### Are there any special considerations for merging or releasing this change?
<!--
Consider whether this change depends on a different change in another repository that should be released first.
-->
- **No special considerations** - This change can be merged at any time.
- **Special considerations** - This change should only be merged once certain preconditions are met. Please provide details of those or link to this PR from an internal issue.
### Merge / deployment checklist
- Confirm this change is backwards compatible with existing workflows.

View File

@@ -27,6 +27,11 @@ on:
description: The version of Go to install
required: false
default: '>=1.21.0'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
workflow_call:
inputs:
go-version:
@@ -34,6 +39,11 @@ on:
description: The version of Go to install
required: false
default: '>=1.21.0'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
defaults:
run:
shell: bash
@@ -74,6 +84,10 @@ jobs:
with:
go-version: ${{ inputs.go-version || '>=1.21.0' }}
cache: false
- name: Install .NET
uses: actions/setup-dotnet@v5
with:
dotnet-version: ${{ inputs.dotnet-version || '9.x' }}
- id: init
uses: ./../action/init
with:

View File

@@ -32,6 +32,11 @@ on:
description: The version of Python to install
required: false
default: '3.13'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
workflow_call:
inputs:
go-version:
@@ -44,6 +49,11 @@ on:
description: The version of Python to install
required: false
default: '3.13'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
defaults:
run:
shell: bash
@@ -85,6 +95,10 @@ jobs:
uses: actions/setup-python@v6
with:
python-version: ${{ inputs.python-version || '3.13' }}
- name: Install .NET
uses: actions/setup-dotnet@v5
with:
dotnet-version: ${{ inputs.dotnet-version || '9.x' }}
- uses: ./../action/init
with:
tools: ${{ steps.prepare-test.outputs.tools-url }}

View File

@@ -21,9 +21,19 @@ on:
schedule:
- cron: '0 5 * * *'
workflow_dispatch:
inputs: {}
inputs:
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
workflow_call:
inputs: {}
inputs:
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
defaults:
run:
shell: bash
@@ -59,6 +69,10 @@ jobs:
version: ${{ matrix.version }}
use-all-platform-bundle: 'false'
setup-kotlin: 'true'
- name: Install .NET
uses: actions/setup-dotnet@v5
with:
dotnet-version: ${{ inputs.dotnet-version || '9.x' }}
- uses: ./../action/init
with:
languages: csharp

View File

@@ -27,6 +27,11 @@ on:
description: The version of Go to install
required: false
default: '>=1.21.0'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
workflow_call:
inputs:
go-version:
@@ -34,6 +39,11 @@ on:
description: The version of Go to install
required: false
default: '>=1.21.0'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
defaults:
run:
shell: bash
@@ -70,6 +80,10 @@ jobs:
with:
go-version: ${{ inputs.go-version || '>=1.21.0' }}
cache: false
- name: Install .NET
uses: actions/setup-dotnet@v5
with:
dotnet-version: ${{ inputs.dotnet-version || '9.x' }}
- uses: ./../action/init
id: init
with:

View File

@@ -27,6 +27,11 @@ on:
description: The version of Go to install
required: false
default: '>=1.21.0'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
workflow_call:
inputs:
go-version:
@@ -34,6 +39,11 @@ on:
description: The version of Go to install
required: false
default: '>=1.21.0'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
defaults:
run:
shell: bash
@@ -74,6 +84,10 @@ jobs:
with:
go-version: ${{ inputs.go-version || '>=1.21.0' }}
cache: false
- name: Install .NET
uses: actions/setup-dotnet@v5
with:
dotnet-version: ${{ inputs.dotnet-version || '9.x' }}
- uses: ./../action/init
id: init
with:

View File

@@ -27,6 +27,11 @@ on:
description: The version of Go to install
required: false
default: '>=1.21.0'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
workflow_call:
inputs:
go-version:
@@ -34,6 +39,11 @@ on:
description: The version of Go to install
required: false
default: '>=1.21.0'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
defaults:
run:
shell: bash
@@ -72,6 +82,10 @@ jobs:
with:
go-version: ${{ inputs.go-version || '>=1.21.0' }}
cache: false
- name: Install .NET
uses: actions/setup-dotnet@v5
with:
dotnet-version: ${{ inputs.dotnet-version || '9.x' }}
- uses: ./../action/init
with:
languages: go

6
.github/workflows/__go.yml generated vendored
View File

@@ -18,6 +18,11 @@ on:
description: The version of Go to install
required: false
default: '>=1.21.0'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
jobs:
go-custom-queries:
name: 'Go: Custom queries'
@@ -27,6 +32,7 @@ jobs:
uses: ./.github/workflows/__go-custom-queries.yml
with:
go-version: ${{ inputs.go-version }}
dotnet-version: ${{ inputs.dotnet-version }}
go-indirect-tracing-workaround-diagnostic:
name: 'Go: diagnostic when Go is changed after init step'
permissions:

14
.github/workflows/__local-bundle.yml generated vendored
View File

@@ -32,6 +32,11 @@ on:
description: The version of Python to install
required: false
default: '3.13'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
workflow_call:
inputs:
go-version:
@@ -44,6 +49,11 @@ on:
description: The version of Python to install
required: false
default: '3.13'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
defaults:
run:
shell: bash
@@ -85,6 +95,10 @@ jobs:
uses: actions/setup-python@v6
with:
python-version: ${{ inputs.python-version || '3.13' }}
- name: Install .NET
uses: actions/setup-dotnet@v5
with:
dotnet-version: ${{ inputs.dotnet-version || '9.x' }}
- name: Fetch latest CodeQL bundle
run: |
wget https://github.com/github/codeql-action/releases/latest/download/codeql-bundle-linux64.tar.zst

View File

@@ -32,6 +32,11 @@ on:
description: The version of Python to install
required: false
default: '3.13'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
workflow_call:
inputs:
go-version:
@@ -44,6 +49,11 @@ on:
description: The version of Python to install
required: false
default: '3.13'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
defaults:
run:
shell: bash
@@ -119,6 +129,10 @@ jobs:
uses: actions/setup-python@v6
with:
python-version: ${{ inputs.python-version || '3.13' }}
- name: Install .NET
uses: actions/setup-dotnet@v5
with:
dotnet-version: ${{ inputs.dotnet-version || '9.x' }}
- name: Use Xcode 16
if: runner.os == 'macOS' && matrix.version != 'nightly-latest'
run: sudo xcode-select -s "/Applications/Xcode_16.app"

View File

@@ -32,6 +32,11 @@ on:
description: The version of Python to install
required: false
default: '3.13'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
workflow_call:
inputs:
go-version:
@@ -44,6 +49,11 @@ on:
description: The version of Python to install
required: false
default: '3.13'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
defaults:
run:
shell: bash
@@ -96,6 +106,10 @@ jobs:
uses: actions/setup-python@v6
with:
python-version: ${{ inputs.python-version || '3.13' }}
- name: Install .NET
uses: actions/setup-dotnet@v5
with:
dotnet-version: ${{ inputs.dotnet-version || '9.x' }}
- uses: ./../action/init
with:
config-file: .github/codeql/codeql-config-packaging3.yml

View File

@@ -27,6 +27,11 @@ on:
description: The version of Go to install
required: false
default: '>=1.21.0'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
workflow_call:
inputs:
go-version:
@@ -34,6 +39,11 @@ on:
description: The version of Go to install
required: false
default: '>=1.21.0'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
defaults:
run:
shell: bash
@@ -81,6 +91,10 @@ jobs:
with:
go-version: ${{ inputs.go-version || '>=1.21.0' }}
cache: false
- name: Install .NET
uses: actions/setup-dotnet@v5
with:
dotnet-version: ${{ inputs.dotnet-version || '9.x' }}
- uses: ./../action/init
with:
config-file: .github/codeql/codeql-config-packaging3.yml

View File

@@ -27,6 +27,11 @@ on:
description: The version of Go to install
required: false
default: '>=1.21.0'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
workflow_call:
inputs:
go-version:
@@ -34,6 +39,11 @@ on:
description: The version of Go to install
required: false
default: '>=1.21.0'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
defaults:
run:
shell: bash
@@ -81,6 +91,10 @@ jobs:
with:
go-version: ${{ inputs.go-version || '>=1.21.0' }}
cache: false
- name: Install .NET
uses: actions/setup-dotnet@v5
with:
dotnet-version: ${{ inputs.dotnet-version || '9.x' }}
- uses: ./../action/init
with:
config-file: .github/codeql/codeql-config-packaging.yml

View File

@@ -27,6 +27,11 @@ on:
description: The version of Go to install
required: false
default: '>=1.21.0'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
workflow_call:
inputs:
go-version:
@@ -34,6 +39,11 @@ on:
description: The version of Go to install
required: false
default: '>=1.21.0'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
defaults:
run:
shell: bash
@@ -81,6 +91,10 @@ jobs:
with:
go-version: ${{ inputs.go-version || '>=1.21.0' }}
cache: false
- name: Install .NET
uses: actions/setup-dotnet@v5
with:
dotnet-version: ${{ inputs.dotnet-version || '9.x' }}
- uses: ./../action/init
with:
config-file: .github/codeql/codeql-config-packaging2.yml

View File

@@ -32,6 +32,11 @@ on:
description: The version of Python to install
required: false
default: '3.13'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
workflow_call:
inputs:
go-version:
@@ -44,6 +49,11 @@ on:
description: The version of Python to install
required: false
default: '3.13'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
defaults:
run:
shell: bash
@@ -87,6 +97,10 @@ jobs:
uses: actions/setup-python@v6
with:
python-version: ${{ inputs.python-version || '3.13' }}
- name: Install .NET
uses: actions/setup-dotnet@v5
with:
dotnet-version: ${{ inputs.dotnet-version || '9.x' }}
- uses: ./../action/init
with:
tools: ${{ steps.prepare-test.outputs.tools-url }}

View File

@@ -56,7 +56,7 @@ jobs:
use-all-platform-bundle: 'false'
setup-kotlin: 'true'
- name: Set up Ruby
uses: ruby/setup-ruby@d5126b9b3579e429dd52e51e68624dda2e05be25 # v1.267.0
uses: ruby/setup-ruby@8aeb6ff8030dd539317f8e1769a044873b56ea71 # v1.268.0
with:
ruby-version: 2.6
- name: Install Code Scanning integration

View File

@@ -27,6 +27,11 @@ on:
description: The version of Go to install
required: false
default: '>=1.21.0'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
workflow_call:
inputs:
go-version:
@@ -34,6 +39,11 @@ on:
description: The version of Go to install
required: false
default: '>=1.21.0'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
defaults:
run:
shell: bash
@@ -80,6 +90,10 @@ jobs:
with:
go-version: ${{ inputs.go-version || '>=1.21.0' }}
cache: false
- name: Install .NET
uses: actions/setup-dotnet@v5
with:
dotnet-version: ${{ inputs.dotnet-version || '9.x' }}
- uses: ./../action/init
with:
config-file: .github/codeql/codeql-config-packaging3.yml

View File

@@ -27,6 +27,11 @@ on:
description: The version of Go to install
required: false
default: '>=1.21.0'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
workflow_call:
inputs:
go-version:
@@ -34,6 +39,11 @@ on:
description: The version of Go to install
required: false
default: '>=1.21.0'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
defaults:
run:
shell: bash
@@ -74,6 +84,10 @@ jobs:
with:
go-version: ${{ inputs.go-version || '>=1.21.0' }}
cache: false
- name: Install .NET
uses: actions/setup-dotnet@v5
with:
dotnet-version: ${{ inputs.dotnet-version || '9.x' }}
- name: Use Xcode 16
if: runner.os == 'macOS' && matrix.version != 'nightly-latest'
run: sudo xcode-select -s "/Applications/Xcode_16.app"

View File

@@ -32,6 +32,11 @@ on:
description: The version of Python to install
required: false
default: '3.13'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
workflow_call:
inputs:
go-version:
@@ -44,6 +49,11 @@ on:
description: The version of Python to install
required: false
default: '3.13'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
defaults:
run:
shell: bash
@@ -87,6 +97,10 @@ jobs:
uses: actions/setup-python@v6
with:
python-version: ${{ inputs.python-version || '3.13' }}
- name: Install .NET
uses: actions/setup-dotnet@v5
with:
dotnet-version: ${{ inputs.dotnet-version || '9.x' }}
- uses: ./../action/init
id: init
with:

View File

@@ -32,6 +32,11 @@ on:
description: The version of Python to install
required: false
default: '3.13'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
workflow_call:
inputs:
go-version:
@@ -44,6 +49,11 @@ on:
description: The version of Python to install
required: false
default: '3.13'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
defaults:
run:
shell: bash
@@ -85,6 +95,10 @@ jobs:
uses: actions/setup-python@v6
with:
python-version: ${{ inputs.python-version || '3.13' }}
- name: Install .NET
uses: actions/setup-dotnet@v5
with:
dotnet-version: ${{ inputs.dotnet-version || '9.x' }}
- uses: ./../action/init
with:
tools: ${{ steps.prepare-test.outputs.tools-url }}

14
.github/workflows/__upload-sarif.yml generated vendored
View File

@@ -32,6 +32,11 @@ on:
description: The version of Python to install
required: false
default: '3.13'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
workflow_call:
inputs:
go-version:
@@ -44,6 +49,11 @@ on:
description: The version of Python to install
required: false
default: '3.13'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
defaults:
run:
shell: bash
@@ -92,6 +102,10 @@ jobs:
uses: actions/setup-python@v6
with:
python-version: ${{ inputs.python-version || '3.13' }}
- name: Install .NET
uses: actions/setup-dotnet@v5
with:
dotnet-version: ${{ inputs.dotnet-version || '9.x' }}
- uses: ./../action/init
with:
tools: ${{ steps.prepare-test.outputs.tools-url }}

View File

@@ -32,6 +32,11 @@ on:
description: The version of Python to install
required: false
default: '3.13'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
workflow_call:
inputs:
go-version:
@@ -44,6 +49,11 @@ on:
description: The version of Python to install
required: false
default: '3.13'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
defaults:
run:
shell: bash
@@ -85,6 +95,10 @@ jobs:
uses: actions/setup-python@v6
with:
python-version: ${{ inputs.python-version || '3.13' }}
- name: Install .NET
uses: actions/setup-dotnet@v5
with:
dotnet-version: ${{ inputs.dotnet-version || '9.x' }}
- name: Delete original checkout
run: |
# delete the original checkout so we don't accidentally use it.

View File

@@ -54,6 +54,10 @@ jobs:
- uses: actions/setup-go@v6
with:
go-version: ^1.13.1
- name: Install .NET
uses: actions/setup-dotnet@v5
with:
dotnet-version: '9.x'
- uses: ./../action/init
with:
tools: ${{ steps.prepare-test.outputs.tools-url }}

View File

@@ -50,6 +50,10 @@ jobs:
- uses: actions/setup-go@v6
with:
go-version: ^1.13.1
- name: Install .NET
uses: actions/setup-dotnet@v5
with:
dotnet-version: '9.x'
- uses: ./../action/init
id: init
with:

View File

@@ -43,6 +43,10 @@ jobs:
with:
version: ${{ matrix.version }}
use-all-platform-bundle: true
- name: Install .NET
uses: actions/setup-dotnet@v5
with:
dotnet-version: '9.x'
- id: init
uses: ./../action/init
with:

View File

@@ -4,6 +4,12 @@ on:
schedule:
- cron: "0 0 * * *"
workflow_dispatch:
pull_request:
branches:
- main
paths:
- .github/workflows/update-supported-enterprise-server-versions.yml
- .github/workflows/update-supported-enterprise-server-versions/update.py
jobs:
update-supported-enterprise-server-versions:
@@ -28,6 +34,7 @@ jobs:
repository: github/enterprise-releases
token: ${{ secrets.ENTERPRISE_RELEASE_TOKEN }}
path: ${{ github.workspace }}/enterprise-releases/
sparse-checkout: releases.json
- name: Update Supported Enterprise Server Versions
run: |
cd ./.github/workflows/update-supported-enterprise-server-versions/
@@ -35,6 +42,7 @@ jobs:
pipenv install
pipenv run ./update.py
rm --recursive "$ENTERPRISE_RELEASES_PATH"
npm ci
npm run build
env:
ENTERPRISE_RELEASES_PATH: ${{ github.workspace }}/enterprise-releases/
@@ -44,25 +52,33 @@ jobs:
git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
git config --global user.name "github-actions[bot]"
- name: Commit changes and open PR
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Commit changes
id: prepare-commit
run: |
if [[ -z $(git status --porcelain) ]]; then
echo "No changes to commit"
echo "committed=false" >> $GITHUB_OUTPUT
else
git checkout -b update-supported-enterprise-server-versions
git add .
git commit --message "Update supported GitHub Enterprise Server versions"
git push origin update-supported-enterprise-server-versions
body="This PR updates the list of supported GitHub Enterprise Server versions, either because a new "
body+="version is about to be feature frozen, or because an old release has been deprecated."
body+=$'\n\n'
body+="If an old release has been deprecated, please follow the instructions in CONTRIBUTING.md to "
body+="deprecate the corresponding version of CodeQL."
gh pr create --draft \
--title "Update supported GitHub Enterprise Server versions" \
--body "$body"
echo "committed=true" >> $GITHUB_OUTPUT
fi
- name: Open PR
if: github.event_name != 'pull_request' && steps.prepare-commit.outputs.committed == 'true'
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
git push origin update-supported-enterprise-server-versions
body="This PR updates the list of supported GitHub Enterprise Server versions, either because a new "
body+="version is about to be feature frozen, or because an old release has been deprecated."
body+=$'\n\n'
body+="If an old release has been deprecated, please follow the instructions in CONTRIBUTING.md to "
body+="deprecate the corresponding version of CodeQL."
gh pr create --draft \
--title "Update supported GitHub Enterprise Server versions" \
--body "$body"

View File

@@ -4,7 +4,16 @@ See the [releases page](https://github.com/github/codeql-action/releases) for th
## [UNRELEASED]
No user facing changes.
## 4.31.4 - 18 Nov 2025
No user facing changes.
## 4.31.3 - 13 Nov 2025
- CodeQL Action v3 will be deprecated in December 2026. The Action now logs a warning for customers who are running v3 but could be running v4. For more information, see [Upcoming deprecation of CodeQL Action v3](https://github.blog/changelog/2025-10-28-upcoming-deprecation-of-codeql-action-v3/).
- Update default CodeQL bundle version to 2.23.5. [#3288](https://github.com/github/codeql-action/pull/3288)
## 4.31.2 - 30 Oct 2025

File diff suppressed because it is too large Load Diff

1040
lib/analyze-action.js generated

File diff suppressed because it is too large Load Diff

761
lib/autobuild-action.js generated

File diff suppressed because it is too large Load Diff

View File

@@ -1,6 +1,6 @@
{
"bundleVersion": "codeql-bundle-v2.23.3",
"cliVersion": "2.23.3",
"priorBundleVersion": "codeql-bundle-v2.23.2",
"priorCliVersion": "2.23.2"
"bundleVersion": "codeql-bundle-v2.23.5",
"cliVersion": "2.23.5",
"priorBundleVersion": "codeql-bundle-v2.23.3",
"priorCliVersion": "2.23.3"
}

1470
lib/init-action-post.js generated

File diff suppressed because it is too large Load Diff

1075
lib/init-action.js generated

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

1273
lib/start-proxy-action.js generated

File diff suppressed because it is too large Load Diff

755
lib/upload-lib.js generated

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

894
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@@ -1,6 +1,6 @@
{
"name": "codeql",
"version": "4.31.3",
"version": "4.31.5",
"private": true,
"description": "CodeQL action",
"scripts": {
@@ -35,23 +35,21 @@
"@actions/io": "^2.0.0",
"@actions/tool-cache": "^2.0.2",
"@octokit/plugin-retry": "^6.0.0",
"@octokit/request-error": "^7.0.2",
"@schemastore/package": "0.0.10",
"archiver": "^7.0.1",
"fast-deep-equal": "^3.1.3",
"follow-redirects": "^1.15.11",
"get-folder-size": "^5.0.0",
"js-yaml": "^4.1.0",
"js-yaml": "^4.1.1",
"jsonschema": "1.4.1",
"long": "^5.3.2",
"node-forge": "^1.3.1",
"octokit": "^5.0.5",
"semver": "^7.7.3",
"uuid": "^13.0.0"
},
"devDependencies": {
"@ava/typescript": "6.0.0",
"@eslint/compat": "^1.4.1",
"@eslint/compat": "^2.0.0",
"@eslint/eslintrc": "^3.3.1",
"@eslint/js": "^9.39.1",
"@microsoft/eslint-formatter-sarif": "^3.1.0",
@@ -59,10 +57,10 @@
"@types/archiver": "^7.0.0",
"@types/follow-redirects": "^1.14.4",
"@types/js-yaml": "^4.0.9",
"@types/node": "20.19.9",
"@types/node": "^20.19.9",
"@types/node-forge": "^1.3.14",
"@types/semver": "^7.7.1",
"@types/sinon": "^17.0.4",
"@types/sinon": "^21.0.0",
"@typescript-eslint/eslint-plugin": "^8.46.4",
"@typescript-eslint/parser": "^8.41.0",
"ava": "^6.4.1",
@@ -72,9 +70,9 @@
"eslint-plugin-filenames": "^1.3.2",
"eslint-plugin-github": "^5.1.8",
"eslint-plugin-import": "2.29.1",
"eslint-plugin-jsdoc": "^61.1.12",
"eslint-plugin-jsdoc": "^61.2.1",
"eslint-plugin-no-async-foreach": "^0.1.1",
"glob": "^11.0.3",
"glob": "^11.1.0",
"nock": "^14.0.10",
"sinon": "^21.0.0",
"typescript": "^5.9.3"
@@ -98,6 +96,7 @@
"eslint-plugin-jsx-a11y": {
"semver": ">=6.3.1"
},
"brace-expansion@2.0.1": "2.0.2"
"brace-expansion@2.0.1": "2.0.2",
"glob": "^11.1.0"
}
}

View File

@@ -4,6 +4,7 @@ operatingSystems: ["ubuntu", "macos", "windows"]
versions: ["nightly-latest"]
useAllPlatformBundle: "true"
installGo: true
installDotNet: true
steps:
- id: init
uses: ./../action/init

View File

@@ -3,6 +3,7 @@ description: "Checks that specifying 'ref' and 'sha' as inputs works"
versions: ["default"]
installGo: true
installPython: true
installDotNet: true
steps:
- uses: ./../action/init
with:

View File

@@ -2,6 +2,7 @@ name: "autobuild-action"
description: "Tests that the C# autobuild action works"
operatingSystems: ["ubuntu", "macos", "windows"]
versions: ["linked"]
installDotNet: true
steps:
- uses: ./../action/init
with:

View File

@@ -2,6 +2,7 @@ name: "Build mode manual"
description: "An end-to-end integration test of a Java repository built using 'build-mode: manual'"
versions: ["nightly-latest"]
installGo: true
installDotNet: true
steps:
- uses: ./../action/init
id: init

View File

@@ -3,6 +3,7 @@ description: "Tests that file baseline information is exported when the feature
operatingSystems: ["ubuntu", "macos", "windows"]
versions: ["nightly-latest"]
installGo: true
installDotNet: true
env:
CODEQL_ACTION_SUBLANGUAGE_FILE_COVERAGE: true
steps:

View File

@@ -7,6 +7,7 @@ versions:
- linked
- nightly-latest
installGo: true
installDotNet: true
env:
DOTNET_GENERATE_ASPNET_CERTIFICATE: "false"
steps:

View File

@@ -3,6 +3,7 @@ description: "Tests using a CodeQL bundle from a local file rather than a URL"
versions: ["linked"]
installGo: true
installPython: true
installDotNet: true
steps:
- name: Fetch latest CodeQL bundle
run: |

View File

@@ -5,6 +5,7 @@ env:
CODEQL_ACTION_RESOLVE_SUPPORTED_LANGUAGES_USING_CLI: true
installGo: true
installPython: true
installDotNet: true
steps:
- name: Use Xcode 16
if: runner.os == 'macOS' && matrix.version != 'nightly-latest'

View File

@@ -4,6 +4,7 @@ versions: ["linked", "default", "nightly-latest"] # This feature is not compatib
installGo: true
installNode: true
installPython: true
installDotNet: true
steps:
- uses: ./../action/init
with:

View File

@@ -3,6 +3,7 @@ description: "Checks that specifying packages using a combination of a config fi
versions: ["linked", "default", "nightly-latest"] # This feature is not compatible with old CLIs
installGo: true
installNode: true
installDotNet: true
steps:
- uses: ./../action/init
with:

View File

@@ -3,6 +3,7 @@ description: "Checks that specifying packages using only a config file works"
versions: ["linked", "default", "nightly-latest"] # This feature is not compatible with old CLIs
installGo: true
installNode: true
installDotNet: true
steps:
- uses: ./../action/init
with:

View File

@@ -3,6 +3,7 @@ description: "Checks that specifying packages using the input to the Action work
versions: ["linked", "default", "nightly-latest"] # This feature is not compatible with old CLIs
installGo: true
installNode: true
installDotNet: true
steps:
- uses: ./../action/init
with:

View File

@@ -7,6 +7,7 @@ versions:
- nightly-latest
installGo: true
installPython: true
installDotNet: true
steps:
- uses: ./../action/init
with:

View File

@@ -4,7 +4,7 @@ description: "Tests using RuboCop to analyze a multi-language repository and the
versions: ["default"]
steps:
- name: Set up Ruby
uses: ruby/setup-ruby@d5126b9b3579e429dd52e51e68624dda2e05be25 # v1.267.0
uses: ruby/setup-ruby@8aeb6ff8030dd539317f8e1769a044873b56ea71 # v1.268.0
with:
ruby-version: 2.6
- name: Install Code Scanning integration

View File

@@ -3,6 +3,7 @@ description: "Tests a split-up workflow in which we first build a database and l
operatingSystems: ["ubuntu", "macos"]
versions: ["linked", "default", "nightly-latest"] # This feature is not compatible with old CLIs
installGo: true
installDotNet: true
steps:
- uses: ./../action/init
with:

View File

@@ -3,6 +3,7 @@ description: "Tests creation of a Swift database using custom build"
versions: ["linked", "default", "nightly-latest"]
operatingSystems: ["macos"]
installGo: true
installDotNet: true
env:
DOTNET_GENERATE_ASPNET_CERTIFICATE: "false"
steps:

View File

@@ -7,6 +7,7 @@ versions:
- nightly-latest
installGo: true
installPython: true
installDotNet: true
steps:
- uses: ./../action/init
id: init

View File

@@ -3,6 +3,7 @@ description: "Checks that specifying 'ref' and 'sha' as inputs works"
versions: ["default"]
installGo: true
installPython: true
installDotNet: true
steps:
- uses: ./../action/init
with:

View File

@@ -4,6 +4,7 @@ versions: ["default"]
analysisKinds: ["code-scanning", "code-quality", "code-scanning,code-quality"]
installGo: true
installPython: true
installDotNet: true
steps:
- uses: ./../action/init
with:

View File

@@ -3,6 +3,7 @@ description: "Checks that a custom `checkout_path` will find the proper commit_o
versions: ["linked"]
installGo: true
installPython: true
installDotNet: true
steps:
# This ensures we don't accidentally use the original checkout for any part of the test.
- name: Delete original checkout

View File

@@ -204,6 +204,25 @@ for file in sorted((this_dir / 'checks').glob('*.yml')):
}
})
installDotNet = is_truthy(checkSpecification.get('installDotNet', ''))
if installDotNet:
baseDotNetVersionExpr = '9.x'
workflowInputs['dotnet-version'] = {
'type': 'string',
'description': 'The version of .NET to install',
'required': False,
'default': baseDotNetVersionExpr,
}
steps.append({
'name': 'Install .NET',
'uses': 'actions/setup-dotnet@v5',
'with': {
'dotnet-version': '${{ inputs.dotnet-version || \'' + baseDotNetVersionExpr + '\' }}'
}
})
# If container initialisation steps are present in the check specification,
# make sure to execute them first.
if 'container' in checkSpecification and 'container-init-steps' in checkSpecification:

View File

@@ -80,7 +80,7 @@ export function isRunningLocalAction(): boolean {
*
* This can be used to get the Action's name or tell if we're running a local Action.
*/
export function getRelativeScriptPath(): string {
function getRelativeScriptPath(): string {
const runnerTemp = getRequiredEnvParam("RUNNER_TEMP");
const actionsDirectory = path.join(path.dirname(runnerTemp), "_actions");
return path.relative(actionsDirectory, __filename);

View File

@@ -98,7 +98,7 @@ export async function getAnalysisKinds(
export const codeQualityQueries: string[] = ["code-quality"];
// Enumerates API endpoints that accept SARIF files.
export enum SARIF_UPLOAD_ENDPOINT {
enum SARIF_UPLOAD_ENDPOINT {
CODE_SCANNING = "PUT /repos/:owner/:repo/code-scanning/analysis",
CODE_QUALITY = "PUT /repos/:owner/:repo/code-quality/analysis",
}

View File

@@ -25,7 +25,7 @@ import {
isCodeQualityEnabled,
isCodeScanningEnabled,
} from "./config-utils";
import { uploadDatabases } from "./database-upload";
import { cleanupAndUploadDatabases } from "./database-upload";
import {
DependencyCacheUploadStatusReport,
uploadDependencyCaches,
@@ -35,7 +35,7 @@ import { EnvVar } from "./environment";
import { Feature, Features } from "./feature-flags";
import { KnownLanguage } from "./languages";
import { getActionsLogger, Logger } from "./logging";
import { uploadOverlayBaseDatabaseToCache } from "./overlay-database-utils";
import { cleanupAndUploadOverlayBaseDatabaseToCache } from "./overlay-database-utils";
import { getRepositoryNwo } from "./repository";
import * as statusReport from "./status-report";
import {
@@ -417,12 +417,21 @@ async function run() {
}
// Possibly upload the overlay-base database to actions cache.
// If databases are to be uploaded, they will first be cleaned up at the overlay level.
await uploadOverlayBaseDatabaseToCache(codeql, config, logger);
// Note: Take care with the ordering of this call since databases may be cleaned up
// at the `overlay` level.
await cleanupAndUploadOverlayBaseDatabaseToCache(codeql, config, logger);
// Possibly upload the database bundles for remote queries.
// If databases are to be uploaded, they will first be cleaned up at the clear level.
await uploadDatabases(repositoryNwo, codeql, config, apiDetails, logger);
// Note: Take care with the ordering of this call since databases may be cleaned up
// at the `overlay` or `clear` level.
await cleanupAndUploadDatabases(
repositoryNwo,
codeql,
config,
apiDetails,
features,
logger,
);
// Possibly upload the TRAP caches for later re-use
const trapCacheUploadStartTime = performance.now();
@@ -438,14 +447,11 @@ async function run() {
// Store dependency cache(s) if dependency caching is enabled.
if (shouldStoreCache(config.dependencyCachingEnabled)) {
const minimizeJavaJars = await features.getValue(
Feature.JavaMinimizeDependencyJars,
codeql,
);
dependencyCacheResults = await uploadDependencyCaches(
codeql,
features,
config,
logger,
minimizeJavaJars,
);
}

View File

@@ -18,11 +18,6 @@ import {
const GITHUB_ENTERPRISE_VERSION_HEADER = "x-github-enterprise-version";
export enum DisallowedAPIVersionReason {
ACTION_TOO_OLD,
ACTION_TOO_NEW,
}
export type GitHubApiCombinedDetails = GitHubApiDetails &
GitHubApiExternalRepoDetails;

View File

@@ -1,3 +1,5 @@
import * as crypto from "crypto";
import * as core from "@actions/core";
import { getOptionalInput, isDefaultSetup } from "./actions-util";
@@ -71,6 +73,33 @@ export function getCachingKind(input: string | undefined): CachingKind {
}
}
// The length to which `createCacheKeyHash` truncates hash strings.
export const cacheKeyHashLength = 16;
/**
* Creates a SHA-256 hash of the cache key components to ensure uniqueness
* while keeping the cache key length manageable.
*
* @param components Object containing all components that should influence cache key uniqueness
* @returns A short SHA-256 hash (first 16 characters) of the components
*/
export function createCacheKeyHash(components: Record<string, any>): string {
// From https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/JSON/stringify
//
// "Properties are visited using the same algorithm as Object.keys(), which
// has a well-defined order and is stable across implementations. For example,
// JSON.stringify on the same object will always produce the same string, and
// JSON.parse(JSON.stringify(obj)) would produce an object with the same key
// ordering as the original (assuming the object is completely
// JSON-serializable)."
const componentsJson = JSON.stringify(components);
return crypto
.createHash("sha256")
.update(componentsJson)
.digest("hex")
.substring(0, cacheKeyHashLength);
}
/** Determines whether dependency caching is enabled. */
export function getDependencyCachingEnabled(): CachingKind {
// If the workflow specified something always respect that

View File

@@ -159,10 +159,7 @@ type CliErrorConfiguration = {
* All of our caught CLI error messages that we handle specially: ie. if we
* would like to categorize an error as a configuration error or not.
*/
export const cliErrorsConfig: Record<
CliConfigErrorCategory,
CliErrorConfiguration
> = {
const cliErrorsConfig: Record<CliConfigErrorCategory, CliErrorConfiguration> = {
[CliConfigErrorCategory.AutobuildError]: {
cliErrorMessageCandidates: [
new RegExp("We were unable to automatically build your code"),

View File

@@ -35,7 +35,7 @@ import { ToolsDownloadStatusReport } from "./tools-download";
import { ToolsFeature, isSupportedToolsFeature } from "./tools-features";
import { shouldEnableIndirectTracing } from "./tracer-config";
import * as util from "./util";
import { BuildMode, getErrorMessage } from "./util";
import { BuildMode, CleanupLevel, getErrorMessage } from "./util";
type Options = Array<string | number | boolean>;
@@ -141,7 +141,10 @@ export interface CodeQL {
/**
* Clean up all the databases within a database cluster.
*/
databaseCleanupCluster(config: Config, cleanupLevel: string): Promise<void>;
databaseCleanupCluster(
config: Config,
cleanupLevel: CleanupLevel,
): Promise<void>;
/**
* Run 'codeql database bundle'.
*/
@@ -513,7 +516,7 @@ export async function getCodeQLForTesting(
* version requirement. Must be set to true outside tests.
* @returns A new CodeQL object
*/
export async function getCodeQLForCmd(
async function getCodeQLForCmd(
cmd: string,
checkVersion: boolean,
): Promise<CodeQL> {
@@ -878,7 +881,7 @@ export async function getCodeQLForCmd(
},
async databaseCleanupCluster(
config: Config,
cleanupLevel: string,
cleanupLevel: CleanupLevel,
): Promise<void> {
const cacheCleanupFlag = (await util.codeQlVersionAtLeast(
this,
@@ -1222,7 +1225,7 @@ export async function getTrapCachingExtractorConfigArgsForLang(
*
* This will not exist if the configuration is being parsed in the Action.
*/
export function getGeneratedCodeScanningConfigPath(config: Config): string {
function getGeneratedCodeScanningConfigPath(config: Config): string {
return path.resolve(config.tempDir, "user-config.yaml");
}

View File

@@ -37,7 +37,9 @@ import {
ConfigurationError,
withTmpDir,
BuildMode,
DiskUsage,
} from "./util";
import * as util from "./util";
setupTests(test);
@@ -200,12 +202,9 @@ test("load code quality config", async (t) => {
);
// And the config we expect it to result in
const expectedConfig: configUtils.Config = {
version: actionsUtil.getActionVersion(),
const expectedConfig = createTestConfig({
analysisKinds: [AnalysisKind.CodeQuality],
languages: [KnownLanguage.actions],
buildMode: undefined,
originalUserInput: {},
// This gets set because we only have `AnalysisKind.CodeQuality`
computedConfig: {
"disable-default-queries": true,
@@ -219,14 +218,7 @@ test("load code quality config", async (t) => {
debugMode: false,
debugArtifactName: "",
debugDatabaseName: "",
trapCaches: {},
trapCacheDownloadTime: 0,
dependencyCachingEnabled: CachingKind.None,
extraQueryExclusions: [],
overlayDatabaseMode: OverlayDatabaseMode.None,
useOverlayDatabaseCaching: false,
repositoryProperties: {},
};
});
t.deepEqual(config, expectedConfig);
});
@@ -507,9 +499,7 @@ test("load non-empty input", async (t) => {
};
// And the config we expect it to parse to
const expectedConfig: configUtils.Config = {
version: actionsUtil.getActionVersion(),
analysisKinds: [AnalysisKind.CodeScanning],
const expectedConfig = createTestConfig({
languages: [KnownLanguage.javascript],
buildMode: BuildMode.None,
originalUserInput: userConfig,
@@ -521,14 +511,7 @@ test("load non-empty input", async (t) => {
debugMode: false,
debugArtifactName: "my-artifact",
debugDatabaseName: "my-db",
trapCaches: {},
trapCacheDownloadTime: 0,
dependencyCachingEnabled: CachingKind.None,
extraQueryExclusions: [],
overlayDatabaseMode: OverlayDatabaseMode.None,
useOverlayDatabaseCaching: false,
repositoryProperties: {},
};
});
const languagesInput = "javascript";
const configFilePath = createConfigFile(inputFileContents, tempDir);
@@ -990,12 +973,12 @@ interface OverlayDatabaseModeTestSetup {
features: Feature[];
isPullRequest: boolean;
isDefaultBranch: boolean;
repositoryOwner: string;
buildMode: BuildMode | undefined;
languages: Language[];
codeqlVersion: string;
gitRoot: string | undefined;
codeScanningConfig: configUtils.UserConfig;
diskUsage: DiskUsage | undefined;
}
const defaultOverlayDatabaseModeTestSetup: OverlayDatabaseModeTestSetup = {
@@ -1003,12 +986,15 @@ const defaultOverlayDatabaseModeTestSetup: OverlayDatabaseModeTestSetup = {
features: [],
isPullRequest: false,
isDefaultBranch: false,
repositoryOwner: "github",
buildMode: BuildMode.None,
languages: [KnownLanguage.javascript],
codeqlVersion: CODEQL_OVERLAY_MINIMUM_VERSION,
gitRoot: "/some/git/root",
codeScanningConfig: {},
diskUsage: {
numAvailableBytes: 50_000_000_000,
numTotalBytes: 100_000_000_000,
},
};
const getOverlayDatabaseModeMacro = test.macro({
@@ -1041,6 +1027,8 @@ const getOverlayDatabaseModeMacro = test.macro({
setup.overlayDatabaseEnvVar;
}
sinon.stub(util, "checkDiskUsage").resolves(setup.diskUsage);
// Mock feature flags
const features = createFeatures(setup.features);
@@ -1049,12 +1037,6 @@ const getOverlayDatabaseModeMacro = test.macro({
.stub(actionsUtil, "isAnalyzingPullRequest")
.returns(setup.isPullRequest);
// Mock repository owner
const repository = {
owner: setup.repositoryOwner,
repo: "test-repo",
};
// Set up CodeQL mock
const codeql = mockCodeQLVersion(setup.codeqlVersion);
@@ -1077,7 +1059,6 @@ const getOverlayDatabaseModeMacro = test.macro({
const result = await configUtils.getOverlayDatabaseMode(
codeql,
repository,
features,
setup.languages,
tempDir, // sourceRoot
@@ -1205,6 +1186,45 @@ test(
},
);
test(
getOverlayDatabaseModeMacro,
"No overlay-base database on default branch if runner disk space is too low",
{
languages: [KnownLanguage.javascript],
features: [
Feature.OverlayAnalysis,
Feature.OverlayAnalysisCodeScanningJavascript,
],
isDefaultBranch: true,
diskUsage: {
numAvailableBytes: 1_000_000_000,
numTotalBytes: 100_000_000_000,
},
},
{
overlayDatabaseMode: OverlayDatabaseMode.None,
useOverlayDatabaseCaching: false,
},
);
test(
getOverlayDatabaseModeMacro,
"No overlay-base database on default branch if we can't determine runner disk space",
{
languages: [KnownLanguage.javascript],
features: [
Feature.OverlayAnalysis,
Feature.OverlayAnalysisCodeScanningJavascript,
],
isDefaultBranch: true,
diskUsage: undefined,
},
{
overlayDatabaseMode: OverlayDatabaseMode.None,
useOverlayDatabaseCaching: false,
},
);
test(
getOverlayDatabaseModeMacro,
"No overlay-base database on default branch when code-scanning feature enabled with disable-default-queries",
@@ -1375,6 +1395,45 @@ test(
},
);
test(
getOverlayDatabaseModeMacro,
"No overlay analysis on PR if runner disk space is too low",
{
languages: [KnownLanguage.javascript],
features: [
Feature.OverlayAnalysis,
Feature.OverlayAnalysisCodeScanningJavascript,
],
isPullRequest: true,
diskUsage: {
numAvailableBytes: 1_000_000_000,
numTotalBytes: 100_000_000_000,
},
},
{
overlayDatabaseMode: OverlayDatabaseMode.None,
useOverlayDatabaseCaching: false,
},
);
test(
getOverlayDatabaseModeMacro,
"No overlay analysis on PR if we can't determine runner disk space",
{
languages: [KnownLanguage.javascript],
features: [
Feature.OverlayAnalysis,
Feature.OverlayAnalysisCodeScanningJavascript,
],
isPullRequest: true,
diskUsage: undefined,
},
{
overlayDatabaseMode: OverlayDatabaseMode.None,
useOverlayDatabaseCaching: false,
},
);
test(
getOverlayDatabaseModeMacro,
"No overlay analysis on PR when code-scanning feature enabled with disable-default-queries",
@@ -1499,10 +1558,9 @@ test(
test(
getOverlayDatabaseModeMacro,
"Overlay PR analysis by env for dsp-testing",
"Overlay PR analysis by env",
{
overlayDatabaseEnvVar: "overlay",
repositoryOwner: "dsp-testing",
},
{
overlayDatabaseMode: OverlayDatabaseMode.Overlay,
@@ -1512,10 +1570,10 @@ test(
test(
getOverlayDatabaseModeMacro,
"Overlay PR analysis by env for other-org",
"Overlay PR analysis by env on a runner with low disk space",
{
overlayDatabaseEnvVar: "overlay",
repositoryOwner: "other-org",
diskUsage: { numAvailableBytes: 0, numTotalBytes: 100_000_000_000 },
},
{
overlayDatabaseMode: OverlayDatabaseMode.Overlay,
@@ -1525,12 +1583,11 @@ test(
test(
getOverlayDatabaseModeMacro,
"Overlay PR analysis by feature flag for dsp-testing",
"Overlay PR analysis by feature flag",
{
languages: [KnownLanguage.javascript],
features: [Feature.OverlayAnalysis, Feature.OverlayAnalysisJavascript],
isPullRequest: true,
repositoryOwner: "dsp-testing",
},
{
overlayDatabaseMode: OverlayDatabaseMode.Overlay,
@@ -1538,21 +1595,6 @@ test(
},
);
test(
getOverlayDatabaseModeMacro,
"No overlay PR analysis by feature flag for other-org",
{
languages: [KnownLanguage.javascript],
features: [Feature.OverlayAnalysis, Feature.OverlayAnalysisJavascript],
isPullRequest: true,
repositoryOwner: "other-org",
},
{
overlayDatabaseMode: OverlayDatabaseMode.None,
useOverlayDatabaseCaching: false,
},
);
test(
getOverlayDatabaseModeMacro,
"Fallback due to autobuild with traced language",

View File

@@ -43,10 +43,22 @@ import {
codeQlVersionAtLeast,
cloneObject,
isDefined,
checkDiskUsage,
} from "./util";
export * from "./config/db-config";
/**
* The minimum available disk space (in MB) required to perform overlay analysis.
* If the available disk space on the runner is below the threshold when deciding
* whether to perform overlay analysis, then the action will not perform overlay
* analysis unless overlay analysis has been explicitly enabled via environment
* variable.
*/
const OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_MB = 20000;
const OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_BYTES =
OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_MB * 1_000_000;
export type RegistryConfigWithCredentials = RegistryConfigNoCredentials & {
// Token to use when downloading packs from this registry.
token: string;
@@ -148,6 +160,9 @@ export interface Config {
/** A value indicating how dependency caching should be used. */
dependencyCachingEnabled: CachingKind;
/** The keys of caches that we restored, if any. */
dependencyCachingRestoredKeys: string[];
/**
* Extra query exclusions to append to the config.
*/
@@ -176,7 +191,7 @@ export interface Config {
repositoryProperties: RepositoryProperties;
}
export async function getSupportedLanguageMap(
async function getSupportedLanguageMap(
codeql: CodeQL,
logger: Logger,
): Promise<Record<string, string>> {
@@ -239,7 +254,7 @@ export function hasActionsWorkflows(sourceRoot: string): boolean {
/**
* Gets the set of languages in the current repository.
*/
export async function getRawLanguagesInRepo(
async function getRawLanguagesInRepo(
repository: RepositoryNwo,
sourceRoot: string,
logger: Logger,
@@ -348,7 +363,7 @@ export function getRawLanguagesNoAutodetect(
* @returns A tuple containing a list of languages in this repository that might be
* analyzable and whether or not this list was determined automatically.
*/
export async function getRawLanguages(
async function getRawLanguages(
languagesInput: string | undefined,
repository: RepositoryNwo,
sourceRoot: string,
@@ -496,6 +511,7 @@ export async function initActionState(
trapCaches,
trapCacheDownloadTime,
dependencyCachingEnabled: getCachingKind(dependencyCachingEnabled),
dependencyCachingRestoredKeys: [],
extraQueryExclusions: [],
overlayDatabaseMode: OverlayDatabaseMode.None,
useOverlayDatabaseCaching: false,
@@ -579,17 +595,11 @@ const OVERLAY_ANALYSIS_CODE_SCANNING_FEATURES: Record<Language, Feature> = {
};
async function isOverlayAnalysisFeatureEnabled(
repository: RepositoryNwo,
features: FeatureEnablement,
codeql: CodeQL,
languages: Language[],
codeScanningConfig: UserConfig,
): Promise<boolean> {
// TODO: Remove the repository owner check once support for overlay analysis
// stabilizes, and no more backward-incompatible changes are expected.
if (!["github", "dsp-testing"].includes(repository.owner)) {
return false;
}
if (!(await features.getValue(Feature.OverlayAnalysis, codeql))) {
return false;
}
@@ -647,7 +657,6 @@ async function isOverlayAnalysisFeatureEnabled(
*/
export async function getOverlayDatabaseMode(
codeql: CodeQL,
repository: RepositoryNwo,
features: FeatureEnablement,
languages: Language[],
sourceRoot: string,
@@ -676,27 +685,43 @@ export async function getOverlayDatabaseMode(
);
} else if (
await isOverlayAnalysisFeatureEnabled(
repository,
features,
codeql,
languages,
codeScanningConfig,
)
) {
if (isAnalyzingPullRequest()) {
overlayDatabaseMode = OverlayDatabaseMode.Overlay;
useOverlayDatabaseCaching = true;
const diskUsage = await checkDiskUsage(logger);
if (
diskUsage === undefined ||
diskUsage.numAvailableBytes < OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_BYTES
) {
const diskSpaceMb =
diskUsage === undefined
? 0
: Math.round(diskUsage.numAvailableBytes / 1_000_000);
overlayDatabaseMode = OverlayDatabaseMode.None;
useOverlayDatabaseCaching = false;
logger.info(
`Setting overlay database mode to ${overlayDatabaseMode} ` +
"with caching because we are analyzing a pull request.",
);
} else if (await isAnalyzingDefaultBranch()) {
overlayDatabaseMode = OverlayDatabaseMode.OverlayBase;
useOverlayDatabaseCaching = true;
logger.info(
`Setting overlay database mode to ${overlayDatabaseMode} ` +
"with caching because we are analyzing the default branch.",
`due to insufficient disk space (${diskSpaceMb} MB).`,
);
} else {
if (isAnalyzingPullRequest()) {
overlayDatabaseMode = OverlayDatabaseMode.Overlay;
useOverlayDatabaseCaching = true;
logger.info(
`Setting overlay database mode to ${overlayDatabaseMode} ` +
"with caching because we are analyzing a pull request.",
);
} else if (await isAnalyzingDefaultBranch()) {
overlayDatabaseMode = OverlayDatabaseMode.OverlayBase;
useOverlayDatabaseCaching = true;
logger.info(
`Setting overlay database mode to ${overlayDatabaseMode} ` +
"with caching because we are analyzing the default branch.",
);
}
}
}
@@ -846,7 +871,6 @@ export async function initConfig(
const { overlayDatabaseMode, useOverlayDatabaseCaching } =
await getOverlayDatabaseMode(
inputs.codeql,
inputs.repository,
inputs.features,
config.languages,
inputs.sourceRoot,
@@ -1235,7 +1259,7 @@ export function isCodeQualityEnabled(config: Config): boolean {
* @returns Returns `AnalysisKind.CodeScanning` if `AnalysisKind.CodeScanning` is enabled;
* otherwise `AnalysisKind.CodeQuality`.
*/
export function getPrimaryAnalysisKind(config: Config): AnalysisKind {
function getPrimaryAnalysisKind(config: Config): AnalysisKind {
return isCodeScanningEnabled(config)
? AnalysisKind.CodeScanning
: AnalysisKind.CodeQuality;

View File

@@ -10,11 +10,12 @@ import { GitHubApiDetails } from "./api-client";
import * as apiClient from "./api-client";
import { createStubCodeQL } from "./codeql";
import { Config } from "./config-utils";
import { uploadDatabases } from "./database-upload";
import { cleanupAndUploadDatabases } from "./database-upload";
import * as gitUtils from "./git-utils";
import { KnownLanguage } from "./languages";
import { RepositoryNwo } from "./repository";
import {
createFeatures,
createTestConfig,
getRecordingLogger,
LoggedMessage,
@@ -91,11 +92,12 @@ test("Abort database upload if 'upload-database' input set to false", async (t)
sinon.stub(gitUtils, "isAnalyzingDefaultBranch").resolves(true);
const loggedMessages = [];
await uploadDatabases(
await cleanupAndUploadDatabases(
testRepoName,
getCodeQL(),
getTestConfig(tmpDir),
testApiDetails,
createFeatures([]),
getRecordingLogger(loggedMessages),
);
t.assert(
@@ -121,7 +123,7 @@ test("Abort database upload if 'analysis-kinds: code-scanning' is not enabled",
await mockHttpRequests(201);
const loggedMessages = [];
await uploadDatabases(
await cleanupAndUploadDatabases(
testRepoName,
getCodeQL(),
{
@@ -129,6 +131,7 @@ test("Abort database upload if 'analysis-kinds: code-scanning' is not enabled",
analysisKinds: [AnalysisKind.CodeQuality],
},
testApiDetails,
createFeatures([]),
getRecordingLogger(loggedMessages),
);
t.assert(
@@ -155,11 +158,12 @@ test("Abort database upload if running against GHES", async (t) => {
config.gitHubVersion = { type: GitHubVariant.GHES, version: "3.0" };
const loggedMessages = [];
await uploadDatabases(
await cleanupAndUploadDatabases(
testRepoName,
getCodeQL(),
config,
testApiDetails,
createFeatures([]),
getRecordingLogger(loggedMessages),
);
t.assert(
@@ -183,11 +187,12 @@ test("Abort database upload if not analyzing default branch", async (t) => {
sinon.stub(gitUtils, "isAnalyzingDefaultBranch").resolves(false);
const loggedMessages = [];
await uploadDatabases(
await cleanupAndUploadDatabases(
testRepoName,
getCodeQL(),
getTestConfig(tmpDir),
testApiDetails,
createFeatures([]),
getRecordingLogger(loggedMessages),
);
t.assert(
@@ -212,11 +217,12 @@ test("Don't crash if uploading a database fails", async (t) => {
await mockHttpRequests(500);
const loggedMessages = [] as LoggedMessage[];
await uploadDatabases(
await cleanupAndUploadDatabases(
testRepoName,
getCodeQL(),
getTestConfig(tmpDir),
testApiDetails,
createFeatures([]),
getRecordingLogger(loggedMessages),
);
@@ -243,11 +249,12 @@ test("Successfully uploading a database to github.com", async (t) => {
await mockHttpRequests(201);
const loggedMessages = [] as LoggedMessage[];
await uploadDatabases(
await cleanupAndUploadDatabases(
testRepoName,
getCodeQL(),
getTestConfig(tmpDir),
testApiDetails,
createFeatures([]),
getRecordingLogger(loggedMessages),
);
t.assert(
@@ -272,7 +279,7 @@ test("Successfully uploading a database to GHEC-DR", async (t) => {
const databaseUploadSpy = await mockHttpRequests(201);
const loggedMessages = [] as LoggedMessage[];
await uploadDatabases(
await cleanupAndUploadDatabases(
testRepoName,
getCodeQL(),
getTestConfig(tmpDir),
@@ -281,6 +288,7 @@ test("Successfully uploading a database to GHEC-DR", async (t) => {
url: "https://tenant.ghe.com",
apiURL: undefined,
},
createFeatures([]),
getRecordingLogger(loggedMessages),
);
t.assert(

View File

@@ -5,17 +5,20 @@ import { AnalysisKind } from "./analyses";
import { getApiClient, GitHubApiDetails } from "./api-client";
import { type CodeQL } from "./codeql";
import { Config } from "./config-utils";
import { Feature, FeatureEnablement } from "./feature-flags";
import * as gitUtils from "./git-utils";
import { Logger, withGroupAsync } from "./logging";
import { OverlayDatabaseMode } from "./overlay-database-utils";
import { RepositoryNwo } from "./repository";
import * as util from "./util";
import { bundleDb, parseGitHubUrl } from "./util";
import { bundleDb, CleanupLevel, parseGitHubUrl } from "./util";
export async function uploadDatabases(
export async function cleanupAndUploadDatabases(
repositoryNwo: RepositoryNwo,
codeql: CodeQL,
config: Config,
apiDetails: GitHubApiDetails,
features: FeatureEnablement,
logger: Logger,
): Promise<void> {
if (actionsUtil.getRequiredInput("upload-database") !== "true") {
@@ -50,10 +53,16 @@ export async function uploadDatabases(
return;
}
const cleanupLevel =
config.overlayDatabaseMode === OverlayDatabaseMode.OverlayBase &&
(await features.getValue(Feature.UploadOverlayDbToApi))
? CleanupLevel.Overlay
: CleanupLevel.Clear;
// Clean up the database, since intermediate results may still be written to the
// database if there is high RAM pressure.
await withGroupAsync("Cleaning up databases", async () => {
await codeql.databaseCleanupCluster(config, "clear");
await codeql.databaseCleanupCluster(config, cleanupLevel);
});
const client = getApiClient();

View File

@@ -1,6 +1,6 @@
{
"bundleVersion": "codeql-bundle-v2.23.3",
"cliVersion": "2.23.3",
"priorBundleVersion": "codeql-bundle-v2.23.2",
"priorCliVersion": "2.23.2"
"bundleVersion": "codeql-bundle-v2.23.5",
"cliVersion": "2.23.5",
"priorBundleVersion": "codeql-bundle-v2.23.3",
"priorCliVersion": "2.23.3"
}

View File

@@ -0,0 +1,627 @@
import * as fs from "fs";
import path from "path";
import * as actionsCache from "@actions/cache";
import * as glob from "@actions/glob";
import test from "ava";
import * as sinon from "sinon";
import { cacheKeyHashLength } from "./caching-utils";
import * as cachingUtils from "./caching-utils";
import { createStubCodeQL } from "./codeql";
import {
CacheConfig,
checkHashPatterns,
getCsharpHashPatterns,
getFeaturePrefix,
makePatternCheck,
internal,
CSHARP_BASE_PATTERNS,
CSHARP_EXTRA_PATTERNS,
downloadDependencyCaches,
CacheHitKind,
cacheKey,
uploadDependencyCaches,
CacheStoreResult,
} from "./dependency-caching";
import { Feature } from "./feature-flags";
import { KnownLanguage } from "./languages";
import {
setupTests,
createFeatures,
getRecordingLogger,
checkExpectedLogMessages,
LoggedMessage,
createTestConfig,
} from "./testing-utils";
import { withTmpDir } from "./util";
setupTests(test);
function makeAbsolutePatterns(tmpDir: string, patterns: string[]): string[] {
return patterns.map((pattern) => path.join(tmpDir, pattern));
}
test("makePatternCheck - returns undefined if no patterns match", async (t) => {
await withTmpDir(async (tmpDir) => {
fs.writeFileSync(path.join(tmpDir, "test.java"), "");
const result = await makePatternCheck(
makeAbsolutePatterns(tmpDir, ["**/*.cs"]),
);
t.is(result, undefined);
});
});
test("makePatternCheck - returns all patterns if any pattern matches", async (t) => {
await withTmpDir(async (tmpDir) => {
fs.writeFileSync(path.join(tmpDir, "test.java"), "");
const patterns = makeAbsolutePatterns(tmpDir, ["**/*.cs", "**/*.java"]);
const result = await makePatternCheck(patterns);
t.deepEqual(result, patterns);
});
});
test("getCsharpHashPatterns - returns base patterns if any pattern matches", async (t) => {
const codeql = createStubCodeQL({});
const features = createFeatures([]);
const makePatternCheckStub = sinon.stub(internal, "makePatternCheck");
makePatternCheckStub
.withArgs(CSHARP_BASE_PATTERNS)
.resolves(CSHARP_BASE_PATTERNS);
makePatternCheckStub.withArgs(CSHARP_EXTRA_PATTERNS).rejects();
await t.notThrowsAsync(async () => {
const result = await getCsharpHashPatterns(codeql, features);
t.deepEqual(result, CSHARP_BASE_PATTERNS);
});
});
test("getCsharpHashPatterns - returns base patterns if any base pattern matches and CsharpNewCacheKey is enabled", async (t) => {
const codeql = createStubCodeQL({});
const features = createFeatures([Feature.CsharpNewCacheKey]);
const makePatternCheckStub = sinon.stub(internal, "makePatternCheck");
makePatternCheckStub
.withArgs(CSHARP_BASE_PATTERNS)
.resolves(CSHARP_BASE_PATTERNS);
makePatternCheckStub
.withArgs(CSHARP_EXTRA_PATTERNS)
.resolves(CSHARP_EXTRA_PATTERNS);
await t.notThrowsAsync(async () => {
const result = await getCsharpHashPatterns(codeql, features);
t.deepEqual(result, CSHARP_BASE_PATTERNS);
});
});
test("getCsharpHashPatterns - returns extra patterns if any extra pattern matches and CsharpNewCacheKey is enabled", async (t) => {
const codeql = createStubCodeQL({});
const features = createFeatures([Feature.CsharpNewCacheKey]);
const makePatternCheckStub = sinon.stub(internal, "makePatternCheck");
makePatternCheckStub.withArgs(CSHARP_BASE_PATTERNS).resolves(undefined);
makePatternCheckStub
.withArgs(CSHARP_EXTRA_PATTERNS)
.resolves(CSHARP_EXTRA_PATTERNS);
await t.notThrowsAsync(async () => {
const result = await getCsharpHashPatterns(codeql, features);
t.deepEqual(result, CSHARP_EXTRA_PATTERNS);
});
});
test("getCsharpHashPatterns - returns undefined if neither base nor extra patterns match", async (t) => {
const codeql = createStubCodeQL({});
const features = createFeatures([Feature.CsharpNewCacheKey]);
const makePatternCheckStub = sinon.stub(internal, "makePatternCheck");
makePatternCheckStub.withArgs(CSHARP_BASE_PATTERNS).resolves(undefined);
makePatternCheckStub.withArgs(CSHARP_EXTRA_PATTERNS).resolves(undefined);
await t.notThrowsAsync(async () => {
const result = await getCsharpHashPatterns(codeql, features);
t.deepEqual(result, undefined);
});
});
test("checkHashPatterns - logs when no patterns match", async (t) => {
const codeql = createStubCodeQL({});
const features = createFeatures([]);
const messages: LoggedMessage[] = [];
const config: CacheConfig = {
getDependencyPaths: () => [],
getHashPatterns: async () => undefined,
};
const result = await checkHashPatterns(
codeql,
features,
KnownLanguage.csharp,
config,
"download",
getRecordingLogger(messages),
);
t.is(result, undefined);
checkExpectedLogMessages(t, messages, [
"Skipping download of dependency cache",
]);
});
test("checkHashPatterns - returns patterns when patterns match", async (t) => {
await withTmpDir(async (tmpDir) => {
const codeql = createStubCodeQL({});
const features = createFeatures([]);
const messages: LoggedMessage[] = [];
const patterns = makeAbsolutePatterns(tmpDir, ["**/*.cs", "**/*.java"]);
fs.writeFileSync(path.join(tmpDir, "test.java"), "");
const config: CacheConfig = {
getDependencyPaths: () => [],
getHashPatterns: async () => makePatternCheck(patterns),
};
const result = await checkHashPatterns(
codeql,
features,
KnownLanguage.csharp,
config,
"upload",
getRecordingLogger(messages),
);
t.deepEqual(result, patterns);
t.deepEqual(messages, []);
});
});
type RestoreCacheFunc = (
paths: string[],
primaryKey: string,
restoreKeys: string[] | undefined,
) => Promise<string | undefined>;
/**
* Constructs a function that `actionsCache.restoreCache` can be stubbed with.
*
* @param mockCacheKeys The keys of caches that we want to exist in the Actions cache.
*
* @returns Returns a function that `actionsCache.restoreCache` can be stubbed with.
*/
function makeMockCacheCheck(mockCacheKeys: string[]): RestoreCacheFunc {
return async (
_paths: string[],
primaryKey: string,
restoreKeys: string[] | undefined,
) => {
// The behaviour here mirrors what the real `restoreCache` would do:
// - Starting with the primary restore key, check all caches for a match:
// even for the primary restore key, this only has to be a prefix match.
// - If the primary restore key doesn't prefix-match any cache, then proceed
// in the same way for each restore key in turn.
for (const restoreKey of [primaryKey, ...(restoreKeys || [])]) {
for (const mockCacheKey of mockCacheKeys) {
if (mockCacheKey.startsWith(restoreKey)) {
return mockCacheKey;
}
}
}
// Only if no restore key matches any cache key prefix, there is no matching
// cache and we return `undefined`.
return undefined;
};
}
test("downloadDependencyCaches - does not restore caches with feature keys if no features are enabled", async (t) => {
process.env["RUNNER_OS"] = "Linux";
const codeql = createStubCodeQL({});
const messages: LoggedMessage[] = [];
const logger = getRecordingLogger(messages);
sinon.stub(glob, "hashFiles").resolves("abcdef");
const keyWithFeature = await cacheKey(
codeql,
createFeatures([Feature.CsharpNewCacheKey]),
KnownLanguage.csharp,
// Patterns don't matter here because we have stubbed `hashFiles` to always return a specific hash above.
[],
);
const restoreCacheStub = sinon
.stub(actionsCache, "restoreCache")
.callsFake(makeMockCacheCheck([keyWithFeature]));
const makePatternCheckStub = sinon.stub(internal, "makePatternCheck");
makePatternCheckStub
.withArgs(CSHARP_BASE_PATTERNS)
.resolves(CSHARP_BASE_PATTERNS);
makePatternCheckStub.withArgs(CSHARP_EXTRA_PATTERNS).resolves(undefined);
const result = await downloadDependencyCaches(
codeql,
createFeatures([]),
[KnownLanguage.csharp],
logger,
);
const statusReport = result.statusReport;
t.is(statusReport.length, 1);
t.is(statusReport[0].language, KnownLanguage.csharp);
t.is(statusReport[0].hit_kind, CacheHitKind.Miss);
t.deepEqual(result.restoredKeys, []);
t.assert(restoreCacheStub.calledOnce);
});
test("downloadDependencyCaches - restores caches with feature keys if features are enabled", async (t) => {
process.env["RUNNER_OS"] = "Linux";
const codeql = createStubCodeQL({});
const messages: LoggedMessage[] = [];
const logger = getRecordingLogger(messages);
const features = createFeatures([Feature.CsharpNewCacheKey]);
const mockHash = "abcdef";
sinon.stub(glob, "hashFiles").resolves(mockHash);
const keyWithFeature = await cacheKey(
codeql,
features,
KnownLanguage.csharp,
// Patterns don't matter here because we have stubbed `hashFiles` to always return a specific hash above.
[],
);
const restoreCacheStub = sinon
.stub(actionsCache, "restoreCache")
.callsFake(makeMockCacheCheck([keyWithFeature]));
const makePatternCheckStub = sinon.stub(internal, "makePatternCheck");
makePatternCheckStub
.withArgs(CSHARP_BASE_PATTERNS)
.resolves(CSHARP_BASE_PATTERNS);
makePatternCheckStub.withArgs(CSHARP_EXTRA_PATTERNS).resolves(undefined);
const result = await downloadDependencyCaches(
codeql,
features,
[KnownLanguage.csharp],
logger,
);
// Check that the status report for telemetry indicates that one cache was restored with an exact match.
const statusReport = result.statusReport;
t.is(statusReport.length, 1);
t.is(statusReport[0].language, KnownLanguage.csharp);
t.is(statusReport[0].hit_kind, CacheHitKind.Exact);
// Check that the restored key has been returned.
const restoredKeys = result.restoredKeys;
t.is(restoredKeys.length, 1);
t.assert(
restoredKeys[0].endsWith(mockHash),
"Expected restored key to end with hash returned by `hashFiles`",
);
// `restoreCache` should have been called exactly once.
t.assert(restoreCacheStub.calledOnce);
});
test("downloadDependencyCaches - restores caches with feature keys if features are enabled for partial matches", async (t) => {
process.env["RUNNER_OS"] = "Linux";
const codeql = createStubCodeQL({});
const messages: LoggedMessage[] = [];
const logger = getRecordingLogger(messages);
const features = createFeatures([Feature.CsharpNewCacheKey]);
// We expect two calls to `hashFiles`: the first by the call to `cacheKey` below,
// and the second by `downloadDependencyCaches`. We use the result of the first
// call as part of the cache key that identifies a mock, existing cache. The result
// of the second call is for the primary restore key, which we don't want to match
// the first key so that we can test the restore keys logic.
const restoredHash = "abcdef";
const hashFilesStub = sinon.stub(glob, "hashFiles");
hashFilesStub.onFirstCall().resolves(restoredHash);
hashFilesStub.onSecondCall().resolves("123456");
const keyWithFeature = await cacheKey(
codeql,
features,
KnownLanguage.csharp,
// Patterns don't matter here because we have stubbed `hashFiles` to always return a specific hash above.
[],
);
const restoreCacheStub = sinon
.stub(actionsCache, "restoreCache")
.callsFake(makeMockCacheCheck([keyWithFeature]));
const makePatternCheckStub = sinon.stub(internal, "makePatternCheck");
makePatternCheckStub
.withArgs(CSHARP_BASE_PATTERNS)
.resolves(CSHARP_BASE_PATTERNS);
makePatternCheckStub.withArgs(CSHARP_EXTRA_PATTERNS).resolves(undefined);
const result = await downloadDependencyCaches(
codeql,
features,
[KnownLanguage.csharp],
logger,
);
// Check that the status report for telemetry indicates that one cache was restored with a partial match.
const statusReport = result.statusReport;
t.is(statusReport.length, 1);
t.is(statusReport[0].language, KnownLanguage.csharp);
t.is(statusReport[0].hit_kind, CacheHitKind.Partial);
// Check that the restored key has been returned.
const restoredKeys = result.restoredKeys;
t.is(restoredKeys.length, 1);
t.assert(
restoredKeys[0].endsWith(restoredHash),
"Expected restored key to end with hash returned by `hashFiles`",
);
t.assert(restoreCacheStub.calledOnce);
});
test("uploadDependencyCaches - skips upload for a language with no cache config", async (t) => {
const codeql = createStubCodeQL({});
const messages: LoggedMessage[] = [];
const logger = getRecordingLogger(messages);
const features = createFeatures([]);
const config = createTestConfig({
languages: [KnownLanguage.actions],
});
const result = await uploadDependencyCaches(codeql, features, config, logger);
t.is(result.length, 0);
checkExpectedLogMessages(t, messages, [
"Skipping upload of dependency cache for actions",
]);
});
test("uploadDependencyCaches - skips upload if no files for the hash exist", async (t) => {
const codeql = createStubCodeQL({});
const messages: LoggedMessage[] = [];
const logger = getRecordingLogger(messages);
const features = createFeatures([]);
const config = createTestConfig({
languages: [KnownLanguage.go],
});
const makePatternCheckStub = sinon.stub(internal, "makePatternCheck");
makePatternCheckStub.resolves(undefined);
const result = await uploadDependencyCaches(codeql, features, config, logger);
t.is(result.length, 1);
t.is(result[0].language, KnownLanguage.go);
t.is(result[0].result, CacheStoreResult.NoHash);
});
test("uploadDependencyCaches - skips upload if we know the cache already exists", async (t) => {
process.env["RUNNER_OS"] = "Linux";
const codeql = createStubCodeQL({});
const messages: LoggedMessage[] = [];
const logger = getRecordingLogger(messages);
const features = createFeatures([]);
const mockHash = "abcdef";
sinon.stub(glob, "hashFiles").resolves(mockHash);
const makePatternCheckStub = sinon.stub(internal, "makePatternCheck");
makePatternCheckStub
.withArgs(CSHARP_BASE_PATTERNS)
.resolves(CSHARP_BASE_PATTERNS);
const primaryCacheKey = await cacheKey(
codeql,
features,
KnownLanguage.csharp,
CSHARP_BASE_PATTERNS,
);
const config = createTestConfig({
languages: [KnownLanguage.csharp],
dependencyCachingRestoredKeys: [primaryCacheKey],
});
const result = await uploadDependencyCaches(codeql, features, config, logger);
t.is(result.length, 1);
t.is(result[0].language, KnownLanguage.csharp);
t.is(result[0].result, CacheStoreResult.Duplicate);
});
test("uploadDependencyCaches - skips upload if cache size is 0", async (t) => {
process.env["RUNNER_OS"] = "Linux";
const codeql = createStubCodeQL({});
const messages: LoggedMessage[] = [];
const logger = getRecordingLogger(messages);
const features = createFeatures([]);
const mockHash = "abcdef";
sinon.stub(glob, "hashFiles").resolves(mockHash);
const makePatternCheckStub = sinon.stub(internal, "makePatternCheck");
makePatternCheckStub
.withArgs(CSHARP_BASE_PATTERNS)
.resolves(CSHARP_BASE_PATTERNS);
sinon.stub(cachingUtils, "getTotalCacheSize").resolves(0);
const config = createTestConfig({
languages: [KnownLanguage.csharp],
});
const result = await uploadDependencyCaches(codeql, features, config, logger);
t.is(result.length, 1);
t.is(result[0].language, KnownLanguage.csharp);
t.is(result[0].result, CacheStoreResult.Empty);
checkExpectedLogMessages(t, messages, [
"Skipping upload of dependency cache",
]);
});
test("uploadDependencyCaches - uploads caches when all requirements are met", async (t) => {
process.env["RUNNER_OS"] = "Linux";
const codeql = createStubCodeQL({});
const messages: LoggedMessage[] = [];
const logger = getRecordingLogger(messages);
const features = createFeatures([]);
const mockHash = "abcdef";
sinon.stub(glob, "hashFiles").resolves(mockHash);
const makePatternCheckStub = sinon.stub(internal, "makePatternCheck");
makePatternCheckStub
.withArgs(CSHARP_BASE_PATTERNS)
.resolves(CSHARP_BASE_PATTERNS);
sinon.stub(cachingUtils, "getTotalCacheSize").resolves(1024);
sinon.stub(actionsCache, "saveCache").resolves();
const config = createTestConfig({
languages: [KnownLanguage.csharp],
});
const result = await uploadDependencyCaches(codeql, features, config, logger);
t.is(result.length, 1);
t.is(result[0].language, KnownLanguage.csharp);
t.is(result[0].result, CacheStoreResult.Stored);
t.is(result[0].upload_size_bytes, 1024);
checkExpectedLogMessages(t, messages, ["Uploading cache of size"]);
});
test("uploadDependencyCaches - catches `ReserveCacheError` exceptions", async (t) => {
process.env["RUNNER_OS"] = "Linux";
const codeql = createStubCodeQL({});
const messages: LoggedMessage[] = [];
const logger = getRecordingLogger(messages);
const features = createFeatures([]);
const mockHash = "abcdef";
sinon.stub(glob, "hashFiles").resolves(mockHash);
const makePatternCheckStub = sinon.stub(internal, "makePatternCheck");
makePatternCheckStub
.withArgs(CSHARP_BASE_PATTERNS)
.resolves(CSHARP_BASE_PATTERNS);
sinon.stub(cachingUtils, "getTotalCacheSize").resolves(1024);
sinon
.stub(actionsCache, "saveCache")
.throws(new actionsCache.ReserveCacheError("Already in use"));
const config = createTestConfig({
languages: [KnownLanguage.csharp],
});
await t.notThrowsAsync(async () => {
const result = await uploadDependencyCaches(
codeql,
features,
config,
logger,
);
t.is(result.length, 1);
t.is(result[0].language, KnownLanguage.csharp);
t.is(result[0].result, CacheStoreResult.Duplicate);
checkExpectedLogMessages(t, messages, ["Not uploading cache for"]);
});
});
test("uploadDependencyCaches - throws other exceptions", async (t) => {
process.env["RUNNER_OS"] = "Linux";
const codeql = createStubCodeQL({});
const messages: LoggedMessage[] = [];
const logger = getRecordingLogger(messages);
const features = createFeatures([]);
const mockHash = "abcdef";
sinon.stub(glob, "hashFiles").resolves(mockHash);
const makePatternCheckStub = sinon.stub(internal, "makePatternCheck");
makePatternCheckStub
.withArgs(CSHARP_BASE_PATTERNS)
.resolves(CSHARP_BASE_PATTERNS);
sinon.stub(cachingUtils, "getTotalCacheSize").resolves(1024);
sinon.stub(actionsCache, "saveCache").throws();
const config = createTestConfig({
languages: [KnownLanguage.csharp],
});
await t.throwsAsync(async () => {
await uploadDependencyCaches(codeql, features, config, logger);
});
});
test("getFeaturePrefix - returns empty string if no features are enabled", async (t) => {
const codeql = createStubCodeQL({});
const features = createFeatures([]);
for (const knownLanguage of Object.values(KnownLanguage)) {
const result = await getFeaturePrefix(codeql, features, knownLanguage);
t.deepEqual(result, "", `Expected no feature prefix for ${knownLanguage}`);
}
});
test("getFeaturePrefix - Java - returns 'minify-' if JavaMinimizeDependencyJars is enabled", async (t) => {
const codeql = createStubCodeQL({});
const features = createFeatures([Feature.JavaMinimizeDependencyJars]);
const result = await getFeaturePrefix(codeql, features, KnownLanguage.java);
t.deepEqual(result, "minify-");
});
test("getFeaturePrefix - non-Java - returns '' if JavaMinimizeDependencyJars is enabled", async (t) => {
const codeql = createStubCodeQL({});
const features = createFeatures([Feature.JavaMinimizeDependencyJars]);
for (const knownLanguage of Object.values(KnownLanguage)) {
// Skip Java since we expect a result for it, which is tested in the previous test.
if (knownLanguage === KnownLanguage.java) {
continue;
}
const result = await getFeaturePrefix(codeql, features, knownLanguage);
t.deepEqual(result, "", `Expected no feature prefix for ${knownLanguage}`);
}
});
test("getFeaturePrefix - C# - returns prefix if CsharpNewCacheKey is enabled", async (t) => {
const codeql = createStubCodeQL({});
const features = createFeatures([Feature.CsharpNewCacheKey]);
const result = await getFeaturePrefix(codeql, features, KnownLanguage.csharp);
t.notDeepEqual(result, "");
t.assert(result.endsWith("-"));
// Check the length of the prefix, which should correspond to `cacheKeyHashLength` + 1 for the trailing `-`.
t.is(result.length, cacheKeyHashLength + 1);
});
test("getFeaturePrefix - non-C# - returns '' if CsharpNewCacheKey is enabled", async (t) => {
const codeql = createStubCodeQL({});
const features = createFeatures([Feature.CsharpNewCacheKey]);
for (const knownLanguage of Object.values(KnownLanguage)) {
// Skip C# since we expect a result for it, which is tested in the previous test.
if (knownLanguage === KnownLanguage.csharp) {
continue;
}
const result = await getFeaturePrefix(codeql, features, knownLanguage);
t.deepEqual(result, "", `Expected no feature prefix for ${knownLanguage}`);
}
});

View File

@@ -6,9 +6,11 @@ import * as glob from "@actions/glob";
import { getTemporaryDirectory } from "./actions-util";
import { listActionsCaches } from "./api-client";
import { getTotalCacheSize } from "./caching-utils";
import { createCacheKeyHash, getTotalCacheSize } from "./caching-utils";
import { CodeQL } from "./codeql";
import { Config } from "./config-utils";
import { EnvVar } from "./environment";
import { Feature, FeatureEnablement } from "./feature-flags";
import { KnownLanguage, Language } from "./languages";
import { Logger } from "./logging";
import { getErrorMessage, getRequiredEnvParam } from "./util";
@@ -16,15 +18,21 @@ import { getErrorMessage, getRequiredEnvParam } from "./util";
/**
* Caching configuration for a particular language.
*/
interface CacheConfig {
/** The paths of directories on the runner that should be included in the cache. */
paths: string[];
export interface CacheConfig {
/** Gets the paths of directories on the runner that should be included in the cache. */
getDependencyPaths: () => string[];
/**
* Patterns for the paths of files whose contents affect which dependencies are used
* by a project. We find all files which match these patterns, calculate a hash for
* their contents, and use that hash as part of the cache key.
* Gets an array of glob patterns for the paths of files whose contents affect which dependencies are used
* by a project. This function also checks whether there are any matching files and returns
* `undefined` if no files match.
*
* The glob patterns are intended to be used for cache keys, where we find all files which match these
* patterns, calculate a hash for their contents, and use that hash as part of the cache key.
*/
hash: string[];
getHashPatterns: (
codeql: CodeQL,
features: FeatureEnablement,
) => Promise<string[] | undefined>;
}
const CODEQL_DEPENDENCY_CACHE_PREFIX = "codeql-dependencies";
@@ -39,21 +47,105 @@ export function getJavaTempDependencyDir(): string {
return join(getTemporaryDirectory(), "codeql_java", "repository");
}
/**
* Returns an array of paths of directories on the runner that should be included in a dependency cache
* for a Java analysis. It is important that this is a function, because we call `getTemporaryDirectory`
* which would otherwise fail in tests if we haven't had a chance to initialise `RUNNER_TEMP`.
*
* @returns The paths of directories on the runner that should be included in a dependency cache
* for a Java analysis.
*/
function getJavaDependencyDirs(): string[] {
return [
// Maven
join(os.homedir(), ".m2", "repository"),
// Gradle
join(os.homedir(), ".gradle", "caches"),
// CodeQL Java build-mode: none
getJavaTempDependencyDir(),
];
}
/**
* Checks that there are files which match `patterns`. If there are matching files for any of the patterns,
* this function returns all `patterns`. Otherwise, `undefined` is returned.
*
* @param patterns The glob patterns to find matching files for.
* @returns The array of glob patterns if there are matching files, or `undefined` otherwise.
*/
export async function makePatternCheck(
patterns: string[],
): Promise<string[] | undefined> {
const globber = await makeGlobber(patterns);
if ((await globber.glob()).length === 0) {
return undefined;
}
return patterns;
}
/** These files contain accurate information about dependencies, including the exact versions
* that the relevant package manager has determined for the project. Using these gives us
* stable hashes unless the dependencies change.
*/
export const CSHARP_BASE_PATTERNS = [
// NuGet
"**/packages.lock.json",
// Paket
"**/paket.lock",
];
/** These are less accurate for use in cache key calculations, because they:
*
* - Don't contain the exact versions used. They may only contain version ranges or none at all.
* - They contain information unrelated to dependencies, which we don't care about.
*
* As a result, the hash we compute from these files may change, even if
* the dependencies haven't changed.
*/
export const CSHARP_EXTRA_PATTERNS = [
"**/*.csproj",
"**/packages.config",
"**/nuget.config",
];
/**
* Returns the list of glob patterns that should be used to calculate the cache key hash
* for a C# dependency cache. This will try to use `CSHARP_BASE_PATTERNS` whenever possible.
* As a fallback, it will also use `CSHARP_EXTRA_PATTERNS` if the corresponding FF is enabled.
*
* @param codeql The CodeQL instance to use.
* @param features Information about which FFs are enabled.
* @returns A list of glob patterns to use for hashing.
*/
export async function getCsharpHashPatterns(
codeql: CodeQL,
features: FeatureEnablement,
): Promise<string[] | undefined> {
const basePatterns = await internal.makePatternCheck(CSHARP_BASE_PATTERNS);
if (basePatterns !== undefined) {
return basePatterns;
}
if (await features.getValue(Feature.CsharpNewCacheKey, codeql)) {
return internal.makePatternCheck(CSHARP_EXTRA_PATTERNS);
}
// If we get to this point, we didn't find any files with `CSHARP_BASE_PATTERNS`,
// and `Feature.CsharpNewCacheKey` is not enabled.
return undefined;
}
/**
* Default caching configurations per language.
*/
function getDefaultCacheConfig(): { [language: string]: CacheConfig } {
return {
java: {
paths: [
// Maven
join(os.homedir(), ".m2", "repository"),
// Gradle
join(os.homedir(), ".gradle", "caches"),
// CodeQL Java build-mode: none
getJavaTempDependencyDir(),
],
hash: [
const defaultCacheConfigs: { [language: string]: CacheConfig } = {
java: {
getDependencyPaths: getJavaDependencyDirs,
getHashPatterns: async () =>
internal.makePatternCheck([
// Maven
"**/pom.xml",
// Gradle
@@ -63,23 +155,17 @@ function getDefaultCacheConfig(): { [language: string]: CacheConfig } {
"buildSrc/**/Dependencies.kt",
"gradle/*.versions.toml",
"**/versions.properties",
],
},
csharp: {
paths: [join(os.homedir(), ".nuget", "packages")],
hash: [
// NuGet
"**/packages.lock.json",
// Paket
"**/paket.lock",
],
},
go: {
paths: [join(os.homedir(), "go", "pkg", "mod")],
hash: ["**/go.sum"],
},
};
}
]),
},
csharp: {
getDependencyPaths: () => [join(os.homedir(), ".nuget", "packages")],
getHashPatterns: getCsharpHashPatterns,
},
go: {
getDependencyPaths: () => [join(os.homedir(), "go", "pkg", "mod")],
getHashPatterns: async () => internal.makePatternCheck(["**/go.sum"]),
},
};
async function makeGlobber(patterns: string[]): Promise<glob.Globber> {
return glob.create(patterns.join("\n"));
@@ -107,23 +193,66 @@ export interface DependencyCacheRestoreStatus {
/** An array of `DependencyCacheRestoreStatus` objects for each analysed language with a caching configuration. */
export type DependencyCacheRestoreStatusReport = DependencyCacheRestoreStatus[];
/** Represents the results of `downloadDependencyCaches`. */
export interface DownloadDependencyCachesResult {
/** The status report for telemetry */
statusReport: DependencyCacheRestoreStatusReport;
/** An array of cache keys that we have restored and therefore know to exist. */
restoredKeys: string[];
}
/**
* A wrapper around `cacheConfig.getHashPatterns` which logs when there are no files to calculate
* a hash for the cache key from.
*
* @param codeql The CodeQL instance to use.
* @param features Information about which FFs are enabled.
* @param language The language the `CacheConfig` is for. For use in the log message.
* @param cacheConfig The caching configuration to call `getHashPatterns` on.
* @param checkType Whether we are checking the patterns for a download or upload.
* @param logger The logger to write the log message to if there is an error.
* @returns An array of glob patterns to use for hashing files, or `undefined` if there are no matching files.
*/
export async function checkHashPatterns(
codeql: CodeQL,
features: FeatureEnablement,
language: Language,
cacheConfig: CacheConfig,
checkType: "download" | "upload",
logger: Logger,
): Promise<string[] | undefined> {
const patterns = await cacheConfig.getHashPatterns(codeql, features);
if (patterns === undefined) {
logger.info(
`Skipping ${checkType} of dependency cache for ${language} as we cannot calculate a hash for the cache key.`,
);
}
return patterns;
}
/**
* Attempts to restore dependency caches for the languages being analyzed.
*
* @param codeql The CodeQL instance to use.
* @param features Information about which FFs are enabled.
* @param languages The languages being analyzed.
* @param logger A logger to record some informational messages to.
* @param minimizeJavaJars Whether the Java extractor should rewrite downloaded JARs to minimize their size.
*
* @returns An array of `DependencyCacheRestoreStatus` objects for each analysed language with a caching configuration.
*/
export async function downloadDependencyCaches(
codeql: CodeQL,
features: FeatureEnablement,
languages: Language[],
logger: Logger,
minimizeJavaJars: boolean,
): Promise<DependencyCacheRestoreStatusReport> {
): Promise<DownloadDependencyCachesResult> {
const status: DependencyCacheRestoreStatusReport = [];
const restoredKeys: string[] = [];
for (const language of languages) {
const cacheConfig = getDefaultCacheConfig()[language];
const cacheConfig = defaultCacheConfigs[language];
if (cacheConfig === undefined) {
logger.info(
@@ -134,19 +263,22 @@ export async function downloadDependencyCaches(
// Check that we can find files to calculate the hash for the cache key from, so we don't end up
// with an empty string.
const globber = await makeGlobber(cacheConfig.hash);
if ((await globber.glob()).length === 0) {
const patterns = await checkHashPatterns(
codeql,
features,
language,
cacheConfig,
"download",
logger,
);
if (patterns === undefined) {
status.push({ language, hit_kind: CacheHitKind.NoHash });
logger.info(
`Skipping download of dependency cache for ${language} as we cannot calculate a hash for the cache key.`,
);
continue;
}
const primaryKey = await cacheKey(language, cacheConfig, minimizeJavaJars);
const primaryKey = await cacheKey(codeql, features, language, patterns);
const restoreKeys: string[] = [
await cachePrefix(language, minimizeJavaJars),
await cachePrefix(codeql, features, language),
];
logger.info(
@@ -157,7 +289,7 @@ export async function downloadDependencyCaches(
const start = performance.now();
const hitKey = await actionsCache.restoreCache(
cacheConfig.paths,
cacheConfig.getDependencyPaths(),
primaryKey,
restoreKeys,
);
@@ -165,16 +297,27 @@ export async function downloadDependencyCaches(
if (hitKey !== undefined) {
logger.info(`Cache hit on key ${hitKey} for ${language}.`);
const hit_kind =
hitKey === primaryKey ? CacheHitKind.Exact : CacheHitKind.Partial;
status.push({ language, hit_kind, download_duration_ms });
// We have a partial cache hit, unless the key of the restored cache matches the
// primary restore key.
let hit_kind = CacheHitKind.Partial;
if (hitKey === primaryKey) {
hit_kind = CacheHitKind.Exact;
}
status.push({
language,
hit_kind,
download_duration_ms,
});
restoredKeys.push(hitKey);
} else {
status.push({ language, hit_kind: CacheHitKind.Miss });
logger.info(`No suitable cache found for ${language}.`);
}
}
return status;
return { statusReport: status, restoredKeys };
}
/** Enumerates possible outcomes for storing caches. */
@@ -203,20 +346,22 @@ export type DependencyCacheUploadStatusReport = DependencyCacheUploadStatus[];
/**
* Attempts to store caches for the languages that were analyzed.
*
* @param codeql The CodeQL instance to use.
* @param features Information about which FFs are enabled.
* @param config The configuration for this workflow.
* @param logger A logger to record some informational messages to.
* @param minimizeJavaJars Whether the Java extractor should rewrite downloaded JARs to minimize their size.
*
* @returns An array of `DependencyCacheUploadStatus` objects for each analysed language with a caching configuration.
*/
export async function uploadDependencyCaches(
codeql: CodeQL,
features: FeatureEnablement,
config: Config,
logger: Logger,
minimizeJavaJars: boolean,
): Promise<DependencyCacheUploadStatusReport> {
const status: DependencyCacheUploadStatusReport = [];
for (const language of config.languages) {
const cacheConfig = getDefaultCacheConfig()[language];
const cacheConfig = defaultCacheConfigs[language];
if (cacheConfig === undefined) {
logger.info(
@@ -227,13 +372,28 @@ export async function uploadDependencyCaches(
// Check that we can find files to calculate the hash for the cache key from, so we don't end up
// with an empty string.
const globber = await makeGlobber(cacheConfig.hash);
if ((await globber.glob()).length === 0) {
const patterns = await checkHashPatterns(
codeql,
features,
language,
cacheConfig,
"upload",
logger,
);
if (patterns === undefined) {
status.push({ language, result: CacheStoreResult.NoHash });
logger.info(
`Skipping upload of dependency cache for ${language} as we cannot calculate a hash for the cache key.`,
);
continue;
}
// Now that we have verified that there are suitable files, compute the hash for the cache key.
const key = await cacheKey(codeql, features, language, patterns);
// Check that we haven't previously restored this exact key. If a cache with this key
// already exists in the Actions Cache, performing the next steps is pointless as the cache
// will not get overwritten. We can therefore skip the expensive work of measuring the size
// of the cache contents and attempting to upload it if we know that the cache already exists.
if (config.dependencyCachingRestoredKeys.includes(key)) {
status.push({ language, result: CacheStoreResult.Duplicate });
continue;
}
@@ -247,7 +407,11 @@ export async function uploadDependencyCaches(
// use the cache quota that we compete with. In that case, we do not wish to use up all of the quota
// with the dependency caches. For this, we could use the Cache API to check whether other workflows
// are using the quota and how full it is.
const size = await getTotalCacheSize(cacheConfig.paths, logger, true);
const size = await getTotalCacheSize(
cacheConfig.getDependencyPaths(),
logger,
true,
);
// Skip uploading an empty cache.
if (size === 0) {
@@ -258,15 +422,13 @@ export async function uploadDependencyCaches(
continue;
}
const key = await cacheKey(language, cacheConfig, minimizeJavaJars);
logger.info(
`Uploading cache of size ${size} for ${language} with key ${key}...`,
);
try {
const start = performance.now();
await actionsCache.saveCache(cacheConfig.paths, key);
await actionsCache.saveCache(cacheConfig.getDependencyPaths(), key);
const upload_duration_ms = Math.round(performance.now() - start);
status.push({
@@ -299,31 +461,86 @@ export async function uploadDependencyCaches(
/**
* Computes a cache key for the specified language.
*
* @param codeql The CodeQL instance to use.
* @param features Information about which FFs are enabled.
* @param language The language being analyzed.
* @param cacheConfig The cache configuration for the language.
* @param minimizeJavaJars Whether the Java extractor should rewrite downloaded JARs to minimize their size.
* @param patterns The file patterns to hash.
*
* @returns A cache key capturing information about the project(s) being analyzed in the specified language.
*/
async function cacheKey(
export async function cacheKey(
codeql: CodeQL,
features: FeatureEnablement,
language: Language,
cacheConfig: CacheConfig,
minimizeJavaJars: boolean = false,
patterns: string[],
): Promise<string> {
const hash = await glob.hashFiles(cacheConfig.hash.join("\n"));
return `${await cachePrefix(language, minimizeJavaJars)}${hash}`;
const hash = await glob.hashFiles(patterns.join("\n"));
return `${await cachePrefix(codeql, features, language)}${hash}`;
}
/**
* If experimental features which the cache contents depend on are enabled for the current language,
* this function returns a prefix that uniquely identifies the set of enabled features. The purpose of
* this is to avoid restoring caches whose contents depended on experimental features, if those
* experimental features are later disabled.
*
* @param codeql The CodeQL instance.
* @param features Information about enabled features.
* @param language The language we are creating the key for.
*
* @returns A cache key prefix identifying the enabled, experimental features that the cache depends on.
*/
export async function getFeaturePrefix(
codeql: CodeQL,
features: FeatureEnablement,
language: Language,
): Promise<string> {
const enabledFeatures: Feature[] = [];
const addFeatureIfEnabled = async (feature: Feature) => {
if (await features.getValue(feature, codeql)) {
enabledFeatures.push(feature);
}
};
if (language === KnownLanguage.java) {
// To ensure a safe rollout of JAR minimization, we change the key when the feature is enabled.
const minimizeJavaJars = await features.getValue(
Feature.JavaMinimizeDependencyJars,
codeql,
);
// To maintain backwards compatibility with this, we return "minify-" instead of a hash.
if (minimizeJavaJars) {
return "minify-";
}
} else if (language === KnownLanguage.csharp) {
await addFeatureIfEnabled(Feature.CsharpNewCacheKey);
}
// If any features that affect the cache are enabled, return a feature prefix by
// computing a hash of the feature array.
if (enabledFeatures.length > 0) {
return `${createCacheKeyHash(enabledFeatures)}-`;
}
// No feature prefix.
return "";
}
/**
* Constructs a prefix for the cache key, comprised of a CodeQL-specific prefix, a version number that
* can be changed to invalidate old caches, the runner's operating system, and the specified language name.
*
* @param codeql The CodeQL instance to use.
* @param features Information about which FFs are enabled.
* @param language The language being analyzed.
* @param minimizeJavaJars Whether the Java extractor should rewrite downloaded JARs to minimize their size.
* @returns The prefix that identifies what a cache is for.
*/
async function cachePrefix(
codeql: CodeQL,
features: FeatureEnablement,
language: Language,
minimizeJavaJars: boolean,
): Promise<string> {
const runnerOs = getRequiredEnvParam("RUNNER_OS");
const customPrefix = process.env[EnvVar.DEPENDENCY_CACHING_PREFIX];
@@ -333,12 +550,18 @@ async function cachePrefix(
prefix = `${prefix}-${customPrefix}`;
}
// To ensure a safe rollout of JAR minimization, we change the key when the feature is enabled.
if (language === KnownLanguage.java && minimizeJavaJars) {
prefix = `minify-${prefix}`;
}
// Calculate the feature prefix for the cache, if any. This is a hash that identifies
// experimental features that affect the cache contents.
const featurePrefix = await getFeaturePrefix(codeql, features, language);
return `${prefix}-${CODEQL_DEPENDENCY_CACHE_VERSION}-${runnerOs}-${language}-`;
// Assemble the cache key. For backwards compatibility with the JAR minification experiment's existing
// feature prefix usage, we add that feature prefix at the start. Other feature prefixes are inserted
// after the general CodeQL dependency cache prefix.
if (featurePrefix === "minify-") {
return `${featurePrefix}${prefix}-${CODEQL_DEPENDENCY_CACHE_VERSION}-${runnerOs}-${language}-`;
} else {
return `${prefix}-${featurePrefix}${CODEQL_DEPENDENCY_CACHE_VERSION}-${runnerOs}-${language}-`;
}
}
/** Represents information about our overall cache usage for CodeQL dependency caches. */
@@ -371,3 +594,7 @@ export async function getDependencyCacheUsage(
return undefined;
}
export const internal = {
makePatternCheck,
};

View File

@@ -20,12 +20,6 @@ export enum EnvVar {
/** Whether the CodeQL Action has invoked the Go autobuilder. */
DID_AUTOBUILD_GOLANG = "CODEQL_ACTION_DID_AUTOBUILD_GOLANG",
/**
* Whether to disable the SARIF post-processing in the Action that removes duplicate locations from
* notifications in the `run[].invocations[].toolExecutionNotifications` SARIF property.
*/
DISABLE_DUPLICATE_LOCATION_FIX = "CODEQL_ACTION_DISABLE_DUPLICATE_LOCATION_FIX",
/**
* Whether the CodeQL Action is using its own deprecated and non-standard way of scanning for
* multiple languages.
@@ -56,20 +50,12 @@ export enum EnvVar {
/** Whether the error for a deprecated version of the CodeQL Action was logged. */
LOG_VERSION_DEPRECATION = "CODEQL_ACTION_DID_LOG_VERSION_DEPRECATION",
/**
* For macOS. Result of `csrutil status` to determine whether System Integrity
* Protection is enabled.
*/
IS_SIP_ENABLED = "CODEQL_ACTION_IS_SIP_ENABLED",
/** UUID representing the current job run. */
JOB_RUN_UUID = "JOB_RUN_UUID",
/** Status for the entire job, submitted to the status report in `init-post` */
JOB_STATUS = "CODEQL_ACTION_JOB_STATUS",
ODASA_TRACER_CONFIGURATION = "ODASA_TRACER_CONFIGURATION",
/** The value of the `output` input for the analyze action. */
SARIF_RESULTS_OUTPUT_DIR = "CODEQL_ACTION_SARIF_RESULTS_OUTPUT_DIR",

View File

@@ -47,6 +47,7 @@ export enum Feature {
AnalyzeUseNewUpload = "analyze_use_new_upload",
CleanupTrapCaches = "cleanup_trap_caches",
CppDependencyInstallation = "cpp_dependency_installation_enabled",
CsharpNewCacheKey = "csharp_new_cache_key",
DiffInformedQueries = "diff_informed_queries",
DisableCsharpBuildless = "disable_csharp_buildless",
DisableJavaBuildlessEnabled = "disable_java_buildless_enabled",
@@ -76,6 +77,7 @@ export enum Feature {
OverlayAnalysisSwift = "overlay_analysis_swift",
PythonDefaultIsToNotExtractStdlib = "python_default_is_to_not_extract_stdlib",
QaTelemetryEnabled = "qa_telemetry_enabled",
UploadOverlayDbToApi = "upload_overlay_db_to_api",
UseRepositoryProperties = "use_repository_properties",
ValidateDbConfig = "validate_db_config",
}
@@ -132,6 +134,11 @@ export const featureConfig: Record<
legacyApi: true,
minimumVersion: "2.15.0",
},
[Feature.CsharpNewCacheKey]: {
defaultValue: false,
envVar: "CODEQL_ACTION_CSHARP_NEW_CACHE_KEY",
minimumVersion: undefined,
},
[Feature.DiffInformedQueries]: {
defaultValue: true,
envVar: "CODEQL_ACTION_DIFF_INFORMED_QUERIES",
@@ -160,6 +167,11 @@ export const featureConfig: Record<
legacyApi: true,
minimumVersion: undefined,
},
[Feature.JavaMinimizeDependencyJars]: {
defaultValue: false,
envVar: "CODEQL_ACTION_JAVA_MINIMIZE_DEPENDENCY_JARS",
minimumVersion: "2.23.0",
},
[Feature.OverlayAnalysis]: {
defaultValue: false,
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS",
@@ -271,21 +283,21 @@ export const featureConfig: Record<
minimumVersion: undefined,
toolsFeature: ToolsFeature.PythonDefaultIsToNotExtractStdlib,
},
[Feature.UseRepositoryProperties]: {
defaultValue: false,
envVar: "CODEQL_ACTION_USE_REPOSITORY_PROPERTIES",
minimumVersion: undefined,
},
[Feature.QaTelemetryEnabled]: {
defaultValue: false,
envVar: "CODEQL_ACTION_QA_TELEMETRY",
legacyApi: true,
minimumVersion: undefined,
},
[Feature.JavaMinimizeDependencyJars]: {
[Feature.UploadOverlayDbToApi]: {
defaultValue: false,
envVar: "CODEQL_ACTION_JAVA_MINIMIZE_DEPENDENCY_JARS",
minimumVersion: "2.23.0",
envVar: "CODEQL_ACTION_UPLOAD_OVERLAY_DB_TO_API",
minimumVersion: undefined,
},
[Feature.UseRepositoryProperties]: {
defaultValue: false,
envVar: "CODEQL_ACTION_USE_REPOSITORY_PROPERTIES",
minimumVersion: undefined,
},
[Feature.ValidateDbConfig]: {
defaultValue: false,

View File

@@ -122,67 +122,6 @@ export const determineBaseBranchHeadCommitOid = async function (
}
};
/**
* Deepen the git history of HEAD by one level. Errors are logged.
*
* This function uses the `checkout_path` to determine the repository path and
* works only when called from `analyze` or `upload-sarif`.
*/
export const deepenGitHistory = async function () {
try {
await runGitCommand(
getOptionalInput("checkout_path"),
[
"fetch",
"origin",
"HEAD",
"--no-tags",
"--no-recurse-submodules",
"--deepen=1",
],
"Cannot deepen the shallow repository.",
);
} catch {
// Errors are already logged by runGitCommand()
}
};
/**
* Fetch the given remote branch. Errors are logged.
*
* This function uses the `checkout_path` to determine the repository path and
* works only when called from `analyze` or `upload-sarif`.
*/
export const gitFetch = async function (branch: string, extraFlags: string[]) {
try {
await runGitCommand(
getOptionalInput("checkout_path"),
["fetch", "--no-tags", ...extraFlags, "origin", `${branch}:${branch}`],
`Cannot fetch ${branch}.`,
);
} catch {
// Errors are already logged by runGitCommand()
}
};
/**
* Repack the git repository, using with the given flags. Errors are logged.
*
* This function uses the `checkout_path` to determine the repository path and
* works only when called from `analyze` or `upload-sarif`.
*/
export const gitRepack = async function (flags: string[]) {
try {
await runGitCommand(
getOptionalInput("checkout_path"),
["repack", ...flags],
"Cannot repack the repository.",
);
} catch {
// Errors are already logged by runGitCommand()
}
};
/**
* Decode, if necessary, a file path produced by Git. See
* https://git-scm.com/docs/git-config#Documentation/git-config.txt-corequotePath

View File

@@ -371,7 +371,7 @@ async function run() {
}
let overlayBaseDatabaseStats: OverlayBaseDatabaseDownloadStats | undefined;
let dependencyCachingResults: DependencyCacheRestoreStatusReport | undefined;
let dependencyCachingStatus: DependencyCacheRestoreStatusReport | undefined;
try {
if (
config.overlayDatabaseMode === OverlayDatabaseMode.Overlay &&
@@ -578,16 +578,16 @@ async function run() {
}
// Restore dependency cache(s), if they exist.
const minimizeJavaJars = await features.getValue(
Feature.JavaMinimizeDependencyJars,
codeql,
);
if (shouldRestoreCache(config.dependencyCachingEnabled)) {
dependencyCachingResults = await downloadDependencyCaches(
const dependencyCachingResult = await downloadDependencyCaches(
codeql,
features,
config.languages,
logger,
minimizeJavaJars,
);
dependencyCachingStatus = dependencyCachingResult.statusReport;
config.dependencyCachingRestoredKeys =
dependencyCachingResult.restoredKeys;
}
// Suppress warnings about disabled Python library extraction.
@@ -648,7 +648,7 @@ async function run() {
`${EnvVar.JAVA_EXTRACTOR_MINIMIZE_DEPENDENCY_JARS} is already set to '${process.env[EnvVar.JAVA_EXTRACTOR_MINIMIZE_DEPENDENCY_JARS]}', so the Action will not override it.`,
);
} else if (
minimizeJavaJars &&
(await features.getValue(Feature.JavaMinimizeDependencyJars, codeql)) &&
config.dependencyCachingEnabled &&
config.buildMode === BuildMode.None &&
config.languages.includes(KnownLanguage.java)
@@ -735,7 +735,7 @@ async function run() {
toolsSource,
toolsVersion,
overlayBaseDatabaseStats,
dependencyCachingResults,
dependencyCachingStatus,
logger,
error,
);
@@ -758,7 +758,7 @@ async function run() {
toolsSource,
toolsVersion,
overlayBaseDatabaseStats,
dependencyCachingResults,
dependencyCachingStatus,
logger,
);
}

View File

@@ -1,4 +1,3 @@
import * as crypto from "crypto";
import * as fs from "fs";
import * as path from "path";
@@ -11,11 +10,13 @@ import {
getWorkflowRunID,
} from "./actions-util";
import { getAutomationID } from "./api-client";
import { createCacheKeyHash } from "./caching-utils";
import { type CodeQL } from "./codeql";
import { type Config } from "./config-utils";
import { getCommitOid, getFileOidsUnderPath } from "./git-utils";
import { Logger, withGroupAsync } from "./logging";
import {
CleanupLevel,
getErrorMessage,
isInTestMode,
tryGetFolderBytes,
@@ -28,7 +29,7 @@ export enum OverlayDatabaseMode {
None = "none",
}
export const CODEQL_OVERLAY_MINIMUM_VERSION = "2.22.4";
export const CODEQL_OVERLAY_MINIMUM_VERSION = "2.23.5";
/**
* The maximum (uncompressed) size of the overlay base database that we will
@@ -175,7 +176,7 @@ const MAX_CACHE_OPERATION_MS = 600_000;
* @param warningPrefix Prefix for the check failure warning message
* @returns True if the verification succeeded, false otherwise
*/
export function checkOverlayBaseDatabase(
function checkOverlayBaseDatabase(
config: Config,
logger: Logger,
warningPrefix: string,
@@ -204,7 +205,7 @@ export function checkOverlayBaseDatabase(
* @returns A promise that resolves to true if the upload was performed and
* successfully completed, or false otherwise
*/
export async function uploadOverlayBaseDatabaseToCache(
export async function cleanupAndUploadOverlayBaseDatabaseToCache(
codeql: CodeQL,
config: Config,
logger: Logger,
@@ -242,7 +243,7 @@ export async function uploadOverlayBaseDatabaseToCache(
// Clean up the database using the overlay cleanup level.
await withGroupAsync("Cleaning up databases", async () => {
await codeql.databaseCleanupCluster(config, "overlay");
await codeql.databaseCleanupCluster(config, CleanupLevel.Overlay);
});
const dbLocation = config.dbLocation;
@@ -514,27 +515,3 @@ export async function getCacheRestoreKeyPrefix(
// easier to debug and understand the cache key structure.
return `${CACHE_PREFIX}-${CACHE_VERSION}-${componentsHash}-${languages}-${codeQlVersion}-`;
}
/**
* Creates a SHA-256 hash of the cache key components to ensure uniqueness
* while keeping the cache key length manageable.
*
* @param components Object containing all components that should influence cache key uniqueness
* @returns A short SHA-256 hash (first 16 characters) of the components
*/
function createCacheKeyHash(components: Record<string, any>): string {
// From https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/JSON/stringify
//
// "Properties are visited using the same algorithm as Object.keys(), which
// has a well-defined order and is stable across implementations. For example,
// JSON.stringify on the same object will always produce the same string, and
// JSON.parse(JSON.stringify(obj)) would produce an object with the same key
// ordering as the original (assuming the object is completely
// JSON-serializable)."
const componentsJson = JSON.stringify(components);
return crypto
.createHash("sha256")
.update(componentsJson)
.digest("hex")
.substring(0, 16);
}

View File

@@ -34,7 +34,7 @@ export enum ToolsSource {
Download = "DOWNLOAD",
}
export const CODEQL_DEFAULT_ACTION_REPOSITORY = "github/codeql-action";
const CODEQL_DEFAULT_ACTION_REPOSITORY = "github/codeql-action";
const CODEQL_NIGHTLIES_REPOSITORY_OWNER = "dsp-testing";
const CODEQL_NIGHTLIES_REPOSITORY_NAME = "codeql-cli-nightlies";
@@ -180,17 +180,6 @@ export function tryGetTagNameFromUrl(
return match[1];
}
export function tryGetBundleVersionFromUrl(
url: string,
logger: Logger,
): string | undefined {
const tagName = tryGetTagNameFromUrl(url, logger);
if (tagName === undefined) {
return undefined;
}
return tryGetBundleVersionFromTagName(tagName, logger);
}
export function convertToSemVer(version: string, logger: Logger): string {
if (!semver.valid(version)) {
logger.debug(
@@ -580,7 +569,7 @@ export async function getCodeQLSource(
* Gets a fallback version number to use when looking for CodeQL in the toolcache if we didn't find
* the `x.y.z` version. This is to support old versions of the toolcache.
*/
export async function tryGetFallbackToolcacheVersion(
async function tryGetFallbackToolcacheVersion(
cliVersion: string | undefined,
tagName: string,
logger: Logger,
@@ -729,7 +718,7 @@ function getCanonicalToolcacheVersion(
return cliVersion;
}
export interface SetupCodeQLResult {
interface SetupCodeQLResult {
codeqlFolder: string;
toolsDownloadStatusReport?: ToolsDownloadStatusReport;
toolsSource: ToolsSource;
@@ -750,7 +739,7 @@ export async function setupCodeQLBundle(
defaultCliVersion: CodeQLDefaultVersionInfo,
features: FeatureEnablement,
logger: Logger,
) {
): Promise<SetupCodeQLResult> {
if (!(await util.isBinaryAccessible("tar", logger))) {
throw new util.ConfigurationError(
"Could not find tar in PATH, so unable to extract CodeQL bundle.",

View File

@@ -8,7 +8,7 @@ import { ConfigurationError, getErrorMessage, isDefined } from "./util";
export const UPDATEJOB_PROXY = "update-job-proxy";
export const UPDATEJOB_PROXY_VERSION = "v2.0.20250624110901";
export const UPDATEJOB_PROXY_URL_PREFIX =
const UPDATEJOB_PROXY_URL_PREFIX =
"https://github.com/github/codeql-action/releases/download/codeql-bundle-v2.22.0/";
export type Credential = {
@@ -202,7 +202,7 @@ export function getFallbackUrl(proxyPackage: string): string {
*
* @returns The response from the GitHub API.
*/
export async function getLinkedRelease() {
async function getLinkedRelease() {
return getApiClient().rest.repos.getReleaseByTag({
owner: "github",
repo: "codeql-action",

View File

@@ -54,7 +54,7 @@ export enum ActionName {
* considered to be a third party analysis and is treated differently when calculating SLOs. To ensure
* misconfigured workflows are not treated as third party, only the upload-sarif action can return false.
*/
export function isFirstPartyAnalysis(actionName: ActionName): boolean {
function isFirstPartyAnalysis(actionName: ActionName): boolean {
if (actionName !== ActionName.UploadSarif) {
return true;
}

View File

@@ -392,6 +392,7 @@ export function createTestConfig(overrides: Partial<Config>): Config {
trapCaches: {},
trapCacheDownloadTime: 0,
dependencyCachingEnabled: CachingKind.None,
dependencyCachingRestoredKeys: [],
extraQueryExclusions: [],
overlayDatabaseMode: OverlayDatabaseMode.None,
useOverlayDatabaseCaching: false,

View File

@@ -17,7 +17,7 @@ import { cleanUpPath, getErrorMessage, getRequiredEnvParam } from "./util";
/**
* High watermark to use when streaming the download and extraction of the CodeQL tools.
*/
export const STREAMING_HIGH_WATERMARK_BYTES = 4 * 1024 * 1024; // 4 MiB
const STREAMING_HIGH_WATERMARK_BYTES = 4 * 1024 * 1024; // 4 MiB
/**
* The name of the tool cache directory for the CodeQL tools.

View File

@@ -76,7 +76,7 @@ export async function endTracingForCluster(
}
}
export async function getTracerConfigForCluster(
async function getTracerConfigForCluster(
config: Config,
): Promise<TracerConfig> {
const tracingEnvVariables = JSON.parse(

View File

@@ -412,7 +412,7 @@ export function findSarifFilesInDir(
return sarifFiles;
}
export function getSarifFilePaths(
function getSarifFilePaths(
sarifPath: string,
isSarif: (name: string) => boolean,
) {

View File

@@ -476,7 +476,7 @@ for (const [
githubVersion,
)}`;
test(`checkActionVersion ${reportErrorDescription} for ${versionsDescription}`, async (t) => {
const warningSpy = sinon.spy(core, "error");
const warningSpy = sinon.spy(core, "warning");
const versionStub = sinon
.stub(api, "getGitHubVersion")
.resolves(githubVersion);

View File

@@ -4,7 +4,6 @@ import * as os from "os";
import * as path from "path";
import * as core from "@actions/core";
import * as exec from "@actions/exec/lib/exec";
import * as io from "@actions/io";
import getFolderSize from "get-folder-size";
import * as yaml from "js-yaml";
@@ -1026,34 +1025,6 @@ export function fixInvalidNotifications(
return newSarif;
}
/**
* Removes duplicates from the sarif file.
*
* When `CODEQL_ACTION_DISABLE_DUPLICATE_LOCATION_FIX` is set to true, this will
* simply rename the input file to the output file. Otherwise, it will parse the
* input file as JSON, remove duplicate locations from the SARIF notification
* objects, and write the result to the output file.
*
* For context, see documentation of:
* `CODEQL_ACTION_DISABLE_DUPLICATE_LOCATION_FIX`. */
export function fixInvalidNotificationsInFile(
inputPath: string,
outputPath: string,
logger: Logger,
): void {
if (process.env[EnvVar.DISABLE_DUPLICATE_LOCATION_FIX] === "true") {
logger.info(
"SARIF notification object duplicate location fix disabled by the " +
`${EnvVar.DISABLE_DUPLICATE_LOCATION_FIX} environment variable.`,
);
fs.renameSync(inputPath, outputPath);
} else {
let sarif = JSON.parse(fs.readFileSync(inputPath, "utf8")) as SarifFile;
sarif = fixInvalidNotifications(sarif, logger);
fs.writeFileSync(outputPath, JSON.stringify(sarif));
}
}
export function wrapError(error: unknown): Error {
return error instanceof Error ? error : new Error(String(error));
}
@@ -1141,7 +1112,7 @@ export function checkActionVersion(
">=3.20",
))
) {
core.error(
core.warning(
"CodeQL Action v3 will be deprecated in December 2026. " +
"Please update all occurrences of the CodeQL Action in your workflow files to v4. " +
"For more information, see " +
@@ -1197,49 +1168,6 @@ export function cloneObject<T>(obj: T): T {
return JSON.parse(JSON.stringify(obj)) as T;
}
// The first time this function is called, it runs `csrutil status` to determine
// whether System Integrity Protection is enabled; and saves the result in an
// environment variable. Afterwards, simply return the value of the environment
// variable.
export async function checkSipEnablement(
logger: Logger,
): Promise<boolean | undefined> {
if (
process.env[EnvVar.IS_SIP_ENABLED] !== undefined &&
["true", "false"].includes(process.env[EnvVar.IS_SIP_ENABLED])
) {
return process.env[EnvVar.IS_SIP_ENABLED] === "true";
}
try {
const sipStatusOutput = await exec.getExecOutput("csrutil status");
if (sipStatusOutput.exitCode === 0) {
if (
sipStatusOutput.stdout.includes(
"System Integrity Protection status: enabled.",
)
) {
core.exportVariable(EnvVar.IS_SIP_ENABLED, "true");
return true;
}
if (
sipStatusOutput.stdout.includes(
"System Integrity Protection status: disabled.",
)
) {
core.exportVariable(EnvVar.IS_SIP_ENABLED, "false");
return false;
}
}
return undefined;
} catch (e) {
logger.warning(
`Failed to determine if System Integrity Protection was enabled: ${e}`,
);
return undefined;
}
}
export async function cleanUpPath(file: string, name: string, logger: Logger) {
logger.debug(`Cleaning up ${name}.`);
try {
@@ -1291,17 +1219,6 @@ export function isDefined<T>(value: T | null | undefined): value is T {
return value !== undefined && value !== null;
}
/** Like `Object.keys`, but typed so that the elements of the resulting array have the
* same type as the keys of the input object. Note that this may not be sound if the input
* object has been cast to `T` from a subtype of `T` and contains additional keys that
* are not represented by `keyof T`.
*/
export function unsafeKeysInvariant<T extends Record<string, any>>(
object: T,
): Array<keyof T> {
return Object.keys(object) as Array<keyof T>;
}
/** Like `Object.entries`, but typed so that the key elements of the result have the
* same type as the keys of the input object. Note that this may not be sound if the input
* object has been cast to `T` from a subtype of `T` and contains additional keys that
@@ -1314,3 +1231,8 @@ export function unsafeEntriesInvariant<T extends Record<string, any>>(
([_, val]) => val !== undefined,
) as Array<[keyof T, Exclude<T[keyof T], undefined>]>;
}
export enum CleanupLevel {
Clear = "clear",
Overlay = "overlay",
}

View File

@@ -0,0 +1,6 @@
{
"sdk": {
"version": "9.0.307",
"rollForward": "latestFeature"
}
}