Compare commits

...

65 Commits

Author SHA1 Message Date
Henry Mercer
5d063dd3af Populate database upload results telemetry 2025-12-15 12:55:12 +00:00
Henry Mercer
8e921c3145 Return status report from cleanupAndUploadDatabases 2025-12-15 12:55:12 +00:00
Paolo Tranquilli
52f930e50a Merge pull request #3323 from github/mergeback/v4.31.5-to-main-fdbfb4d2
Mergeback v4.31.5 refs/heads/releases/v4 into main
2025-11-24 12:18:45 +01:00
github-actions[bot]
478350182f Rebuild 2025-11-24 10:55:14 +00:00
github-actions[bot]
29e11fdce1 Update changelog and version after v4.31.5 2025-11-24 09:31:18 +00:00
Paolo Tranquilli
fdbfb4d275 Merge pull request #3322 from github/update-v4.31.5-ec2ee575c
Merge main into releases/v4
2025-11-24 10:29:19 +01:00
github-actions[bot]
81f6d649ae Update changelog for v4.31.5 2025-11-24 09:03:58 +00:00
Paolo Tranquilli
ec2ee575c0 Merge pull request #3321 from github/update-bundle/codeql-bundle-v2.23.6
Update default bundle to 2.23.6
2025-11-24 09:14:29 +01:00
github-actions[bot]
ecc87875ee Add changelog note 2025-11-24 07:51:53 +00:00
github-actions[bot]
1d2a238d7d Update default bundle to codeql-bundle-v2.23.6 2025-11-24 07:51:46 +00:00
Henry Mercer
ce729e4d35 Merge pull request #3315 from github/henrymercer/dead-code-elimination
Delete unused exports
2025-11-19 15:24:22 +00:00
Henry Mercer
ac359aad20 Add return type 2025-11-19 14:59:16 +00:00
Henry Mercer
112cd075bd Merge branch 'main' into henrymercer/dead-code-elimination 2025-11-19 14:56:28 +00:00
Michael B. Gale
0b4317954f Merge pull request #3306 from github/dependabot/npm_and_yarn/types/sinon-21.0.0
Bump @types/sinon from 17.0.4 to 21.0.0
2025-11-19 14:13:16 +00:00
Michael B. Gale
e818008b54 Merge pull request #3305 from github/dependabot/npm_and_yarn/eslint/compat-2.0.0
Bump @eslint/compat from 1.4.1 to 2.0.0
2025-11-19 13:41:43 +00:00
Michael B. Gale
90871e185b Merge pull request #3304 from github/dependabot/npm_and_yarn/npm-minor-7439af33e4
Bump the npm-minor group with 2 updates
2025-11-19 13:18:38 +00:00
Kasper Svendsen
a102014397 Merge pull request #3317 from github/kaspersv/bump-minimum-overlay-version
Overlay: Increase minimum CLI version required for overlay analysis
2025-11-19 14:18:24 +01:00
Kasper Svendsen
de74d762a3 Overlay: Increase minimum CLI version 2025-11-19 13:04:23 +01:00
Kasper Svendsen
ce07e7d196 Merge pull request #3310 from github/kaspersv/overlay-disk-available-limit
Overlay: Fall back to full analysis if runner disk space is low
2025-11-19 12:57:53 +01:00
Henry Mercer
86d2aa55c0 Merge pull request #3316 from github/henrymercer/upload-overlay-to-api
Upload overlay base DBs to GitHub API behind FF
2025-11-19 10:29:28 +00:00
Kasper Svendsen
4eccb3798e Overlay: Round available disk space in MB 2025-11-19 08:40:56 +01:00
Kasper Svendsen
ed80d6e5e9 Overlay: Reorder available disk space check 2025-11-19 07:54:05 +01:00
Henry Mercer
378219ced2 Merge pull request #3313 from github/mergeback/v4.31.4-to-main-e12f0178
Mergeback v4.31.4 refs/heads/releases/v4 into main
2025-11-18 18:46:24 +00:00
Henry Mercer
c649c5993d Upload overlay base DB to API behind FF 2025-11-18 18:43:19 +00:00
Henry Mercer
31042e9879 Rename function calls to make destructive operation clearer 2025-11-18 18:42:15 +00:00
Henry Mercer
5da2098551 Add feature flag for uploading overlay DBs to API 2025-11-18 18:40:51 +00:00
Henry Mercer
cac5926de5 Delete unused exports 2025-11-18 18:16:54 +00:00
Henry Mercer
e24190a70c Remove unused dependencies 2025-11-18 18:14:49 +00:00
github-actions[bot]
ce9b526448 Rebuild 2025-11-18 16:17:35 +00:00
github-actions[bot]
28f4a61417 Merge remote-tracking branch 'origin/main' into mergeback/v4.31.4-to-main-e12f0178 2025-11-18 16:16:46 +00:00
github-actions[bot]
fea250010c Update changelog and version after v4.31.4 2025-11-18 16:14:11 +00:00
Michael B. Gale
e12f017898 Merge pull request #3312 from github/update-v4.31.4-70434f6dd
Merge main into releases/v4
2025-11-18 16:12:25 +00:00
Michael B. Gale
249458aab2 Merge pull request #3296 from github/mbg/dependency-caching/skip-uploads-for-exact-matches
Skip uploading dependency caches if we know they exist
2025-11-18 15:44:06 +00:00
github-actions[bot]
c9cb6f9c13 Update changelog for v4.31.4 2025-11-18 15:18:43 +00:00
Kasper Svendsen
726a2a01b8 Overlay: Increase disk storage threshold to 20GB 2025-11-18 15:37:27 +01:00
Michael B. Gale
70434f6dd2 Merge pull request #3311 from github/mbg/deps/bump-glob
Bump `glob` to at least `11.1.0`
2025-11-18 12:39:21 +00:00
Michael B. Gale
528362a7c1 Bump glob to at least 11.1.0 2025-11-18 12:20:00 +00:00
Michael B. Gale
de12435376 Merge pull request #3308 from github/mbg/pr-template/nov25
Add additional options to PR template and clarify some
2025-11-18 11:52:08 +00:00
Kasper Svendsen
4f746e4a60 Overlay: Fall back to full analysis if runner disk space is low 2025-11-18 08:19:13 +01:00
Michael B. Gale
ffa63f0dac Merge pull request #3307 from github/dependabot/github_actions/dot-github/workflows/actions-minor-761b22fa12
Bump ruby/setup-ruby from 1.267.0 to 1.268.0 in /.github/workflows in the actions-minor group across 1 directory
2025-11-17 18:06:59 +00:00
Michael B. Gale
7bcdb4bc66 Add additional options to PR template and clarify some 2025-11-17 17:48:39 +00:00
Mario Campos
07eae6420a Merge pull request #3303 from github/mario-campos/v3-core-warning
Change v3 deprecation message to warning.
2025-11-17 11:35:30 -06:00
github-actions[bot]
e546fff076 Rebuild 2025-11-17 17:18:36 +00:00
dependabot[bot]
c418a0fc93 Bump ruby/setup-ruby
Bumps the actions-minor group with 1 update in the /.github/workflows directory: [ruby/setup-ruby](https://github.com/ruby/setup-ruby).


Updates `ruby/setup-ruby` from 1.267.0 to 1.268.0
- [Release notes](https://github.com/ruby/setup-ruby/releases)
- [Changelog](https://github.com/ruby/setup-ruby/blob/master/release.rb)
- [Commits](d5126b9b35...8aeb6ff803)

---
updated-dependencies:
- dependency-name: ruby/setup-ruby
  dependency-version: 1.268.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: actions-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-11-17 17:17:07 +00:00
Mario Campos
fc329e3bb5 Revert "Add CHANGELOG.md entry for "v3 deprecation" to warning change."
This reverts commit 023fd08cc9.
2025-11-17 11:08:58 -06:00
github-actions[bot]
b595847fa5 Rebuild 2025-11-17 17:04:50 +00:00
github-actions[bot]
4f39cef4c6 Rebuild 2025-11-17 17:03:39 +00:00
github-actions[bot]
d4a7ccd1f0 Rebuild 2025-11-17 17:03:22 +00:00
dependabot[bot]
cd808e1260 Bump @types/sinon from 17.0.4 to 21.0.0
Bumps [@types/sinon](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/HEAD/types/sinon) from 17.0.4 to 21.0.0.
- [Release notes](https://github.com/DefinitelyTyped/DefinitelyTyped/releases)
- [Commits](https://github.com/DefinitelyTyped/DefinitelyTyped/commits/HEAD/types/sinon)

---
updated-dependencies:
- dependency-name: "@types/sinon"
  dependency-version: 21.0.0
  dependency-type: direct:development
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-11-17 17:02:13 +00:00
dependabot[bot]
01577d4797 Bump @eslint/compat from 1.4.1 to 2.0.0
Bumps [@eslint/compat](https://github.com/eslint/rewrite/tree/HEAD/packages/compat) from 1.4.1 to 2.0.0.
- [Release notes](https://github.com/eslint/rewrite/releases)
- [Changelog](https://github.com/eslint/rewrite/blob/main/packages/compat/CHANGELOG.md)
- [Commits](https://github.com/eslint/rewrite/commits/compat-v2.0.0/packages/compat)

---
updated-dependencies:
- dependency-name: "@eslint/compat"
  dependency-version: 2.0.0
  dependency-type: direct:development
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-11-17 17:01:53 +00:00
dependabot[bot]
3b635815d6 Bump the npm-minor group with 2 updates
Bumps the npm-minor group with 2 updates: [@octokit/request-error](https://github.com/octokit/request-error.js) and [eslint-plugin-jsdoc](https://github.com/gajus/eslint-plugin-jsdoc).


Updates `@octokit/request-error` from 7.0.2 to 7.1.0
- [Release notes](https://github.com/octokit/request-error.js/releases)
- [Commits](https://github.com/octokit/request-error.js/compare/v7.0.2...v7.1.0)

Updates `eslint-plugin-jsdoc` from 61.1.12 to 61.2.1
- [Release notes](https://github.com/gajus/eslint-plugin-jsdoc/releases)
- [Changelog](https://github.com/gajus/eslint-plugin-jsdoc/blob/main/.releaserc)
- [Commits](https://github.com/gajus/eslint-plugin-jsdoc/compare/v61.1.12...v61.2.1)

---
updated-dependencies:
- dependency-name: "@octokit/request-error"
  dependency-version: 7.1.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: npm-minor
- dependency-name: eslint-plugin-jsdoc
  dependency-version: 61.2.1
  dependency-type: direct:development
  update-type: version-update:semver-minor
  dependency-group: npm-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-11-17 17:01:47 +00:00
Mario Campos
023fd08cc9 Add CHANGELOG.md entry for "v3 deprecation" to warning change. 2025-11-17 09:04:58 -06:00
Mario Campos
ed3a01336f Change v3 deprecation message to warning. 2025-11-17 08:59:44 -06:00
Michael B. Gale
c1a2b73420 Merge pull request #3301 from github/dependabot/npm_and_yarn/js-yaml-4.1.1
Bump js-yaml from 4.1.0 to 4.1.1
2025-11-16 17:54:05 +00:00
github-actions[bot]
8c254d05f3 Rebuild 2025-11-15 10:57:22 +00:00
dependabot[bot]
b9620e1249 Bump js-yaml from 4.1.0 to 4.1.1
Bumps [js-yaml](https://github.com/nodeca/js-yaml) from 4.1.0 to 4.1.1.
- [Changelog](https://github.com/nodeca/js-yaml/blob/master/CHANGELOG.md)
- [Commits](https://github.com/nodeca/js-yaml/compare/4.1.0...4.1.1)

---
updated-dependencies:
- dependency-name: js-yaml
  dependency-version: 4.1.1
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-11-15 10:55:57 +00:00
Michael B. Gale
1ed85b4501 Add test coverage for uploadDependencyCaches 2025-11-14 14:30:54 +00:00
Michael B. Gale
51c9af3a3b Don't try to upload cache if we have restored a cache with the same key 2025-11-14 14:30:54 +00:00
Michael B. Gale
594c0cc369 Store restored keys in action state 2025-11-14 14:30:54 +00:00
Michael B. Gale
11889c27fd Return keys of restored caches from downloadDependencyCaches 2025-11-14 14:30:54 +00:00
Kasper Svendsen
85f1517bb4 Merge pull request #3285 from github/kaspersv/remove-overlay-org-restriction
Overlay: Remove repository owner restriction
2025-11-14 08:28:09 +01:00
Michael B. Gale
86b7d4fc36 Merge pull request #3294 from github/mergeback/v4.31.3-to-main-014f16e7
Mergeback v4.31.3 refs/heads/releases/v4 into main
2025-11-13 22:22:18 +00:00
github-actions[bot]
246edb9b1d Rebuild 2025-11-13 21:59:57 +00:00
github-actions[bot]
497c7f627a Update changelog and version after v4.31.3 2025-11-13 21:54:56 +00:00
Kasper Svendsen
5091e42a03 Overlay: Remove repository owner restriction 2025-11-13 10:48:25 +01:00
46 changed files with 7274 additions and 7150 deletions

View File

@@ -18,14 +18,25 @@ For internal use only. Please select the risk level of this change:
#### Which use cases does this change impact?
<!-- Delete options that don't apply. -->
<!-- Delete options that don't apply. If in doubt, do not delete an option. -->
- **Advanced setup** - Impacts users who have custom workflows.
- **Default setup** - Impacts users who use default setup.
- **Code Scanning** - Impacts Code Scanning (i.e. `analysis-kinds: code-scanning`).
- **Code Quality** - Impacts Code Quality (i.e. `analysis-kinds: code-quality`).
- **Third-party analyses** - Impacts third-party analyses (i.e. `upload-sarif`).
- **GHES** - Impacts GitHub Enterprise Server.
Workflow types:
- **Advanced setup** - Impacts users who have custom CodeQL workflows.
- **Managed** - Impacts users with `dynamic` workflows (Default Setup, CCR, ...).
Products:
- **Code Scanning** - The changes impact analyses when `analysis-kinds: code-scanning`.
- **Code Quality** - The changes impact analyses when `analysis-kinds: code-quality`.
- **CCR** - The changes impact analyses for Copilot Code Reviews.
- **Third-party analyses** - The changes affect the `upload-sarif` action.
Environments:
- **Dotcom** - Impacts CodeQL workflows on `github.com`.
- **GHES** - Impacts CodeQL workflows on GitHub Enterprise Server.
- **Testing/None** - This change does not impact any CodeQL workflows in production.
#### How did/will you validate this change?
@@ -54,6 +65,15 @@ For internal use only. Please select the risk level of this change:
- **Alerts** - New or existing monitors will trip if something goes wrong with this change.
- **Other** - Please provide details.
#### Are there any special considerations for merging or releasing this change?
<!--
Consider whether this change depends on a different change in another repository that should be released first.
-->
- **No special considerations** - This change can be merged at any time.
- **Special considerations** - This change should only be merged once certain preconditions are met. Please provide details of those or link to this PR from an internal issue.
### Merge / deployment checklist
- Confirm this change is backwards compatible with existing workflows.

View File

@@ -56,7 +56,7 @@ jobs:
use-all-platform-bundle: 'false'
setup-kotlin: 'true'
- name: Set up Ruby
uses: ruby/setup-ruby@d5126b9b3579e429dd52e51e68624dda2e05be25 # v1.267.0
uses: ruby/setup-ruby@8aeb6ff8030dd539317f8e1769a044873b56ea71 # v1.268.0
with:
ruby-version: 2.6
- name: Install Code Scanning integration

View File

@@ -2,6 +2,18 @@
See the [releases page](https://github.com/github/codeql-action/releases) for the relevant changes to the CodeQL CLI and language packs.
## [UNRELEASED]
No user facing changes.
## 4.31.5 - 24 Nov 2025
- Update default CodeQL bundle version to 2.23.6. [#3321](https://github.com/github/codeql-action/pull/3321)
## 4.31.4 - 18 Nov 2025
No user facing changes.
## 4.31.3 - 13 Nov 2025
- CodeQL Action v3 will be deprecated in December 2026. The Action now logs a warning for customers who are running v3 but could be running v4. For more information, see [Upcoming deprecation of CodeQL Action v3](https://github.blog/changelog/2025-10-28-upcoming-deprecation-of-codeql-action-v3/).

File diff suppressed because it is too large Load Diff

873
lib/analyze-action.js generated

File diff suppressed because it is too large Load Diff

756
lib/autobuild-action.js generated

File diff suppressed because it is too large Load Diff

View File

@@ -1,6 +1,6 @@
{
"bundleVersion": "codeql-bundle-v2.23.5",
"cliVersion": "2.23.5",
"priorBundleVersion": "codeql-bundle-v2.23.3",
"priorCliVersion": "2.23.3"
"bundleVersion": "codeql-bundle-v2.23.6",
"cliVersion": "2.23.6",
"priorBundleVersion": "codeql-bundle-v2.23.5",
"priorCliVersion": "2.23.5"
}

1465
lib/init-action-post.js generated

File diff suppressed because it is too large Load Diff

871
lib/init-action.js generated

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

1268
lib/start-proxy-action.js generated

File diff suppressed because it is too large Load Diff

750
lib/upload-lib.js generated

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

905
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@@ -1,6 +1,6 @@
{
"name": "codeql",
"version": "4.31.3",
"version": "4.31.6",
"private": true,
"description": "CodeQL action",
"scripts": {
@@ -35,23 +35,21 @@
"@actions/io": "^2.0.0",
"@actions/tool-cache": "^2.0.2",
"@octokit/plugin-retry": "^6.0.0",
"@octokit/request-error": "^7.0.2",
"@schemastore/package": "0.0.10",
"archiver": "^7.0.1",
"fast-deep-equal": "^3.1.3",
"follow-redirects": "^1.15.11",
"get-folder-size": "^5.0.0",
"js-yaml": "^4.1.0",
"js-yaml": "^4.1.1",
"jsonschema": "1.4.1",
"long": "^5.3.2",
"node-forge": "^1.3.1",
"octokit": "^5.0.5",
"semver": "^7.7.3",
"uuid": "^13.0.0"
},
"devDependencies": {
"@ava/typescript": "6.0.0",
"@eslint/compat": "^1.4.1",
"@eslint/compat": "^2.0.0",
"@eslint/eslintrc": "^3.3.1",
"@eslint/js": "^9.39.1",
"@microsoft/eslint-formatter-sarif": "^3.1.0",
@@ -59,10 +57,10 @@
"@types/archiver": "^7.0.0",
"@types/follow-redirects": "^1.14.4",
"@types/js-yaml": "^4.0.9",
"@types/node": "20.19.9",
"@types/node": "^20.19.9",
"@types/node-forge": "^1.3.14",
"@types/semver": "^7.7.1",
"@types/sinon": "^17.0.4",
"@types/sinon": "^21.0.0",
"@typescript-eslint/eslint-plugin": "^8.46.4",
"@typescript-eslint/parser": "^8.41.0",
"ava": "^6.4.1",
@@ -72,9 +70,9 @@
"eslint-plugin-filenames": "^1.3.2",
"eslint-plugin-github": "^5.1.8",
"eslint-plugin-import": "2.29.1",
"eslint-plugin-jsdoc": "^61.1.12",
"eslint-plugin-jsdoc": "^61.2.1",
"eslint-plugin-no-async-foreach": "^0.1.1",
"glob": "^11.0.3",
"glob": "^11.1.0",
"nock": "^14.0.10",
"sinon": "^21.0.0",
"typescript": "^5.9.3"
@@ -98,6 +96,7 @@
"eslint-plugin-jsx-a11y": {
"semver": ">=6.3.1"
},
"brace-expansion@2.0.1": "2.0.2"
"brace-expansion@2.0.1": "2.0.2",
"glob": "^11.1.0"
}
}

View File

@@ -4,7 +4,7 @@ description: "Tests using RuboCop to analyze a multi-language repository and the
versions: ["default"]
steps:
- name: Set up Ruby
uses: ruby/setup-ruby@d5126b9b3579e429dd52e51e68624dda2e05be25 # v1.267.0
uses: ruby/setup-ruby@8aeb6ff8030dd539317f8e1769a044873b56ea71 # v1.268.0
with:
ruby-version: 2.6
- name: Install Code Scanning integration

View File

@@ -80,7 +80,7 @@ export function isRunningLocalAction(): boolean {
*
* This can be used to get the Action's name or tell if we're running a local Action.
*/
export function getRelativeScriptPath(): string {
function getRelativeScriptPath(): string {
const runnerTemp = getRequiredEnvParam("RUNNER_TEMP");
const actionsDirectory = path.join(path.dirname(runnerTemp), "_actions");
return path.relative(actionsDirectory, __filename);

View File

@@ -98,7 +98,7 @@ export async function getAnalysisKinds(
export const codeQualityQueries: string[] = ["code-quality"];
// Enumerates API endpoints that accept SARIF files.
export enum SARIF_UPLOAD_ENDPOINT {
enum SARIF_UPLOAD_ENDPOINT {
CODE_SCANNING = "PUT /repos/:owner/:repo/code-scanning/analysis",
CODE_QUALITY = "PUT /repos/:owner/:repo/code-quality/analysis",
}

View File

@@ -25,7 +25,10 @@ import {
isCodeQualityEnabled,
isCodeScanningEnabled,
} from "./config-utils";
import { uploadDatabases } from "./database-upload";
import {
cleanupAndUploadDatabases,
DatabaseUploadResult,
} from "./database-upload";
import {
DependencyCacheUploadStatusReport,
uploadDependencyCaches,
@@ -35,7 +38,7 @@ import { EnvVar } from "./environment";
import { Feature, Features } from "./feature-flags";
import { KnownLanguage } from "./languages";
import { getActionsLogger, Logger } from "./logging";
import { uploadOverlayBaseDatabaseToCache } from "./overlay-database-utils";
import { cleanupAndUploadOverlayBaseDatabaseToCache } from "./overlay-database-utils";
import { getRepositoryNwo } from "./repository";
import * as statusReport from "./status-report";
import {
@@ -59,15 +62,13 @@ interface AnalysisStatusReport
extends uploadLib.UploadStatusReport,
QueriesStatusReport {}
interface DependencyCachingUploadStatusReport {
dependency_caching_upload_results?: DependencyCacheUploadStatusReport;
}
interface FinishStatusReport
extends StatusReportBase,
DatabaseCreationTimings,
AnalysisStatusReport,
DependencyCachingUploadStatusReport {}
AnalysisStatusReport {
dependency_caching_upload_results?: DependencyCacheUploadStatusReport;
database_upload_results: DatabaseUploadResult[];
}
interface FinishWithTrapUploadStatusReport extends FinishStatusReport {
/** Size of TRAP caches that we uploaded, in bytes. */
@@ -86,6 +87,7 @@ async function sendStatusReport(
didUploadTrapCaches: boolean,
trapCacheCleanup: TrapCacheCleanupStatusReport | undefined,
dependencyCacheResults: DependencyCacheUploadStatusReport | undefined,
databaseUploadResults: DatabaseUploadResult[],
logger: Logger,
) {
const status = getActionsStatus(error, stats?.analyze_failure_language);
@@ -106,6 +108,7 @@ async function sendStatusReport(
...(dbCreationTimings || {}),
...(trapCacheCleanup || {}),
dependency_caching_upload_results: dependencyCacheResults,
database_upload_results: databaseUploadResults,
};
if (config && didUploadTrapCaches) {
const trapCacheUploadStatusReport: FinishWithTrapUploadStatusReport = {
@@ -223,6 +226,7 @@ async function run() {
let dbCreationTimings: DatabaseCreationTimings | undefined = undefined;
let didUploadTrapCaches = false;
let dependencyCacheResults: DependencyCacheUploadStatusReport | undefined;
let databaseUploadResults: DatabaseUploadResult[] = [];
util.initializeEnvironment(actionsUtil.getActionVersion());
// Make inputs accessible in the `post` step, details at
@@ -417,12 +421,21 @@ async function run() {
}
// Possibly upload the overlay-base database to actions cache.
// If databases are to be uploaded, they will first be cleaned up at the overlay level.
await uploadOverlayBaseDatabaseToCache(codeql, config, logger);
// Note: Take care with the ordering of this call since databases may be cleaned up
// at the `overlay` level.
await cleanupAndUploadOverlayBaseDatabaseToCache(codeql, config, logger);
// Possibly upload the database bundles for remote queries.
// If databases are to be uploaded, they will first be cleaned up at the clear level.
await uploadDatabases(repositoryNwo, codeql, config, apiDetails, logger);
// Note: Take care with the ordering of this call since databases may be cleaned up
// at the `overlay` or `clear` level.
databaseUploadResults = await cleanupAndUploadDatabases(
repositoryNwo,
codeql,
config,
apiDetails,
features,
logger,
);
// Possibly upload the TRAP caches for later re-use
const trapCacheUploadStartTime = performance.now();
@@ -487,6 +500,7 @@ async function run() {
didUploadTrapCaches,
trapCacheCleanupTelemetry,
dependencyCacheResults,
databaseUploadResults,
logger,
);
return;
@@ -509,6 +523,7 @@ async function run() {
didUploadTrapCaches,
trapCacheCleanupTelemetry,
dependencyCacheResults,
databaseUploadResults,
logger,
);
} else if (runStats !== undefined) {
@@ -522,6 +537,7 @@ async function run() {
didUploadTrapCaches,
trapCacheCleanupTelemetry,
dependencyCacheResults,
databaseUploadResults,
logger,
);
} else {
@@ -535,6 +551,7 @@ async function run() {
didUploadTrapCaches,
trapCacheCleanupTelemetry,
dependencyCacheResults,
databaseUploadResults,
logger,
);
}

View File

@@ -18,11 +18,6 @@ import {
const GITHUB_ENTERPRISE_VERSION_HEADER = "x-github-enterprise-version";
export enum DisallowedAPIVersionReason {
ACTION_TOO_OLD,
ACTION_TOO_NEW,
}
export type GitHubApiCombinedDetails = GitHubApiDetails &
GitHubApiExternalRepoDetails;

View File

@@ -159,10 +159,7 @@ type CliErrorConfiguration = {
* All of our caught CLI error messages that we handle specially: ie. if we
* would like to categorize an error as a configuration error or not.
*/
export const cliErrorsConfig: Record<
CliConfigErrorCategory,
CliErrorConfiguration
> = {
const cliErrorsConfig: Record<CliConfigErrorCategory, CliErrorConfiguration> = {
[CliConfigErrorCategory.AutobuildError]: {
cliErrorMessageCandidates: [
new RegExp("We were unable to automatically build your code"),

View File

@@ -35,7 +35,7 @@ import { ToolsDownloadStatusReport } from "./tools-download";
import { ToolsFeature, isSupportedToolsFeature } from "./tools-features";
import { shouldEnableIndirectTracing } from "./tracer-config";
import * as util from "./util";
import { BuildMode, getErrorMessage } from "./util";
import { BuildMode, CleanupLevel, getErrorMessage } from "./util";
type Options = Array<string | number | boolean>;
@@ -141,7 +141,10 @@ export interface CodeQL {
/**
* Clean up all the databases within a database cluster.
*/
databaseCleanupCluster(config: Config, cleanupLevel: string): Promise<void>;
databaseCleanupCluster(
config: Config,
cleanupLevel: CleanupLevel,
): Promise<void>;
/**
* Run 'codeql database bundle'.
*/
@@ -513,7 +516,7 @@ export async function getCodeQLForTesting(
* version requirement. Must be set to true outside tests.
* @returns A new CodeQL object
*/
export async function getCodeQLForCmd(
async function getCodeQLForCmd(
cmd: string,
checkVersion: boolean,
): Promise<CodeQL> {
@@ -878,7 +881,7 @@ export async function getCodeQLForCmd(
},
async databaseCleanupCluster(
config: Config,
cleanupLevel: string,
cleanupLevel: CleanupLevel,
): Promise<void> {
const cacheCleanupFlag = (await util.codeQlVersionAtLeast(
this,
@@ -1222,7 +1225,7 @@ export async function getTrapCachingExtractorConfigArgsForLang(
*
* This will not exist if the configuration is being parsed in the Action.
*/
export function getGeneratedCodeScanningConfigPath(config: Config): string {
function getGeneratedCodeScanningConfigPath(config: Config): string {
return path.resolve(config.tempDir, "user-config.yaml");
}

View File

@@ -37,7 +37,9 @@ import {
ConfigurationError,
withTmpDir,
BuildMode,
DiskUsage,
} from "./util";
import * as util from "./util";
setupTests(test);
@@ -200,12 +202,9 @@ test("load code quality config", async (t) => {
);
// And the config we expect it to result in
const expectedConfig: configUtils.Config = {
version: actionsUtil.getActionVersion(),
const expectedConfig = createTestConfig({
analysisKinds: [AnalysisKind.CodeQuality],
languages: [KnownLanguage.actions],
buildMode: undefined,
originalUserInput: {},
// This gets set because we only have `AnalysisKind.CodeQuality`
computedConfig: {
"disable-default-queries": true,
@@ -219,14 +218,7 @@ test("load code quality config", async (t) => {
debugMode: false,
debugArtifactName: "",
debugDatabaseName: "",
trapCaches: {},
trapCacheDownloadTime: 0,
dependencyCachingEnabled: CachingKind.None,
extraQueryExclusions: [],
overlayDatabaseMode: OverlayDatabaseMode.None,
useOverlayDatabaseCaching: false,
repositoryProperties: {},
};
});
t.deepEqual(config, expectedConfig);
});
@@ -507,9 +499,7 @@ test("load non-empty input", async (t) => {
};
// And the config we expect it to parse to
const expectedConfig: configUtils.Config = {
version: actionsUtil.getActionVersion(),
analysisKinds: [AnalysisKind.CodeScanning],
const expectedConfig = createTestConfig({
languages: [KnownLanguage.javascript],
buildMode: BuildMode.None,
originalUserInput: userConfig,
@@ -521,14 +511,7 @@ test("load non-empty input", async (t) => {
debugMode: false,
debugArtifactName: "my-artifact",
debugDatabaseName: "my-db",
trapCaches: {},
trapCacheDownloadTime: 0,
dependencyCachingEnabled: CachingKind.None,
extraQueryExclusions: [],
overlayDatabaseMode: OverlayDatabaseMode.None,
useOverlayDatabaseCaching: false,
repositoryProperties: {},
};
});
const languagesInput = "javascript";
const configFilePath = createConfigFile(inputFileContents, tempDir);
@@ -990,12 +973,12 @@ interface OverlayDatabaseModeTestSetup {
features: Feature[];
isPullRequest: boolean;
isDefaultBranch: boolean;
repositoryOwner: string;
buildMode: BuildMode | undefined;
languages: Language[];
codeqlVersion: string;
gitRoot: string | undefined;
codeScanningConfig: configUtils.UserConfig;
diskUsage: DiskUsage | undefined;
}
const defaultOverlayDatabaseModeTestSetup: OverlayDatabaseModeTestSetup = {
@@ -1003,12 +986,15 @@ const defaultOverlayDatabaseModeTestSetup: OverlayDatabaseModeTestSetup = {
features: [],
isPullRequest: false,
isDefaultBranch: false,
repositoryOwner: "github",
buildMode: BuildMode.None,
languages: [KnownLanguage.javascript],
codeqlVersion: CODEQL_OVERLAY_MINIMUM_VERSION,
gitRoot: "/some/git/root",
codeScanningConfig: {},
diskUsage: {
numAvailableBytes: 50_000_000_000,
numTotalBytes: 100_000_000_000,
},
};
const getOverlayDatabaseModeMacro = test.macro({
@@ -1041,6 +1027,8 @@ const getOverlayDatabaseModeMacro = test.macro({
setup.overlayDatabaseEnvVar;
}
sinon.stub(util, "checkDiskUsage").resolves(setup.diskUsage);
// Mock feature flags
const features = createFeatures(setup.features);
@@ -1049,12 +1037,6 @@ const getOverlayDatabaseModeMacro = test.macro({
.stub(actionsUtil, "isAnalyzingPullRequest")
.returns(setup.isPullRequest);
// Mock repository owner
const repository = {
owner: setup.repositoryOwner,
repo: "test-repo",
};
// Set up CodeQL mock
const codeql = mockCodeQLVersion(setup.codeqlVersion);
@@ -1077,7 +1059,6 @@ const getOverlayDatabaseModeMacro = test.macro({
const result = await configUtils.getOverlayDatabaseMode(
codeql,
repository,
features,
setup.languages,
tempDir, // sourceRoot
@@ -1205,6 +1186,45 @@ test(
},
);
test(
getOverlayDatabaseModeMacro,
"No overlay-base database on default branch if runner disk space is too low",
{
languages: [KnownLanguage.javascript],
features: [
Feature.OverlayAnalysis,
Feature.OverlayAnalysisCodeScanningJavascript,
],
isDefaultBranch: true,
diskUsage: {
numAvailableBytes: 1_000_000_000,
numTotalBytes: 100_000_000_000,
},
},
{
overlayDatabaseMode: OverlayDatabaseMode.None,
useOverlayDatabaseCaching: false,
},
);
test(
getOverlayDatabaseModeMacro,
"No overlay-base database on default branch if we can't determine runner disk space",
{
languages: [KnownLanguage.javascript],
features: [
Feature.OverlayAnalysis,
Feature.OverlayAnalysisCodeScanningJavascript,
],
isDefaultBranch: true,
diskUsage: undefined,
},
{
overlayDatabaseMode: OverlayDatabaseMode.None,
useOverlayDatabaseCaching: false,
},
);
test(
getOverlayDatabaseModeMacro,
"No overlay-base database on default branch when code-scanning feature enabled with disable-default-queries",
@@ -1375,6 +1395,45 @@ test(
},
);
test(
getOverlayDatabaseModeMacro,
"No overlay analysis on PR if runner disk space is too low",
{
languages: [KnownLanguage.javascript],
features: [
Feature.OverlayAnalysis,
Feature.OverlayAnalysisCodeScanningJavascript,
],
isPullRequest: true,
diskUsage: {
numAvailableBytes: 1_000_000_000,
numTotalBytes: 100_000_000_000,
},
},
{
overlayDatabaseMode: OverlayDatabaseMode.None,
useOverlayDatabaseCaching: false,
},
);
test(
getOverlayDatabaseModeMacro,
"No overlay analysis on PR if we can't determine runner disk space",
{
languages: [KnownLanguage.javascript],
features: [
Feature.OverlayAnalysis,
Feature.OverlayAnalysisCodeScanningJavascript,
],
isPullRequest: true,
diskUsage: undefined,
},
{
overlayDatabaseMode: OverlayDatabaseMode.None,
useOverlayDatabaseCaching: false,
},
);
test(
getOverlayDatabaseModeMacro,
"No overlay analysis on PR when code-scanning feature enabled with disable-default-queries",
@@ -1499,10 +1558,9 @@ test(
test(
getOverlayDatabaseModeMacro,
"Overlay PR analysis by env for dsp-testing",
"Overlay PR analysis by env",
{
overlayDatabaseEnvVar: "overlay",
repositoryOwner: "dsp-testing",
},
{
overlayDatabaseMode: OverlayDatabaseMode.Overlay,
@@ -1512,10 +1570,10 @@ test(
test(
getOverlayDatabaseModeMacro,
"Overlay PR analysis by env for other-org",
"Overlay PR analysis by env on a runner with low disk space",
{
overlayDatabaseEnvVar: "overlay",
repositoryOwner: "other-org",
diskUsage: { numAvailableBytes: 0, numTotalBytes: 100_000_000_000 },
},
{
overlayDatabaseMode: OverlayDatabaseMode.Overlay,
@@ -1525,12 +1583,11 @@ test(
test(
getOverlayDatabaseModeMacro,
"Overlay PR analysis by feature flag for dsp-testing",
"Overlay PR analysis by feature flag",
{
languages: [KnownLanguage.javascript],
features: [Feature.OverlayAnalysis, Feature.OverlayAnalysisJavascript],
isPullRequest: true,
repositoryOwner: "dsp-testing",
},
{
overlayDatabaseMode: OverlayDatabaseMode.Overlay,
@@ -1538,21 +1595,6 @@ test(
},
);
test(
getOverlayDatabaseModeMacro,
"No overlay PR analysis by feature flag for other-org",
{
languages: [KnownLanguage.javascript],
features: [Feature.OverlayAnalysis, Feature.OverlayAnalysisJavascript],
isPullRequest: true,
repositoryOwner: "other-org",
},
{
overlayDatabaseMode: OverlayDatabaseMode.None,
useOverlayDatabaseCaching: false,
},
);
test(
getOverlayDatabaseModeMacro,
"Fallback due to autobuild with traced language",

View File

@@ -43,10 +43,22 @@ import {
codeQlVersionAtLeast,
cloneObject,
isDefined,
checkDiskUsage,
} from "./util";
export * from "./config/db-config";
/**
* The minimum available disk space (in MB) required to perform overlay analysis.
* If the available disk space on the runner is below the threshold when deciding
* whether to perform overlay analysis, then the action will not perform overlay
* analysis unless overlay analysis has been explicitly enabled via environment
* variable.
*/
const OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_MB = 20000;
const OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_BYTES =
OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_MB * 1_000_000;
export type RegistryConfigWithCredentials = RegistryConfigNoCredentials & {
// Token to use when downloading packs from this registry.
token: string;
@@ -148,6 +160,9 @@ export interface Config {
/** A value indicating how dependency caching should be used. */
dependencyCachingEnabled: CachingKind;
/** The keys of caches that we restored, if any. */
dependencyCachingRestoredKeys: string[];
/**
* Extra query exclusions to append to the config.
*/
@@ -176,7 +191,7 @@ export interface Config {
repositoryProperties: RepositoryProperties;
}
export async function getSupportedLanguageMap(
async function getSupportedLanguageMap(
codeql: CodeQL,
logger: Logger,
): Promise<Record<string, string>> {
@@ -239,7 +254,7 @@ export function hasActionsWorkflows(sourceRoot: string): boolean {
/**
* Gets the set of languages in the current repository.
*/
export async function getRawLanguagesInRepo(
async function getRawLanguagesInRepo(
repository: RepositoryNwo,
sourceRoot: string,
logger: Logger,
@@ -348,7 +363,7 @@ export function getRawLanguagesNoAutodetect(
* @returns A tuple containing a list of languages in this repository that might be
* analyzable and whether or not this list was determined automatically.
*/
export async function getRawLanguages(
async function getRawLanguages(
languagesInput: string | undefined,
repository: RepositoryNwo,
sourceRoot: string,
@@ -496,6 +511,7 @@ export async function initActionState(
trapCaches,
trapCacheDownloadTime,
dependencyCachingEnabled: getCachingKind(dependencyCachingEnabled),
dependencyCachingRestoredKeys: [],
extraQueryExclusions: [],
overlayDatabaseMode: OverlayDatabaseMode.None,
useOverlayDatabaseCaching: false,
@@ -579,17 +595,11 @@ const OVERLAY_ANALYSIS_CODE_SCANNING_FEATURES: Record<Language, Feature> = {
};
async function isOverlayAnalysisFeatureEnabled(
repository: RepositoryNwo,
features: FeatureEnablement,
codeql: CodeQL,
languages: Language[],
codeScanningConfig: UserConfig,
): Promise<boolean> {
// TODO: Remove the repository owner check once support for overlay analysis
// stabilizes, and no more backward-incompatible changes are expected.
if (!["github", "dsp-testing"].includes(repository.owner)) {
return false;
}
if (!(await features.getValue(Feature.OverlayAnalysis, codeql))) {
return false;
}
@@ -647,7 +657,6 @@ async function isOverlayAnalysisFeatureEnabled(
*/
export async function getOverlayDatabaseMode(
codeql: CodeQL,
repository: RepositoryNwo,
features: FeatureEnablement,
languages: Language[],
sourceRoot: string,
@@ -676,27 +685,43 @@ export async function getOverlayDatabaseMode(
);
} else if (
await isOverlayAnalysisFeatureEnabled(
repository,
features,
codeql,
languages,
codeScanningConfig,
)
) {
if (isAnalyzingPullRequest()) {
overlayDatabaseMode = OverlayDatabaseMode.Overlay;
useOverlayDatabaseCaching = true;
const diskUsage = await checkDiskUsage(logger);
if (
diskUsage === undefined ||
diskUsage.numAvailableBytes < OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_BYTES
) {
const diskSpaceMb =
diskUsage === undefined
? 0
: Math.round(diskUsage.numAvailableBytes / 1_000_000);
overlayDatabaseMode = OverlayDatabaseMode.None;
useOverlayDatabaseCaching = false;
logger.info(
`Setting overlay database mode to ${overlayDatabaseMode} ` +
"with caching because we are analyzing a pull request.",
);
} else if (await isAnalyzingDefaultBranch()) {
overlayDatabaseMode = OverlayDatabaseMode.OverlayBase;
useOverlayDatabaseCaching = true;
logger.info(
`Setting overlay database mode to ${overlayDatabaseMode} ` +
"with caching because we are analyzing the default branch.",
`due to insufficient disk space (${diskSpaceMb} MB).`,
);
} else {
if (isAnalyzingPullRequest()) {
overlayDatabaseMode = OverlayDatabaseMode.Overlay;
useOverlayDatabaseCaching = true;
logger.info(
`Setting overlay database mode to ${overlayDatabaseMode} ` +
"with caching because we are analyzing a pull request.",
);
} else if (await isAnalyzingDefaultBranch()) {
overlayDatabaseMode = OverlayDatabaseMode.OverlayBase;
useOverlayDatabaseCaching = true;
logger.info(
`Setting overlay database mode to ${overlayDatabaseMode} ` +
"with caching because we are analyzing the default branch.",
);
}
}
}
@@ -846,7 +871,6 @@ export async function initConfig(
const { overlayDatabaseMode, useOverlayDatabaseCaching } =
await getOverlayDatabaseMode(
inputs.codeql,
inputs.repository,
inputs.features,
config.languages,
inputs.sourceRoot,
@@ -1235,7 +1259,7 @@ export function isCodeQualityEnabled(config: Config): boolean {
* @returns Returns `AnalysisKind.CodeScanning` if `AnalysisKind.CodeScanning` is enabled;
* otherwise `AnalysisKind.CodeQuality`.
*/
export function getPrimaryAnalysisKind(config: Config): AnalysisKind {
function getPrimaryAnalysisKind(config: Config): AnalysisKind {
return isCodeScanningEnabled(config)
? AnalysisKind.CodeScanning
: AnalysisKind.CodeQuality;

View File

@@ -10,11 +10,12 @@ import { GitHubApiDetails } from "./api-client";
import * as apiClient from "./api-client";
import { createStubCodeQL } from "./codeql";
import { Config } from "./config-utils";
import { uploadDatabases } from "./database-upload";
import { cleanupAndUploadDatabases } from "./database-upload";
import * as gitUtils from "./git-utils";
import { KnownLanguage } from "./languages";
import { RepositoryNwo } from "./repository";
import {
createFeatures,
createTestConfig,
getRecordingLogger,
LoggedMessage,
@@ -91,11 +92,12 @@ test("Abort database upload if 'upload-database' input set to false", async (t)
sinon.stub(gitUtils, "isAnalyzingDefaultBranch").resolves(true);
const loggedMessages = [];
await uploadDatabases(
await cleanupAndUploadDatabases(
testRepoName,
getCodeQL(),
getTestConfig(tmpDir),
testApiDetails,
createFeatures([]),
getRecordingLogger(loggedMessages),
);
t.assert(
@@ -121,7 +123,7 @@ test("Abort database upload if 'analysis-kinds: code-scanning' is not enabled",
await mockHttpRequests(201);
const loggedMessages = [];
await uploadDatabases(
await cleanupAndUploadDatabases(
testRepoName,
getCodeQL(),
{
@@ -129,6 +131,7 @@ test("Abort database upload if 'analysis-kinds: code-scanning' is not enabled",
analysisKinds: [AnalysisKind.CodeQuality],
},
testApiDetails,
createFeatures([]),
getRecordingLogger(loggedMessages),
);
t.assert(
@@ -155,11 +158,12 @@ test("Abort database upload if running against GHES", async (t) => {
config.gitHubVersion = { type: GitHubVariant.GHES, version: "3.0" };
const loggedMessages = [];
await uploadDatabases(
await cleanupAndUploadDatabases(
testRepoName,
getCodeQL(),
config,
testApiDetails,
createFeatures([]),
getRecordingLogger(loggedMessages),
);
t.assert(
@@ -183,11 +187,12 @@ test("Abort database upload if not analyzing default branch", async (t) => {
sinon.stub(gitUtils, "isAnalyzingDefaultBranch").resolves(false);
const loggedMessages = [];
await uploadDatabases(
await cleanupAndUploadDatabases(
testRepoName,
getCodeQL(),
getTestConfig(tmpDir),
testApiDetails,
createFeatures([]),
getRecordingLogger(loggedMessages),
);
t.assert(
@@ -212,11 +217,12 @@ test("Don't crash if uploading a database fails", async (t) => {
await mockHttpRequests(500);
const loggedMessages = [] as LoggedMessage[];
await uploadDatabases(
await cleanupAndUploadDatabases(
testRepoName,
getCodeQL(),
getTestConfig(tmpDir),
testApiDetails,
createFeatures([]),
getRecordingLogger(loggedMessages),
);
@@ -243,11 +249,12 @@ test("Successfully uploading a database to github.com", async (t) => {
await mockHttpRequests(201);
const loggedMessages = [] as LoggedMessage[];
await uploadDatabases(
await cleanupAndUploadDatabases(
testRepoName,
getCodeQL(),
getTestConfig(tmpDir),
testApiDetails,
createFeatures([]),
getRecordingLogger(loggedMessages),
);
t.assert(
@@ -272,7 +279,7 @@ test("Successfully uploading a database to GHEC-DR", async (t) => {
const databaseUploadSpy = await mockHttpRequests(201);
const loggedMessages = [] as LoggedMessage[];
await uploadDatabases(
await cleanupAndUploadDatabases(
testRepoName,
getCodeQL(),
getTestConfig(tmpDir),
@@ -281,6 +288,7 @@ test("Successfully uploading a database to GHEC-DR", async (t) => {
url: "https://tenant.ghe.com",
apiURL: undefined,
},
createFeatures([]),
getRecordingLogger(loggedMessages),
);
t.assert(

View File

@@ -5,34 +5,51 @@ import { AnalysisKind } from "./analyses";
import { getApiClient, GitHubApiDetails } from "./api-client";
import { type CodeQL } from "./codeql";
import { Config } from "./config-utils";
import { Feature, FeatureEnablement } from "./feature-flags";
import * as gitUtils from "./git-utils";
import { Logger, withGroupAsync } from "./logging";
import { OverlayDatabaseMode } from "./overlay-database-utils";
import { RepositoryNwo } from "./repository";
import * as util from "./util";
import { bundleDb, parseGitHubUrl } from "./util";
import { bundleDb, CleanupLevel, parseGitHubUrl } from "./util";
export async function uploadDatabases(
/** Information about a database upload. */
export interface DatabaseUploadResult {
/** Language of the database. */
language: string;
/** Size of the zipped database in bytes. */
zipped_upload_size_bytes?: number;
/** Whether the uploaded database is an overlay base. */
is_overlay_base?: boolean;
/** Time taken to upload database in milliseconds. */
upload_duration_ms?: number;
/** If there was an error during database upload, this is its message. */
error?: string;
}
export async function cleanupAndUploadDatabases(
repositoryNwo: RepositoryNwo,
codeql: CodeQL,
config: Config,
apiDetails: GitHubApiDetails,
features: FeatureEnablement,
logger: Logger,
): Promise<void> {
): Promise<DatabaseUploadResult[]> {
if (actionsUtil.getRequiredInput("upload-database") !== "true") {
logger.debug("Database upload disabled in workflow. Skipping upload.");
return;
return [];
}
if (!config.analysisKinds.includes(AnalysisKind.CodeScanning)) {
logger.debug(
`Not uploading database because 'analysis-kinds: ${AnalysisKind.CodeScanning}' is not enabled.`,
);
return;
return [];
}
if (util.isInTestMode()) {
logger.debug("In test mode. Skipping database upload.");
return;
return [];
}
// Do nothing when not running against github.com
@@ -41,19 +58,27 @@ export async function uploadDatabases(
config.gitHubVersion.type !== util.GitHubVariant.GHE_DOTCOM
) {
logger.debug("Not running against github.com or GHEC-DR. Skipping upload.");
return;
return [];
}
if (!(await gitUtils.isAnalyzingDefaultBranch())) {
// We only want to upload a database if we are analyzing the default branch.
logger.debug("Not analyzing default branch. Skipping upload.");
return;
return [];
}
// If config.overlayDatabaseMode is OverlayBase, then we have overlay base databases for all languages.
const isOverlayBase =
config.overlayDatabaseMode === OverlayDatabaseMode.OverlayBase &&
(await features.getValue(Feature.UploadOverlayDbToApi));
const cleanupLevel = isOverlayBase
? CleanupLevel.Overlay
: CleanupLevel.Clear;
// Clean up the database, since intermediate results may still be written to the
// database if there is high RAM pressure.
await withGroupAsync("Cleaning up databases", async () => {
await codeql.databaseCleanupCluster(config, "clear");
await codeql.databaseCleanupCluster(config, cleanupLevel);
});
const client = getApiClient();
@@ -68,6 +93,7 @@ export async function uploadDatabases(
uploadsBaseUrl = uploadsBaseUrl.slice(0, -1);
}
const reports: DatabaseUploadResult[] = [];
for (const language of config.languages) {
try {
// Upload the database bundle.
@@ -81,6 +107,7 @@ export async function uploadDatabases(
actionsUtil.getRequiredInput("checkout_path"),
);
try {
const startTime = Date.now();
await client.request(
`POST /repos/:owner/:repo/code-scanning/codeql/databases/:language?name=:name&commit_oid=:commit_oid`,
{
@@ -98,6 +125,13 @@ export async function uploadDatabases(
},
},
);
const endTime = Date.now();
reports.push({
language,
zipped_upload_size_bytes: bundledDbSize,
is_overlay_base: isOverlayBase,
upload_duration_ms: endTime - startTime,
});
logger.debug(`Successfully uploaded database for ${language}`);
} finally {
bundledDbReadStream.close();
@@ -105,6 +139,11 @@ export async function uploadDatabases(
} catch (e) {
// Log a warning but don't fail the workflow
logger.warning(`Failed to upload database for ${language}: ${e}`);
reports.push({
language,
error: util.getErrorMessage(e),
});
}
}
return reports;
}

View File

@@ -1,6 +1,6 @@
{
"bundleVersion": "codeql-bundle-v2.23.5",
"cliVersion": "2.23.5",
"priorBundleVersion": "codeql-bundle-v2.23.3",
"priorCliVersion": "2.23.3"
"bundleVersion": "codeql-bundle-v2.23.6",
"cliVersion": "2.23.6",
"priorBundleVersion": "codeql-bundle-v2.23.5",
"priorCliVersion": "2.23.5"
}

View File

@@ -7,6 +7,7 @@ import test from "ava";
import * as sinon from "sinon";
import { cacheKeyHashLength } from "./caching-utils";
import * as cachingUtils from "./caching-utils";
import { createStubCodeQL } from "./codeql";
import {
CacheConfig,
@@ -20,6 +21,8 @@ import {
downloadDependencyCaches,
CacheHitKind,
cacheKey,
uploadDependencyCaches,
CacheStoreResult,
} from "./dependency-caching";
import { Feature } from "./feature-flags";
import { KnownLanguage } from "./languages";
@@ -29,6 +32,7 @@ import {
getRecordingLogger,
checkExpectedLogMessages,
LoggedMessage,
createTestConfig,
} from "./testing-utils";
import { withTmpDir } from "./util";
@@ -237,15 +241,17 @@ test("downloadDependencyCaches - does not restore caches with feature keys if no
.resolves(CSHARP_BASE_PATTERNS);
makePatternCheckStub.withArgs(CSHARP_EXTRA_PATTERNS).resolves(undefined);
const results = await downloadDependencyCaches(
const result = await downloadDependencyCaches(
codeql,
createFeatures([]),
[KnownLanguage.csharp],
logger,
);
t.is(results.length, 1);
t.is(results[0].language, KnownLanguage.csharp);
t.is(results[0].hit_kind, CacheHitKind.Miss);
const statusReport = result.statusReport;
t.is(statusReport.length, 1);
t.is(statusReport[0].language, KnownLanguage.csharp);
t.is(statusReport[0].hit_kind, CacheHitKind.Miss);
t.deepEqual(result.restoredKeys, []);
t.assert(restoreCacheStub.calledOnce);
});
@@ -257,7 +263,8 @@ test("downloadDependencyCaches - restores caches with feature keys if features a
const logger = getRecordingLogger(messages);
const features = createFeatures([Feature.CsharpNewCacheKey]);
sinon.stub(glob, "hashFiles").resolves("abcdef");
const mockHash = "abcdef";
sinon.stub(glob, "hashFiles").resolves(mockHash);
const keyWithFeature = await cacheKey(
codeql,
@@ -277,15 +284,28 @@ test("downloadDependencyCaches - restores caches with feature keys if features a
.resolves(CSHARP_BASE_PATTERNS);
makePatternCheckStub.withArgs(CSHARP_EXTRA_PATTERNS).resolves(undefined);
const results = await downloadDependencyCaches(
const result = await downloadDependencyCaches(
codeql,
features,
[KnownLanguage.csharp],
logger,
);
t.is(results.length, 1);
t.is(results[0].language, KnownLanguage.csharp);
t.is(results[0].hit_kind, CacheHitKind.Exact);
// Check that the status report for telemetry indicates that one cache was restored with an exact match.
const statusReport = result.statusReport;
t.is(statusReport.length, 1);
t.is(statusReport[0].language, KnownLanguage.csharp);
t.is(statusReport[0].hit_kind, CacheHitKind.Exact);
// Check that the restored key has been returned.
const restoredKeys = result.restoredKeys;
t.is(restoredKeys.length, 1);
t.assert(
restoredKeys[0].endsWith(mockHash),
"Expected restored key to end with hash returned by `hashFiles`",
);
// `restoreCache` should have been called exactly once.
t.assert(restoreCacheStub.calledOnce);
});
@@ -297,8 +317,14 @@ test("downloadDependencyCaches - restores caches with feature keys if features a
const logger = getRecordingLogger(messages);
const features = createFeatures([Feature.CsharpNewCacheKey]);
// We expect two calls to `hashFiles`: the first by the call to `cacheKey` below,
// and the second by `downloadDependencyCaches`. We use the result of the first
// call as part of the cache key that identifies a mock, existing cache. The result
// of the second call is for the primary restore key, which we don't want to match
// the first key so that we can test the restore keys logic.
const restoredHash = "abcdef";
const hashFilesStub = sinon.stub(glob, "hashFiles");
hashFilesStub.onFirstCall().resolves("abcdef");
hashFilesStub.onFirstCall().resolves(restoredHash);
hashFilesStub.onSecondCall().resolves("123456");
const keyWithFeature = await cacheKey(
@@ -319,18 +345,230 @@ test("downloadDependencyCaches - restores caches with feature keys if features a
.resolves(CSHARP_BASE_PATTERNS);
makePatternCheckStub.withArgs(CSHARP_EXTRA_PATTERNS).resolves(undefined);
const results = await downloadDependencyCaches(
const result = await downloadDependencyCaches(
codeql,
features,
[KnownLanguage.csharp],
logger,
);
t.is(results.length, 1);
t.is(results[0].language, KnownLanguage.csharp);
t.is(results[0].hit_kind, CacheHitKind.Partial);
// Check that the status report for telemetry indicates that one cache was restored with a partial match.
const statusReport = result.statusReport;
t.is(statusReport.length, 1);
t.is(statusReport[0].language, KnownLanguage.csharp);
t.is(statusReport[0].hit_kind, CacheHitKind.Partial);
// Check that the restored key has been returned.
const restoredKeys = result.restoredKeys;
t.is(restoredKeys.length, 1);
t.assert(
restoredKeys[0].endsWith(restoredHash),
"Expected restored key to end with hash returned by `hashFiles`",
);
t.assert(restoreCacheStub.calledOnce);
});
test("uploadDependencyCaches - skips upload for a language with no cache config", async (t) => {
const codeql = createStubCodeQL({});
const messages: LoggedMessage[] = [];
const logger = getRecordingLogger(messages);
const features = createFeatures([]);
const config = createTestConfig({
languages: [KnownLanguage.actions],
});
const result = await uploadDependencyCaches(codeql, features, config, logger);
t.is(result.length, 0);
checkExpectedLogMessages(t, messages, [
"Skipping upload of dependency cache for actions",
]);
});
test("uploadDependencyCaches - skips upload if no files for the hash exist", async (t) => {
const codeql = createStubCodeQL({});
const messages: LoggedMessage[] = [];
const logger = getRecordingLogger(messages);
const features = createFeatures([]);
const config = createTestConfig({
languages: [KnownLanguage.go],
});
const makePatternCheckStub = sinon.stub(internal, "makePatternCheck");
makePatternCheckStub.resolves(undefined);
const result = await uploadDependencyCaches(codeql, features, config, logger);
t.is(result.length, 1);
t.is(result[0].language, KnownLanguage.go);
t.is(result[0].result, CacheStoreResult.NoHash);
});
test("uploadDependencyCaches - skips upload if we know the cache already exists", async (t) => {
process.env["RUNNER_OS"] = "Linux";
const codeql = createStubCodeQL({});
const messages: LoggedMessage[] = [];
const logger = getRecordingLogger(messages);
const features = createFeatures([]);
const mockHash = "abcdef";
sinon.stub(glob, "hashFiles").resolves(mockHash);
const makePatternCheckStub = sinon.stub(internal, "makePatternCheck");
makePatternCheckStub
.withArgs(CSHARP_BASE_PATTERNS)
.resolves(CSHARP_BASE_PATTERNS);
const primaryCacheKey = await cacheKey(
codeql,
features,
KnownLanguage.csharp,
CSHARP_BASE_PATTERNS,
);
const config = createTestConfig({
languages: [KnownLanguage.csharp],
dependencyCachingRestoredKeys: [primaryCacheKey],
});
const result = await uploadDependencyCaches(codeql, features, config, logger);
t.is(result.length, 1);
t.is(result[0].language, KnownLanguage.csharp);
t.is(result[0].result, CacheStoreResult.Duplicate);
});
test("uploadDependencyCaches - skips upload if cache size is 0", async (t) => {
process.env["RUNNER_OS"] = "Linux";
const codeql = createStubCodeQL({});
const messages: LoggedMessage[] = [];
const logger = getRecordingLogger(messages);
const features = createFeatures([]);
const mockHash = "abcdef";
sinon.stub(glob, "hashFiles").resolves(mockHash);
const makePatternCheckStub = sinon.stub(internal, "makePatternCheck");
makePatternCheckStub
.withArgs(CSHARP_BASE_PATTERNS)
.resolves(CSHARP_BASE_PATTERNS);
sinon.stub(cachingUtils, "getTotalCacheSize").resolves(0);
const config = createTestConfig({
languages: [KnownLanguage.csharp],
});
const result = await uploadDependencyCaches(codeql, features, config, logger);
t.is(result.length, 1);
t.is(result[0].language, KnownLanguage.csharp);
t.is(result[0].result, CacheStoreResult.Empty);
checkExpectedLogMessages(t, messages, [
"Skipping upload of dependency cache",
]);
});
test("uploadDependencyCaches - uploads caches when all requirements are met", async (t) => {
process.env["RUNNER_OS"] = "Linux";
const codeql = createStubCodeQL({});
const messages: LoggedMessage[] = [];
const logger = getRecordingLogger(messages);
const features = createFeatures([]);
const mockHash = "abcdef";
sinon.stub(glob, "hashFiles").resolves(mockHash);
const makePatternCheckStub = sinon.stub(internal, "makePatternCheck");
makePatternCheckStub
.withArgs(CSHARP_BASE_PATTERNS)
.resolves(CSHARP_BASE_PATTERNS);
sinon.stub(cachingUtils, "getTotalCacheSize").resolves(1024);
sinon.stub(actionsCache, "saveCache").resolves();
const config = createTestConfig({
languages: [KnownLanguage.csharp],
});
const result = await uploadDependencyCaches(codeql, features, config, logger);
t.is(result.length, 1);
t.is(result[0].language, KnownLanguage.csharp);
t.is(result[0].result, CacheStoreResult.Stored);
t.is(result[0].upload_size_bytes, 1024);
checkExpectedLogMessages(t, messages, ["Uploading cache of size"]);
});
test("uploadDependencyCaches - catches `ReserveCacheError` exceptions", async (t) => {
process.env["RUNNER_OS"] = "Linux";
const codeql = createStubCodeQL({});
const messages: LoggedMessage[] = [];
const logger = getRecordingLogger(messages);
const features = createFeatures([]);
const mockHash = "abcdef";
sinon.stub(glob, "hashFiles").resolves(mockHash);
const makePatternCheckStub = sinon.stub(internal, "makePatternCheck");
makePatternCheckStub
.withArgs(CSHARP_BASE_PATTERNS)
.resolves(CSHARP_BASE_PATTERNS);
sinon.stub(cachingUtils, "getTotalCacheSize").resolves(1024);
sinon
.stub(actionsCache, "saveCache")
.throws(new actionsCache.ReserveCacheError("Already in use"));
const config = createTestConfig({
languages: [KnownLanguage.csharp],
});
await t.notThrowsAsync(async () => {
const result = await uploadDependencyCaches(
codeql,
features,
config,
logger,
);
t.is(result.length, 1);
t.is(result[0].language, KnownLanguage.csharp);
t.is(result[0].result, CacheStoreResult.Duplicate);
checkExpectedLogMessages(t, messages, ["Not uploading cache for"]);
});
});
test("uploadDependencyCaches - throws other exceptions", async (t) => {
process.env["RUNNER_OS"] = "Linux";
const codeql = createStubCodeQL({});
const messages: LoggedMessage[] = [];
const logger = getRecordingLogger(messages);
const features = createFeatures([]);
const mockHash = "abcdef";
sinon.stub(glob, "hashFiles").resolves(mockHash);
const makePatternCheckStub = sinon.stub(internal, "makePatternCheck");
makePatternCheckStub
.withArgs(CSHARP_BASE_PATTERNS)
.resolves(CSHARP_BASE_PATTERNS);
sinon.stub(cachingUtils, "getTotalCacheSize").resolves(1024);
sinon.stub(actionsCache, "saveCache").throws();
const config = createTestConfig({
languages: [KnownLanguage.csharp],
});
await t.throwsAsync(async () => {
await uploadDependencyCaches(codeql, features, config, logger);
});
});
test("getFeaturePrefix - returns empty string if no features are enabled", async (t) => {
const codeql = createStubCodeQL({});
const features = createFeatures([]);

View File

@@ -55,7 +55,7 @@ export function getJavaTempDependencyDir(): string {
* @returns The paths of directories on the runner that should be included in a dependency cache
* for a Java analysis.
*/
export function getJavaDependencyDirs(): string[] {
function getJavaDependencyDirs(): string[] {
return [
// Maven
join(os.homedir(), ".m2", "repository"),
@@ -193,6 +193,14 @@ export interface DependencyCacheRestoreStatus {
/** An array of `DependencyCacheRestoreStatus` objects for each analysed language with a caching configuration. */
export type DependencyCacheRestoreStatusReport = DependencyCacheRestoreStatus[];
/** Represents the results of `downloadDependencyCaches`. */
export interface DownloadDependencyCachesResult {
/** The status report for telemetry */
statusReport: DependencyCacheRestoreStatusReport;
/** An array of cache keys that we have restored and therefore know to exist. */
restoredKeys: string[];
}
/**
* A wrapper around `cacheConfig.getHashPatterns` which logs when there are no files to calculate
* a hash for the cache key from.
@@ -239,8 +247,9 @@ export async function downloadDependencyCaches(
features: FeatureEnablement,
languages: Language[],
logger: Logger,
): Promise<DependencyCacheRestoreStatusReport> {
): Promise<DownloadDependencyCachesResult> {
const status: DependencyCacheRestoreStatusReport = [];
const restoredKeys: string[] = [];
for (const language of languages) {
const cacheConfig = defaultCacheConfigs[language];
@@ -288,16 +297,27 @@ export async function downloadDependencyCaches(
if (hitKey !== undefined) {
logger.info(`Cache hit on key ${hitKey} for ${language}.`);
const hit_kind =
hitKey === primaryKey ? CacheHitKind.Exact : CacheHitKind.Partial;
status.push({ language, hit_kind, download_duration_ms });
// We have a partial cache hit, unless the key of the restored cache matches the
// primary restore key.
let hit_kind = CacheHitKind.Partial;
if (hitKey === primaryKey) {
hit_kind = CacheHitKind.Exact;
}
status.push({
language,
hit_kind,
download_duration_ms,
});
restoredKeys.push(hitKey);
} else {
status.push({ language, hit_kind: CacheHitKind.Miss });
logger.info(`No suitable cache found for ${language}.`);
}
}
return status;
return { statusReport: status, restoredKeys };
}
/** Enumerates possible outcomes for storing caches. */
@@ -365,6 +385,18 @@ export async function uploadDependencyCaches(
continue;
}
// Now that we have verified that there are suitable files, compute the hash for the cache key.
const key = await cacheKey(codeql, features, language, patterns);
// Check that we haven't previously restored this exact key. If a cache with this key
// already exists in the Actions Cache, performing the next steps is pointless as the cache
// will not get overwritten. We can therefore skip the expensive work of measuring the size
// of the cache contents and attempting to upload it if we know that the cache already exists.
if (config.dependencyCachingRestoredKeys.includes(key)) {
status.push({ language, result: CacheStoreResult.Duplicate });
continue;
}
// Calculate the size of the files that we would store in the cache. We use this to determine whether the
// cache should be saved or not. For example, if there are no files to store, then we skip creating the
// cache. In the future, we could also:
@@ -390,8 +422,6 @@ export async function uploadDependencyCaches(
continue;
}
const key = await cacheKey(codeql, features, language, patterns);
logger.info(
`Uploading cache of size ${size} for ${language} with key ${key}...`,
);

View File

@@ -20,12 +20,6 @@ export enum EnvVar {
/** Whether the CodeQL Action has invoked the Go autobuilder. */
DID_AUTOBUILD_GOLANG = "CODEQL_ACTION_DID_AUTOBUILD_GOLANG",
/**
* Whether to disable the SARIF post-processing in the Action that removes duplicate locations from
* notifications in the `run[].invocations[].toolExecutionNotifications` SARIF property.
*/
DISABLE_DUPLICATE_LOCATION_FIX = "CODEQL_ACTION_DISABLE_DUPLICATE_LOCATION_FIX",
/**
* Whether the CodeQL Action is using its own deprecated and non-standard way of scanning for
* multiple languages.
@@ -56,20 +50,12 @@ export enum EnvVar {
/** Whether the error for a deprecated version of the CodeQL Action was logged. */
LOG_VERSION_DEPRECATION = "CODEQL_ACTION_DID_LOG_VERSION_DEPRECATION",
/**
* For macOS. Result of `csrutil status` to determine whether System Integrity
* Protection is enabled.
*/
IS_SIP_ENABLED = "CODEQL_ACTION_IS_SIP_ENABLED",
/** UUID representing the current job run. */
JOB_RUN_UUID = "JOB_RUN_UUID",
/** Status for the entire job, submitted to the status report in `init-post` */
JOB_STATUS = "CODEQL_ACTION_JOB_STATUS",
ODASA_TRACER_CONFIGURATION = "ODASA_TRACER_CONFIGURATION",
/** The value of the `output` input for the analyze action. */
SARIF_RESULTS_OUTPUT_DIR = "CODEQL_ACTION_SARIF_RESULTS_OUTPUT_DIR",

View File

@@ -77,6 +77,7 @@ export enum Feature {
OverlayAnalysisSwift = "overlay_analysis_swift",
PythonDefaultIsToNotExtractStdlib = "python_default_is_to_not_extract_stdlib",
QaTelemetryEnabled = "qa_telemetry_enabled",
UploadOverlayDbToApi = "upload_overlay_db_to_api",
UseRepositoryProperties = "use_repository_properties",
ValidateDbConfig = "validate_db_config",
}
@@ -166,6 +167,11 @@ export const featureConfig: Record<
legacyApi: true,
minimumVersion: undefined,
},
[Feature.JavaMinimizeDependencyJars]: {
defaultValue: false,
envVar: "CODEQL_ACTION_JAVA_MINIMIZE_DEPENDENCY_JARS",
minimumVersion: "2.23.0",
},
[Feature.OverlayAnalysis]: {
defaultValue: false,
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS",
@@ -277,21 +283,21 @@ export const featureConfig: Record<
minimumVersion: undefined,
toolsFeature: ToolsFeature.PythonDefaultIsToNotExtractStdlib,
},
[Feature.UseRepositoryProperties]: {
defaultValue: false,
envVar: "CODEQL_ACTION_USE_REPOSITORY_PROPERTIES",
minimumVersion: undefined,
},
[Feature.QaTelemetryEnabled]: {
defaultValue: false,
envVar: "CODEQL_ACTION_QA_TELEMETRY",
legacyApi: true,
minimumVersion: undefined,
},
[Feature.JavaMinimizeDependencyJars]: {
[Feature.UploadOverlayDbToApi]: {
defaultValue: false,
envVar: "CODEQL_ACTION_JAVA_MINIMIZE_DEPENDENCY_JARS",
minimumVersion: "2.23.0",
envVar: "CODEQL_ACTION_UPLOAD_OVERLAY_DB_TO_API",
minimumVersion: undefined,
},
[Feature.UseRepositoryProperties]: {
defaultValue: false,
envVar: "CODEQL_ACTION_USE_REPOSITORY_PROPERTIES",
minimumVersion: undefined,
},
[Feature.ValidateDbConfig]: {
defaultValue: false,

View File

@@ -122,67 +122,6 @@ export const determineBaseBranchHeadCommitOid = async function (
}
};
/**
* Deepen the git history of HEAD by one level. Errors are logged.
*
* This function uses the `checkout_path` to determine the repository path and
* works only when called from `analyze` or `upload-sarif`.
*/
export const deepenGitHistory = async function () {
try {
await runGitCommand(
getOptionalInput("checkout_path"),
[
"fetch",
"origin",
"HEAD",
"--no-tags",
"--no-recurse-submodules",
"--deepen=1",
],
"Cannot deepen the shallow repository.",
);
} catch {
// Errors are already logged by runGitCommand()
}
};
/**
* Fetch the given remote branch. Errors are logged.
*
* This function uses the `checkout_path` to determine the repository path and
* works only when called from `analyze` or `upload-sarif`.
*/
export const gitFetch = async function (branch: string, extraFlags: string[]) {
try {
await runGitCommand(
getOptionalInput("checkout_path"),
["fetch", "--no-tags", ...extraFlags, "origin", `${branch}:${branch}`],
`Cannot fetch ${branch}.`,
);
} catch {
// Errors are already logged by runGitCommand()
}
};
/**
* Repack the git repository, using with the given flags. Errors are logged.
*
* This function uses the `checkout_path` to determine the repository path and
* works only when called from `analyze` or `upload-sarif`.
*/
export const gitRepack = async function (flags: string[]) {
try {
await runGitCommand(
getOptionalInput("checkout_path"),
["repack", ...flags],
"Cannot repack the repository.",
);
} catch {
// Errors are already logged by runGitCommand()
}
};
/**
* Decode, if necessary, a file path produced by Git. See
* https://git-scm.com/docs/git-config#Documentation/git-config.txt-corequotePath

View File

@@ -371,7 +371,7 @@ async function run() {
}
let overlayBaseDatabaseStats: OverlayBaseDatabaseDownloadStats | undefined;
let dependencyCachingResults: DependencyCacheRestoreStatusReport | undefined;
let dependencyCachingStatus: DependencyCacheRestoreStatusReport | undefined;
try {
if (
config.overlayDatabaseMode === OverlayDatabaseMode.Overlay &&
@@ -579,12 +579,15 @@ async function run() {
// Restore dependency cache(s), if they exist.
if (shouldRestoreCache(config.dependencyCachingEnabled)) {
dependencyCachingResults = await downloadDependencyCaches(
const dependencyCachingResult = await downloadDependencyCaches(
codeql,
features,
config.languages,
logger,
);
dependencyCachingStatus = dependencyCachingResult.statusReport;
config.dependencyCachingRestoredKeys =
dependencyCachingResult.restoredKeys;
}
// Suppress warnings about disabled Python library extraction.
@@ -732,7 +735,7 @@ async function run() {
toolsSource,
toolsVersion,
overlayBaseDatabaseStats,
dependencyCachingResults,
dependencyCachingStatus,
logger,
error,
);
@@ -755,7 +758,7 @@ async function run() {
toolsSource,
toolsVersion,
overlayBaseDatabaseStats,
dependencyCachingResults,
dependencyCachingStatus,
logger,
);
}

View File

@@ -16,6 +16,7 @@ import { type Config } from "./config-utils";
import { getCommitOid, getFileOidsUnderPath } from "./git-utils";
import { Logger, withGroupAsync } from "./logging";
import {
CleanupLevel,
getErrorMessage,
isInTestMode,
tryGetFolderBytes,
@@ -28,7 +29,7 @@ export enum OverlayDatabaseMode {
None = "none",
}
export const CODEQL_OVERLAY_MINIMUM_VERSION = "2.22.4";
export const CODEQL_OVERLAY_MINIMUM_VERSION = "2.23.5";
/**
* The maximum (uncompressed) size of the overlay base database that we will
@@ -175,7 +176,7 @@ const MAX_CACHE_OPERATION_MS = 600_000;
* @param warningPrefix Prefix for the check failure warning message
* @returns True if the verification succeeded, false otherwise
*/
export function checkOverlayBaseDatabase(
function checkOverlayBaseDatabase(
config: Config,
logger: Logger,
warningPrefix: string,
@@ -204,7 +205,7 @@ export function checkOverlayBaseDatabase(
* @returns A promise that resolves to true if the upload was performed and
* successfully completed, or false otherwise
*/
export async function uploadOverlayBaseDatabaseToCache(
export async function cleanupAndUploadOverlayBaseDatabaseToCache(
codeql: CodeQL,
config: Config,
logger: Logger,
@@ -242,7 +243,7 @@ export async function uploadOverlayBaseDatabaseToCache(
// Clean up the database using the overlay cleanup level.
await withGroupAsync("Cleaning up databases", async () => {
await codeql.databaseCleanupCluster(config, "overlay");
await codeql.databaseCleanupCluster(config, CleanupLevel.Overlay);
});
const dbLocation = config.dbLocation;

View File

@@ -34,7 +34,7 @@ export enum ToolsSource {
Download = "DOWNLOAD",
}
export const CODEQL_DEFAULT_ACTION_REPOSITORY = "github/codeql-action";
const CODEQL_DEFAULT_ACTION_REPOSITORY = "github/codeql-action";
const CODEQL_NIGHTLIES_REPOSITORY_OWNER = "dsp-testing";
const CODEQL_NIGHTLIES_REPOSITORY_NAME = "codeql-cli-nightlies";
@@ -180,17 +180,6 @@ export function tryGetTagNameFromUrl(
return match[1];
}
export function tryGetBundleVersionFromUrl(
url: string,
logger: Logger,
): string | undefined {
const tagName = tryGetTagNameFromUrl(url, logger);
if (tagName === undefined) {
return undefined;
}
return tryGetBundleVersionFromTagName(tagName, logger);
}
export function convertToSemVer(version: string, logger: Logger): string {
if (!semver.valid(version)) {
logger.debug(
@@ -580,7 +569,7 @@ export async function getCodeQLSource(
* Gets a fallback version number to use when looking for CodeQL in the toolcache if we didn't find
* the `x.y.z` version. This is to support old versions of the toolcache.
*/
export async function tryGetFallbackToolcacheVersion(
async function tryGetFallbackToolcacheVersion(
cliVersion: string | undefined,
tagName: string,
logger: Logger,
@@ -729,7 +718,7 @@ function getCanonicalToolcacheVersion(
return cliVersion;
}
export interface SetupCodeQLResult {
interface SetupCodeQLResult {
codeqlFolder: string;
toolsDownloadStatusReport?: ToolsDownloadStatusReport;
toolsSource: ToolsSource;
@@ -750,7 +739,7 @@ export async function setupCodeQLBundle(
defaultCliVersion: CodeQLDefaultVersionInfo,
features: FeatureEnablement,
logger: Logger,
) {
): Promise<SetupCodeQLResult> {
if (!(await util.isBinaryAccessible("tar", logger))) {
throw new util.ConfigurationError(
"Could not find tar in PATH, so unable to extract CodeQL bundle.",

View File

@@ -8,7 +8,7 @@ import { ConfigurationError, getErrorMessage, isDefined } from "./util";
export const UPDATEJOB_PROXY = "update-job-proxy";
export const UPDATEJOB_PROXY_VERSION = "v2.0.20250624110901";
export const UPDATEJOB_PROXY_URL_PREFIX =
const UPDATEJOB_PROXY_URL_PREFIX =
"https://github.com/github/codeql-action/releases/download/codeql-bundle-v2.22.0/";
export type Credential = {
@@ -202,7 +202,7 @@ export function getFallbackUrl(proxyPackage: string): string {
*
* @returns The response from the GitHub API.
*/
export async function getLinkedRelease() {
async function getLinkedRelease() {
return getApiClient().rest.repos.getReleaseByTag({
owner: "github",
repo: "codeql-action",

View File

@@ -54,7 +54,7 @@ export enum ActionName {
* considered to be a third party analysis and is treated differently when calculating SLOs. To ensure
* misconfigured workflows are not treated as third party, only the upload-sarif action can return false.
*/
export function isFirstPartyAnalysis(actionName: ActionName): boolean {
function isFirstPartyAnalysis(actionName: ActionName): boolean {
if (actionName !== ActionName.UploadSarif) {
return true;
}

View File

@@ -392,6 +392,7 @@ export function createTestConfig(overrides: Partial<Config>): Config {
trapCaches: {},
trapCacheDownloadTime: 0,
dependencyCachingEnabled: CachingKind.None,
dependencyCachingRestoredKeys: [],
extraQueryExclusions: [],
overlayDatabaseMode: OverlayDatabaseMode.None,
useOverlayDatabaseCaching: false,

View File

@@ -17,7 +17,7 @@ import { cleanUpPath, getErrorMessage, getRequiredEnvParam } from "./util";
/**
* High watermark to use when streaming the download and extraction of the CodeQL tools.
*/
export const STREAMING_HIGH_WATERMARK_BYTES = 4 * 1024 * 1024; // 4 MiB
const STREAMING_HIGH_WATERMARK_BYTES = 4 * 1024 * 1024; // 4 MiB
/**
* The name of the tool cache directory for the CodeQL tools.

View File

@@ -76,7 +76,7 @@ export async function endTracingForCluster(
}
}
export async function getTracerConfigForCluster(
async function getTracerConfigForCluster(
config: Config,
): Promise<TracerConfig> {
const tracingEnvVariables = JSON.parse(

View File

@@ -412,7 +412,7 @@ export function findSarifFilesInDir(
return sarifFiles;
}
export function getSarifFilePaths(
function getSarifFilePaths(
sarifPath: string,
isSarif: (name: string) => boolean,
) {

View File

@@ -476,7 +476,7 @@ for (const [
githubVersion,
)}`;
test(`checkActionVersion ${reportErrorDescription} for ${versionsDescription}`, async (t) => {
const warningSpy = sinon.spy(core, "error");
const warningSpy = sinon.spy(core, "warning");
const versionStub = sinon
.stub(api, "getGitHubVersion")
.resolves(githubVersion);

View File

@@ -4,7 +4,6 @@ import * as os from "os";
import * as path from "path";
import * as core from "@actions/core";
import * as exec from "@actions/exec/lib/exec";
import * as io from "@actions/io";
import getFolderSize from "get-folder-size";
import * as yaml from "js-yaml";
@@ -1026,34 +1025,6 @@ export function fixInvalidNotifications(
return newSarif;
}
/**
* Removes duplicates from the sarif file.
*
* When `CODEQL_ACTION_DISABLE_DUPLICATE_LOCATION_FIX` is set to true, this will
* simply rename the input file to the output file. Otherwise, it will parse the
* input file as JSON, remove duplicate locations from the SARIF notification
* objects, and write the result to the output file.
*
* For context, see documentation of:
* `CODEQL_ACTION_DISABLE_DUPLICATE_LOCATION_FIX`. */
export function fixInvalidNotificationsInFile(
inputPath: string,
outputPath: string,
logger: Logger,
): void {
if (process.env[EnvVar.DISABLE_DUPLICATE_LOCATION_FIX] === "true") {
logger.info(
"SARIF notification object duplicate location fix disabled by the " +
`${EnvVar.DISABLE_DUPLICATE_LOCATION_FIX} environment variable.`,
);
fs.renameSync(inputPath, outputPath);
} else {
let sarif = JSON.parse(fs.readFileSync(inputPath, "utf8")) as SarifFile;
sarif = fixInvalidNotifications(sarif, logger);
fs.writeFileSync(outputPath, JSON.stringify(sarif));
}
}
export function wrapError(error: unknown): Error {
return error instanceof Error ? error : new Error(String(error));
}
@@ -1141,7 +1112,7 @@ export function checkActionVersion(
">=3.20",
))
) {
core.error(
core.warning(
"CodeQL Action v3 will be deprecated in December 2026. " +
"Please update all occurrences of the CodeQL Action in your workflow files to v4. " +
"For more information, see " +
@@ -1197,49 +1168,6 @@ export function cloneObject<T>(obj: T): T {
return JSON.parse(JSON.stringify(obj)) as T;
}
// The first time this function is called, it runs `csrutil status` to determine
// whether System Integrity Protection is enabled; and saves the result in an
// environment variable. Afterwards, simply return the value of the environment
// variable.
export async function checkSipEnablement(
logger: Logger,
): Promise<boolean | undefined> {
if (
process.env[EnvVar.IS_SIP_ENABLED] !== undefined &&
["true", "false"].includes(process.env[EnvVar.IS_SIP_ENABLED])
) {
return process.env[EnvVar.IS_SIP_ENABLED] === "true";
}
try {
const sipStatusOutput = await exec.getExecOutput("csrutil status");
if (sipStatusOutput.exitCode === 0) {
if (
sipStatusOutput.stdout.includes(
"System Integrity Protection status: enabled.",
)
) {
core.exportVariable(EnvVar.IS_SIP_ENABLED, "true");
return true;
}
if (
sipStatusOutput.stdout.includes(
"System Integrity Protection status: disabled.",
)
) {
core.exportVariable(EnvVar.IS_SIP_ENABLED, "false");
return false;
}
}
return undefined;
} catch (e) {
logger.warning(
`Failed to determine if System Integrity Protection was enabled: ${e}`,
);
return undefined;
}
}
export async function cleanUpPath(file: string, name: string, logger: Logger) {
logger.debug(`Cleaning up ${name}.`);
try {
@@ -1291,17 +1219,6 @@ export function isDefined<T>(value: T | null | undefined): value is T {
return value !== undefined && value !== null;
}
/** Like `Object.keys`, but typed so that the elements of the resulting array have the
* same type as the keys of the input object. Note that this may not be sound if the input
* object has been cast to `T` from a subtype of `T` and contains additional keys that
* are not represented by `keyof T`.
*/
export function unsafeKeysInvariant<T extends Record<string, any>>(
object: T,
): Array<keyof T> {
return Object.keys(object) as Array<keyof T>;
}
/** Like `Object.entries`, but typed so that the key elements of the result have the
* same type as the keys of the input object. Note that this may not be sound if the input
* object has been cast to `T` from a subtype of `T` and contains additional keys that
@@ -1314,3 +1231,8 @@ export function unsafeEntriesInvariant<T extends Record<string, any>>(
([_, val]) => val !== undefined,
) as Array<[keyof T, Exclude<T[keyof T], undefined>]>;
}
export enum CleanupLevel {
Clear = "clear",
Overlay = "overlay",
}