Compare commits

..

1 Commits

Author SHA1 Message Date
Michael B. Gale
426799f154 Allow query-filters for code-quality 2025-11-04 21:23:50 +00:00
99 changed files with 7908 additions and 9267 deletions

View File

@@ -18,25 +18,14 @@ For internal use only. Please select the risk level of this change:
#### Which use cases does this change impact?
<!-- Delete options that don't apply. If in doubt, do not delete an option. -->
<!-- Delete options that don't apply. -->
Workflow types:
- **Advanced setup** - Impacts users who have custom CodeQL workflows.
- **Managed** - Impacts users with `dynamic` workflows (Default Setup, CCR, ...).
Products:
- **Code Scanning** - The changes impact analyses when `analysis-kinds: code-scanning`.
- **Code Quality** - The changes impact analyses when `analysis-kinds: code-quality`.
- **CCR** - The changes impact analyses for Copilot Code Reviews.
- **Third-party analyses** - The changes affect the `upload-sarif` action.
Environments:
- **Dotcom** - Impacts CodeQL workflows on `github.com`.
- **GHES** - Impacts CodeQL workflows on GitHub Enterprise Server.
- **Testing/None** - This change does not impact any CodeQL workflows in production.
- **Advanced setup** - Impacts users who have custom workflows.
- **Default setup** - Impacts users who use default setup.
- **Code Scanning** - Impacts Code Scanning (i.e. `analysis-kinds: code-scanning`).
- **Code Quality** - Impacts Code Quality (i.e. `analysis-kinds: code-quality`).
- **Third-party analyses** - Impacts third-party analyses (i.e. `upload-sarif`).
- **GHES** - Impacts GitHub Enterprise Server.
#### How did/will you validate this change?
@@ -65,15 +54,6 @@ Environments:
- **Alerts** - New or existing monitors will trip if something goes wrong with this change.
- **Other** - Please provide details.
#### Are there any special considerations for merging or releasing this change?
<!--
Consider whether this change depends on a different change in another repository that should be released first.
-->
- **No special considerations** - This change can be merged at any time.
- **Special considerations** - This change should only be merged once certain preconditions are met. Please provide details of those or link to this PR from an internal issue.
### Merge / deployment checklist
- Confirm this change is backwards compatible with existing workflows.

View File

@@ -27,11 +27,6 @@ on:
description: The version of Go to install
required: false
default: '>=1.21.0'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
workflow_call:
inputs:
go-version:
@@ -39,11 +34,6 @@ on:
description: The version of Go to install
required: false
default: '>=1.21.0'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
defaults:
run:
shell: bash
@@ -84,10 +74,6 @@ jobs:
with:
go-version: ${{ inputs.go-version || '>=1.21.0' }}
cache: false
- name: Install .NET
uses: actions/setup-dotnet@v5
with:
dotnet-version: ${{ inputs.dotnet-version || '9.x' }}
- id: init
uses: ./../action/init
with:

View File

@@ -32,11 +32,6 @@ on:
description: The version of Python to install
required: false
default: '3.13'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
workflow_call:
inputs:
go-version:
@@ -49,11 +44,6 @@ on:
description: The version of Python to install
required: false
default: '3.13'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
defaults:
run:
shell: bash
@@ -95,10 +85,6 @@ jobs:
uses: actions/setup-python@v6
with:
python-version: ${{ inputs.python-version || '3.13' }}
- name: Install .NET
uses: actions/setup-dotnet@v5
with:
dotnet-version: ${{ inputs.dotnet-version || '9.x' }}
- uses: ./../action/init
with:
tools: ${{ steps.prepare-test.outputs.tools-url }}

View File

@@ -21,19 +21,9 @@ on:
schedule:
- cron: '0 5 * * *'
workflow_dispatch:
inputs:
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
inputs: {}
workflow_call:
inputs:
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
inputs: {}
defaults:
run:
shell: bash
@@ -69,10 +59,6 @@ jobs:
version: ${{ matrix.version }}
use-all-platform-bundle: 'false'
setup-kotlin: 'true'
- name: Install .NET
uses: actions/setup-dotnet@v5
with:
dotnet-version: ${{ inputs.dotnet-version || '9.x' }}
- uses: ./../action/init
with:
languages: csharp

View File

@@ -27,11 +27,6 @@ on:
description: The version of Go to install
required: false
default: '>=1.21.0'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
workflow_call:
inputs:
go-version:
@@ -39,11 +34,6 @@ on:
description: The version of Go to install
required: false
default: '>=1.21.0'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
defaults:
run:
shell: bash
@@ -80,10 +70,6 @@ jobs:
with:
go-version: ${{ inputs.go-version || '>=1.21.0' }}
cache: false
- name: Install .NET
uses: actions/setup-dotnet@v5
with:
dotnet-version: ${{ inputs.dotnet-version || '9.x' }}
- uses: ./../action/init
id: init
with:

View File

@@ -27,11 +27,6 @@ on:
description: The version of Go to install
required: false
default: '>=1.21.0'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
workflow_call:
inputs:
go-version:
@@ -39,11 +34,6 @@ on:
description: The version of Go to install
required: false
default: '>=1.21.0'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
defaults:
run:
shell: bash
@@ -84,10 +74,6 @@ jobs:
with:
go-version: ${{ inputs.go-version || '>=1.21.0' }}
cache: false
- name: Install .NET
uses: actions/setup-dotnet@v5
with:
dotnet-version: ${{ inputs.dotnet-version || '9.x' }}
- uses: ./../action/init
id: init
with:

View File

@@ -27,11 +27,6 @@ on:
description: The version of Go to install
required: false
default: '>=1.21.0'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
workflow_call:
inputs:
go-version:
@@ -39,11 +34,6 @@ on:
description: The version of Go to install
required: false
default: '>=1.21.0'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
defaults:
run:
shell: bash
@@ -82,10 +72,6 @@ jobs:
with:
go-version: ${{ inputs.go-version || '>=1.21.0' }}
cache: false
- name: Install .NET
uses: actions/setup-dotnet@v5
with:
dotnet-version: ${{ inputs.dotnet-version || '9.x' }}
- uses: ./../action/init
with:
languages: go

6
.github/workflows/__go.yml generated vendored
View File

@@ -18,11 +18,6 @@ on:
description: The version of Go to install
required: false
default: '>=1.21.0'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
jobs:
go-custom-queries:
name: 'Go: Custom queries'
@@ -32,7 +27,6 @@ jobs:
uses: ./.github/workflows/__go-custom-queries.yml
with:
go-version: ${{ inputs.go-version }}
dotnet-version: ${{ inputs.dotnet-version }}
go-indirect-tracing-workaround-diagnostic:
name: 'Go: diagnostic when Go is changed after init step'
permissions:

14
.github/workflows/__local-bundle.yml generated vendored
View File

@@ -32,11 +32,6 @@ on:
description: The version of Python to install
required: false
default: '3.13'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
workflow_call:
inputs:
go-version:
@@ -49,11 +44,6 @@ on:
description: The version of Python to install
required: false
default: '3.13'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
defaults:
run:
shell: bash
@@ -95,10 +85,6 @@ jobs:
uses: actions/setup-python@v6
with:
python-version: ${{ inputs.python-version || '3.13' }}
- name: Install .NET
uses: actions/setup-dotnet@v5
with:
dotnet-version: ${{ inputs.dotnet-version || '9.x' }}
- name: Fetch latest CodeQL bundle
run: |
wget https://github.com/github/codeql-action/releases/latest/download/codeql-bundle-linux64.tar.zst

View File

@@ -32,11 +32,6 @@ on:
description: The version of Python to install
required: false
default: '3.13'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
workflow_call:
inputs:
go-version:
@@ -49,11 +44,6 @@ on:
description: The version of Python to install
required: false
default: '3.13'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
defaults:
run:
shell: bash
@@ -129,10 +119,6 @@ jobs:
uses: actions/setup-python@v6
with:
python-version: ${{ inputs.python-version || '3.13' }}
- name: Install .NET
uses: actions/setup-dotnet@v5
with:
dotnet-version: ${{ inputs.dotnet-version || '9.x' }}
- name: Use Xcode 16
if: runner.os == 'macOS' && matrix.version != 'nightly-latest'
run: sudo xcode-select -s "/Applications/Xcode_16.app"

View File

@@ -32,11 +32,6 @@ on:
description: The version of Python to install
required: false
default: '3.13'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
workflow_call:
inputs:
go-version:
@@ -49,11 +44,6 @@ on:
description: The version of Python to install
required: false
default: '3.13'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
defaults:
run:
shell: bash
@@ -106,10 +96,6 @@ jobs:
uses: actions/setup-python@v6
with:
python-version: ${{ inputs.python-version || '3.13' }}
- name: Install .NET
uses: actions/setup-dotnet@v5
with:
dotnet-version: ${{ inputs.dotnet-version || '9.x' }}
- uses: ./../action/init
with:
config-file: .github/codeql/codeql-config-packaging3.yml

View File

@@ -27,11 +27,6 @@ on:
description: The version of Go to install
required: false
default: '>=1.21.0'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
workflow_call:
inputs:
go-version:
@@ -39,11 +34,6 @@ on:
description: The version of Go to install
required: false
default: '>=1.21.0'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
defaults:
run:
shell: bash
@@ -91,10 +81,6 @@ jobs:
with:
go-version: ${{ inputs.go-version || '>=1.21.0' }}
cache: false
- name: Install .NET
uses: actions/setup-dotnet@v5
with:
dotnet-version: ${{ inputs.dotnet-version || '9.x' }}
- uses: ./../action/init
with:
config-file: .github/codeql/codeql-config-packaging3.yml

View File

@@ -27,11 +27,6 @@ on:
description: The version of Go to install
required: false
default: '>=1.21.0'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
workflow_call:
inputs:
go-version:
@@ -39,11 +34,6 @@ on:
description: The version of Go to install
required: false
default: '>=1.21.0'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
defaults:
run:
shell: bash
@@ -91,10 +81,6 @@ jobs:
with:
go-version: ${{ inputs.go-version || '>=1.21.0' }}
cache: false
- name: Install .NET
uses: actions/setup-dotnet@v5
with:
dotnet-version: ${{ inputs.dotnet-version || '9.x' }}
- uses: ./../action/init
with:
config-file: .github/codeql/codeql-config-packaging.yml

View File

@@ -27,11 +27,6 @@ on:
description: The version of Go to install
required: false
default: '>=1.21.0'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
workflow_call:
inputs:
go-version:
@@ -39,11 +34,6 @@ on:
description: The version of Go to install
required: false
default: '>=1.21.0'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
defaults:
run:
shell: bash
@@ -91,10 +81,6 @@ jobs:
with:
go-version: ${{ inputs.go-version || '>=1.21.0' }}
cache: false
- name: Install .NET
uses: actions/setup-dotnet@v5
with:
dotnet-version: ${{ inputs.dotnet-version || '9.x' }}
- uses: ./../action/init
with:
config-file: .github/codeql/codeql-config-packaging2.yml

View File

@@ -32,11 +32,6 @@ on:
description: The version of Python to install
required: false
default: '3.13'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
workflow_call:
inputs:
go-version:
@@ -49,11 +44,6 @@ on:
description: The version of Python to install
required: false
default: '3.13'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
defaults:
run:
shell: bash
@@ -97,10 +87,6 @@ jobs:
uses: actions/setup-python@v6
with:
python-version: ${{ inputs.python-version || '3.13' }}
- name: Install .NET
uses: actions/setup-dotnet@v5
with:
dotnet-version: ${{ inputs.dotnet-version || '9.x' }}
- uses: ./../action/init
with:
tools: ${{ steps.prepare-test.outputs.tools-url }}

View File

@@ -56,7 +56,7 @@ jobs:
use-all-platform-bundle: 'false'
setup-kotlin: 'true'
- name: Set up Ruby
uses: ruby/setup-ruby@8aeb6ff8030dd539317f8e1769a044873b56ea71 # v1.268.0
uses: ruby/setup-ruby@d5126b9b3579e429dd52e51e68624dda2e05be25 # v1.267.0
with:
ruby-version: 2.6
- name: Install Code Scanning integration

View File

@@ -27,11 +27,6 @@ on:
description: The version of Go to install
required: false
default: '>=1.21.0'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
workflow_call:
inputs:
go-version:
@@ -39,11 +34,6 @@ on:
description: The version of Go to install
required: false
default: '>=1.21.0'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
defaults:
run:
shell: bash
@@ -90,10 +80,6 @@ jobs:
with:
go-version: ${{ inputs.go-version || '>=1.21.0' }}
cache: false
- name: Install .NET
uses: actions/setup-dotnet@v5
with:
dotnet-version: ${{ inputs.dotnet-version || '9.x' }}
- uses: ./../action/init
with:
config-file: .github/codeql/codeql-config-packaging3.yml

View File

@@ -27,11 +27,6 @@ on:
description: The version of Go to install
required: false
default: '>=1.21.0'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
workflow_call:
inputs:
go-version:
@@ -39,11 +34,6 @@ on:
description: The version of Go to install
required: false
default: '>=1.21.0'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
defaults:
run:
shell: bash
@@ -84,10 +74,6 @@ jobs:
with:
go-version: ${{ inputs.go-version || '>=1.21.0' }}
cache: false
- name: Install .NET
uses: actions/setup-dotnet@v5
with:
dotnet-version: ${{ inputs.dotnet-version || '9.x' }}
- name: Use Xcode 16
if: runner.os == 'macOS' && matrix.version != 'nightly-latest'
run: sudo xcode-select -s "/Applications/Xcode_16.app"

View File

@@ -32,11 +32,6 @@ on:
description: The version of Python to install
required: false
default: '3.13'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
workflow_call:
inputs:
go-version:
@@ -49,11 +44,6 @@ on:
description: The version of Python to install
required: false
default: '3.13'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
defaults:
run:
shell: bash
@@ -97,10 +87,6 @@ jobs:
uses: actions/setup-python@v6
with:
python-version: ${{ inputs.python-version || '3.13' }}
- name: Install .NET
uses: actions/setup-dotnet@v5
with:
dotnet-version: ${{ inputs.dotnet-version || '9.x' }}
- uses: ./../action/init
id: init
with:

View File

@@ -32,11 +32,6 @@ on:
description: The version of Python to install
required: false
default: '3.13'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
workflow_call:
inputs:
go-version:
@@ -49,11 +44,6 @@ on:
description: The version of Python to install
required: false
default: '3.13'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
defaults:
run:
shell: bash
@@ -95,10 +85,6 @@ jobs:
uses: actions/setup-python@v6
with:
python-version: ${{ inputs.python-version || '3.13' }}
- name: Install .NET
uses: actions/setup-dotnet@v5
with:
dotnet-version: ${{ inputs.dotnet-version || '9.x' }}
- uses: ./../action/init
with:
tools: ${{ steps.prepare-test.outputs.tools-url }}

14
.github/workflows/__upload-sarif.yml generated vendored
View File

@@ -32,11 +32,6 @@ on:
description: The version of Python to install
required: false
default: '3.13'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
workflow_call:
inputs:
go-version:
@@ -49,11 +44,6 @@ on:
description: The version of Python to install
required: false
default: '3.13'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
defaults:
run:
shell: bash
@@ -102,10 +92,6 @@ jobs:
uses: actions/setup-python@v6
with:
python-version: ${{ inputs.python-version || '3.13' }}
- name: Install .NET
uses: actions/setup-dotnet@v5
with:
dotnet-version: ${{ inputs.dotnet-version || '9.x' }}
- uses: ./../action/init
with:
tools: ${{ steps.prepare-test.outputs.tools-url }}

View File

@@ -32,11 +32,6 @@ on:
description: The version of Python to install
required: false
default: '3.13'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
workflow_call:
inputs:
go-version:
@@ -49,11 +44,6 @@ on:
description: The version of Python to install
required: false
default: '3.13'
dotnet-version:
type: string
description: The version of .NET to install
required: false
default: 9.x
defaults:
run:
shell: bash
@@ -95,10 +85,6 @@ jobs:
uses: actions/setup-python@v6
with:
python-version: ${{ inputs.python-version || '3.13' }}
- name: Install .NET
uses: actions/setup-dotnet@v5
with:
dotnet-version: ${{ inputs.dotnet-version || '9.x' }}
- name: Delete original checkout
run: |
# delete the original checkout so we don't accidentally use it.

View File

@@ -81,7 +81,7 @@ jobs:
strategy:
fail-fast: false
matrix:
os: [ubuntu-22.04,ubuntu-24.04,windows-2022,windows-2025,macos-14,macos-15]
os: [ubuntu-22.04,ubuntu-24.04,windows-2022,windows-2025,macos-13,macos-14,macos-15]
tools: ${{ fromJson(needs.check-codeql-versions.outputs.versions) }}
runs-on: ${{ matrix.os }}

View File

@@ -54,10 +54,6 @@ jobs:
- uses: actions/setup-go@v6
with:
go-version: ^1.13.1
- name: Install .NET
uses: actions/setup-dotnet@v5
with:
dotnet-version: '9.x'
- uses: ./../action/init
with:
tools: ${{ steps.prepare-test.outputs.tools-url }}

View File

@@ -50,10 +50,6 @@ jobs:
- uses: actions/setup-go@v6
with:
go-version: ^1.13.1
- name: Install .NET
uses: actions/setup-dotnet@v5
with:
dotnet-version: '9.x'
- uses: ./../action/init
id: init
with:

View File

@@ -1,18 +0,0 @@
import os
import re
# Get the PR number from the PR URL.
pr_number = os.environ['PR_URL'].split('/')[-1]
changelog_note = f"- Update default CodeQL bundle version to {os.environ['CLI_VERSION']}. [#{pr_number}]({os.environ['PR_URL']})"
# If the "[UNRELEASED]" section starts with "no user facing changes", remove that line.
with open('CHANGELOG.md', 'r') as f:
changelog = f.read()
changelog = changelog.replace('## [UNRELEASED]\n\nNo user facing changes.', '## [UNRELEASED]\n')
# Add the changelog note to the bottom of the "[UNRELEASED]" section.
changelog = re.sub(r'\n## (\d+\.\d+\.\d+)', f'{changelog_note}\n\n## \\1', changelog, count=1)
with open('CHANGELOG.md', 'w') as f:
f.write(changelog)

View File

@@ -29,7 +29,7 @@ fi
echo "Getting checks for $GITHUB_SHA"
# Ignore any checks with "https://", CodeQL, LGTM, Update, and ESLint checks.
CHECKS="$(gh api repos/github/codeql-action/commits/"${GITHUB_SHA}"/check-runs --paginate | jq --slurp --compact-output --raw-output '[.[].check_runs.[] | select(.conclusion != "skipped") | .name | select(contains("https://") or . == "CodeQL" or . == "Dependabot" or . == "check-expected-release-files" or contains("Update") or contains("ESLint") or contains("update") or contains("test-setup-python-scripts") or . == "Agent" or . == "Cleanup artifacts" or . == "Prepare" or . == "Upload results" | not)] | unique | sort')"
CHECKS="$(gh api repos/github/codeql-action/commits/"${GITHUB_SHA}"/check-runs --paginate | jq --slurp --compact-output --raw-output '[.[].check_runs.[] | select(.conclusion != "skipped") | .name | select(contains("https://") or . == "CodeQL" or . == "Dependabot" or . == "check-expected-release-files" or contains("Update") or contains("ESLint") or contains("update") or contains("test-setup-python-scripts") | not)] | unique | sort')"
echo "$CHECKS" | jq

View File

@@ -43,10 +43,6 @@ jobs:
with:
version: ${{ matrix.version }}
use-all-platform-bundle: true
- name: Install .NET
uses: actions/setup-dotnet@v5
with:
dotnet-version: '9.x'
- id: init
uses: ./../action/init
with:

View File

@@ -40,11 +40,6 @@ jobs:
git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
git config --global user.name "github-actions[bot]"
- name: Set up Python
uses: actions/setup-python@v6
with:
python-version: '3.12'
- name: Set up Node.js
uses: actions/setup-node@v6
with:
@@ -83,8 +78,28 @@ jobs:
echo "PR_URL=$pr_url" | tee -a "$GITHUB_ENV"
- name: Create changelog note
shell: python
run: |
python .github/workflows/script/bundle_changelog.py
import os
import re
# Get the PR number from the PR URL.
pr_number = os.environ['PR_URL'].split('/')[-1]
changelog_note = f"- Update default CodeQL bundle version to {os.environ['CLI_VERSION']}. [#{pr_number}]({os.environ['PR_URL']})"
# If the "[UNRELEASED]" section starts with "no user facing changes", remove that line.
# Use perl to avoid having to escape the newline character.
with open('CHANGELOG.md', 'r') as f:
changelog = f.read()
changelog = changelog.replace('## [UNRELEASED]\n\nNo user facing changes.', '## [UNRELEASED]\n')
# Add the changelog note to the bottom of the "[UNRELEASED]" section.
changelog = re.sub(r'\n## (\d+\.\d+\.\d+)', f'{changelog_note}\n\n## \\1', changelog, count=1)
with open('CHANGELOG.md', 'w') as f:
f.write(changelog)
- name: Push changelog note
run: |

View File

@@ -4,12 +4,6 @@ on:
schedule:
- cron: "0 0 * * *"
workflow_dispatch:
pull_request:
branches:
- main
paths:
- .github/workflows/update-supported-enterprise-server-versions.yml
- .github/workflows/update-supported-enterprise-server-versions/update.py
jobs:
update-supported-enterprise-server-versions:
@@ -34,7 +28,6 @@ jobs:
repository: github/enterprise-releases
token: ${{ secrets.ENTERPRISE_RELEASE_TOKEN }}
path: ${{ github.workspace }}/enterprise-releases/
sparse-checkout: releases.json
- name: Update Supported Enterprise Server Versions
run: |
cd ./.github/workflows/update-supported-enterprise-server-versions/
@@ -42,7 +35,6 @@ jobs:
pipenv install
pipenv run ./update.py
rm --recursive "$ENTERPRISE_RELEASES_PATH"
npm ci
npm run build
env:
ENTERPRISE_RELEASES_PATH: ${{ github.workspace }}/enterprise-releases/
@@ -52,33 +44,25 @@ jobs:
git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
git config --global user.name "github-actions[bot]"
- name: Commit changes
id: prepare-commit
- name: Commit changes and open PR
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
if [[ -z $(git status --porcelain) ]]; then
echo "No changes to commit"
echo "committed=false" >> $GITHUB_OUTPUT
else
git checkout -b update-supported-enterprise-server-versions
git add .
git commit --message "Update supported GitHub Enterprise Server versions"
git push origin update-supported-enterprise-server-versions
echo "committed=true" >> $GITHUB_OUTPUT
body="This PR updates the list of supported GitHub Enterprise Server versions, either because a new "
body+="version is about to be feature frozen, or because an old release has been deprecated."
body+=$'\n\n'
body+="If an old release has been deprecated, please follow the instructions in CONTRIBUTING.md to "
body+="deprecate the corresponding version of CodeQL."
gh pr create --draft \
--title "Update supported GitHub Enterprise Server versions" \
--body "$body"
fi
- name: Open PR
if: github.event_name != 'pull_request' && steps.prepare-commit.outputs.committed == 'true'
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
git push origin update-supported-enterprise-server-versions
body="This PR updates the list of supported GitHub Enterprise Server versions, either because a new "
body+="version is about to be feature frozen, or because an old release has been deprecated."
body+=$'\n\n'
body+="If an old release has been deprecated, please follow the instructions in CONTRIBUTING.md to "
body+="deprecate the corresponding version of CodeQL."
gh pr create --draft \
--title "Update supported GitHub Enterprise Server versions" \
--body "$body"

View File

@@ -6,15 +6,6 @@ See the [releases page](https://github.com/github/codeql-action/releases) for th
No user facing changes.
## 4.31.4 - 18 Nov 2025
No user facing changes.
## 4.31.3 - 13 Nov 2025
- CodeQL Action v3 will be deprecated in December 2026. The Action now logs a warning for customers who are running v3 but could be running v4. For more information, see [Upcoming deprecation of CodeQL Action v3](https://github.blog/changelog/2025-10-28-upcoming-deprecation-of-codeql-action-v3/).
- Update default CodeQL bundle version to 2.23.5. [#3288](https://github.com/github/codeql-action/pull/3288)
## 4.31.2 - 30 Oct 2025
No user facing changes.

View File

@@ -12,7 +12,6 @@ import filenames from "eslint-plugin-filenames";
import github from "eslint-plugin-github";
import _import from "eslint-plugin-import";
import noAsyncForeach from "eslint-plugin-no-async-foreach";
import jsdoc from "eslint-plugin-jsdoc";
import globals from "globals";
const __filename = fileURLToPath(import.meta.url);
@@ -53,7 +52,6 @@ export default [
github: fixupPluginRules(github),
import: fixupPluginRules(_import),
"no-async-foreach": noAsyncForeach,
"jsdoc": jsdoc,
},
languageOptions: {
@@ -135,16 +133,6 @@ export default [
"@typescript-eslint/no-shadow": "error",
"@typescript-eslint/prefer-optional-chain": "error",
"one-var": ["error", "never"],
// Check param names to ensure that we don't have outdated JSDocs.
"jsdoc/check-param-names": [
"error",
{
// We don't currently require full JSDoc coverage, so this rule
// should not error on missing @param annotations.
disableMissingParamChecks: true,
}
],
},
},
{

File diff suppressed because it is too large Load Diff

1053
lib/analyze-action.js generated

File diff suppressed because it is too large Load Diff

774
lib/autobuild-action.js generated

File diff suppressed because it is too large Load Diff

View File

@@ -1,6 +1,6 @@
{
"bundleVersion": "codeql-bundle-v2.23.5",
"cliVersion": "2.23.5",
"priorBundleVersion": "codeql-bundle-v2.23.3",
"priorCliVersion": "2.23.3"
"bundleVersion": "codeql-bundle-v2.23.3",
"cliVersion": "2.23.3",
"priorBundleVersion": "codeql-bundle-v2.23.2",
"priorCliVersion": "2.23.2"
}

1477
lib/init-action-post.js generated

File diff suppressed because it is too large Load Diff

1103
lib/init-action.js generated

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

1280
lib/start-proxy-action.js generated

File diff suppressed because it is too large Load Diff

762
lib/upload-lib.js generated

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

1594
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@@ -1,6 +1,6 @@
{
"name": "codeql",
"version": "4.31.5",
"version": "4.31.3",
"private": true,
"description": "CodeQL action",
"scripts": {
@@ -35,44 +35,45 @@
"@actions/io": "^2.0.0",
"@actions/tool-cache": "^2.0.2",
"@octokit/plugin-retry": "^6.0.0",
"@octokit/request-error": "^7.0.2",
"@schemastore/package": "0.0.10",
"archiver": "^7.0.1",
"fast-deep-equal": "^3.1.3",
"follow-redirects": "^1.15.11",
"get-folder-size": "^5.0.0",
"js-yaml": "^4.1.1",
"js-yaml": "^4.1.0",
"jsonschema": "1.4.1",
"long": "^5.3.2",
"node-forge": "^1.3.1",
"octokit": "^5.0.5",
"semver": "^7.7.3",
"uuid": "^13.0.0"
},
"devDependencies": {
"@ava/typescript": "6.0.0",
"@eslint/compat": "^2.0.0",
"@eslint/compat": "^1.4.1",
"@eslint/eslintrc": "^3.3.1",
"@eslint/js": "^9.39.1",
"@eslint/js": "^9.39.0",
"@microsoft/eslint-formatter-sarif": "^3.1.0",
"@octokit/types": "^16.0.0",
"@types/archiver": "^7.0.0",
"@types/follow-redirects": "^1.14.4",
"@types/js-yaml": "^4.0.9",
"@types/node": "^20.19.9",
"@types/node": "20.19.9",
"@types/node-forge": "^1.3.14",
"@types/semver": "^7.7.1",
"@types/sinon": "^21.0.0",
"@typescript-eslint/eslint-plugin": "^8.46.4",
"@types/sinon": "^17.0.4",
"@typescript-eslint/eslint-plugin": "^8.46.3",
"@typescript-eslint/parser": "^8.41.0",
"ava": "^6.4.1",
"esbuild": "^0.27.0",
"esbuild": "^0.25.12",
"eslint": "^8.57.1",
"eslint-import-resolver-typescript": "^3.8.7",
"eslint-plugin-filenames": "^1.3.2",
"eslint-plugin-github": "^5.1.8",
"eslint-plugin-import": "2.29.1",
"eslint-plugin-jsdoc": "^61.2.1",
"eslint-plugin-no-async-foreach": "^0.1.1",
"glob": "^11.1.0",
"glob": "^11.0.3",
"nock": "^14.0.10",
"sinon": "^21.0.0",
"typescript": "^5.9.3"
@@ -96,7 +97,6 @@
"eslint-plugin-jsx-a11y": {
"semver": ">=6.3.1"
},
"brace-expansion@2.0.1": "2.0.2",
"glob": "^11.1.0"
"brace-expansion@2.0.1": "2.0.2"
}
}

View File

@@ -4,7 +4,6 @@ operatingSystems: ["ubuntu", "macos", "windows"]
versions: ["nightly-latest"]
useAllPlatformBundle: "true"
installGo: true
installDotNet: true
steps:
- id: init
uses: ./../action/init

View File

@@ -3,7 +3,6 @@ description: "Checks that specifying 'ref' and 'sha' as inputs works"
versions: ["default"]
installGo: true
installPython: true
installDotNet: true
steps:
- uses: ./../action/init
with:

View File

@@ -2,7 +2,6 @@ name: "autobuild-action"
description: "Tests that the C# autobuild action works"
operatingSystems: ["ubuntu", "macos", "windows"]
versions: ["linked"]
installDotNet: true
steps:
- uses: ./../action/init
with:

View File

@@ -2,7 +2,6 @@ name: "Build mode manual"
description: "An end-to-end integration test of a Java repository built using 'build-mode: manual'"
versions: ["nightly-latest"]
installGo: true
installDotNet: true
steps:
- uses: ./../action/init
id: init

View File

@@ -3,7 +3,6 @@ description: "Tests that file baseline information is exported when the feature
operatingSystems: ["ubuntu", "macos", "windows"]
versions: ["nightly-latest"]
installGo: true
installDotNet: true
env:
CODEQL_ACTION_SUBLANGUAGE_FILE_COVERAGE: true
steps:

View File

@@ -7,7 +7,6 @@ versions:
- linked
- nightly-latest
installGo: true
installDotNet: true
env:
DOTNET_GENERATE_ASPNET_CERTIFICATE: "false"
steps:

View File

@@ -3,7 +3,6 @@ description: "Tests using a CodeQL bundle from a local file rather than a URL"
versions: ["linked"]
installGo: true
installPython: true
installDotNet: true
steps:
- name: Fetch latest CodeQL bundle
run: |

View File

@@ -5,7 +5,6 @@ env:
CODEQL_ACTION_RESOLVE_SUPPORTED_LANGUAGES_USING_CLI: true
installGo: true
installPython: true
installDotNet: true
steps:
- name: Use Xcode 16
if: runner.os == 'macOS' && matrix.version != 'nightly-latest'

View File

@@ -4,7 +4,6 @@ versions: ["linked", "default", "nightly-latest"] # This feature is not compatib
installGo: true
installNode: true
installPython: true
installDotNet: true
steps:
- uses: ./../action/init
with:

View File

@@ -3,7 +3,6 @@ description: "Checks that specifying packages using a combination of a config fi
versions: ["linked", "default", "nightly-latest"] # This feature is not compatible with old CLIs
installGo: true
installNode: true
installDotNet: true
steps:
- uses: ./../action/init
with:

View File

@@ -3,7 +3,6 @@ description: "Checks that specifying packages using only a config file works"
versions: ["linked", "default", "nightly-latest"] # This feature is not compatible with old CLIs
installGo: true
installNode: true
installDotNet: true
steps:
- uses: ./../action/init
with:

View File

@@ -3,7 +3,6 @@ description: "Checks that specifying packages using the input to the Action work
versions: ["linked", "default", "nightly-latest"] # This feature is not compatible with old CLIs
installGo: true
installNode: true
installDotNet: true
steps:
- uses: ./../action/init
with:

View File

@@ -7,7 +7,6 @@ versions:
- nightly-latest
installGo: true
installPython: true
installDotNet: true
steps:
- uses: ./../action/init
with:

View File

@@ -4,7 +4,7 @@ description: "Tests using RuboCop to analyze a multi-language repository and the
versions: ["default"]
steps:
- name: Set up Ruby
uses: ruby/setup-ruby@8aeb6ff8030dd539317f8e1769a044873b56ea71 # v1.268.0
uses: ruby/setup-ruby@d5126b9b3579e429dd52e51e68624dda2e05be25 # v1.267.0
with:
ruby-version: 2.6
- name: Install Code Scanning integration

View File

@@ -3,7 +3,6 @@ description: "Tests a split-up workflow in which we first build a database and l
operatingSystems: ["ubuntu", "macos"]
versions: ["linked", "default", "nightly-latest"] # This feature is not compatible with old CLIs
installGo: true
installDotNet: true
steps:
- uses: ./../action/init
with:

View File

@@ -3,7 +3,6 @@ description: "Tests creation of a Swift database using custom build"
versions: ["linked", "default", "nightly-latest"]
operatingSystems: ["macos"]
installGo: true
installDotNet: true
env:
DOTNET_GENERATE_ASPNET_CERTIFICATE: "false"
steps:

View File

@@ -7,7 +7,6 @@ versions:
- nightly-latest
installGo: true
installPython: true
installDotNet: true
steps:
- uses: ./../action/init
id: init

View File

@@ -3,7 +3,6 @@ description: "Checks that specifying 'ref' and 'sha' as inputs works"
versions: ["default"]
installGo: true
installPython: true
installDotNet: true
steps:
- uses: ./../action/init
with:

View File

@@ -4,7 +4,6 @@ versions: ["default"]
analysisKinds: ["code-scanning", "code-quality", "code-scanning,code-quality"]
installGo: true
installPython: true
installDotNet: true
steps:
- uses: ./../action/init
with:

View File

@@ -3,7 +3,6 @@ description: "Checks that a custom `checkout_path` will find the proper commit_o
versions: ["linked"]
installGo: true
installPython: true
installDotNet: true
steps:
# This ensures we don't accidentally use the original checkout for any part of the test.
- name: Delete original checkout

View File

@@ -204,25 +204,6 @@ for file in sorted((this_dir / 'checks').glob('*.yml')):
}
})
installDotNet = is_truthy(checkSpecification.get('installDotNet', ''))
if installDotNet:
baseDotNetVersionExpr = '9.x'
workflowInputs['dotnet-version'] = {
'type': 'string',
'description': 'The version of .NET to install',
'required': False,
'default': baseDotNetVersionExpr,
}
steps.append({
'name': 'Install .NET',
'uses': 'actions/setup-dotnet@v5',
'with': {
'dotnet-version': '${{ inputs.dotnet-version || \'' + baseDotNetVersionExpr + '\' }}'
}
})
# If container initialisation steps are present in the check specification,
# make sure to execute them first.
if 'container' in checkSpecification and 'container-init-steps' in checkSpecification:

View File

@@ -80,7 +80,7 @@ export function isRunningLocalAction(): boolean {
*
* This can be used to get the Action's name or tell if we're running a local Action.
*/
function getRelativeScriptPath(): string {
export function getRelativeScriptPath(): string {
const runnerTemp = getRequiredEnvParam("RUNNER_TEMP");
const actionsDirectory = path.join(path.dirname(runnerTemp), "_actions");
return path.relative(actionsDirectory, __filename);

View File

@@ -98,7 +98,7 @@ export async function getAnalysisKinds(
export const codeQualityQueries: string[] = ["code-quality"];
// Enumerates API endpoints that accept SARIF files.
enum SARIF_UPLOAD_ENDPOINT {
export enum SARIF_UPLOAD_ENDPOINT {
CODE_SCANNING = "PUT /repos/:owner/:repo/code-scanning/analysis",
CODE_QUALITY = "PUT /repos/:owner/:repo/code-quality/analysis",
}

View File

@@ -25,7 +25,7 @@ import {
isCodeQualityEnabled,
isCodeScanningEnabled,
} from "./config-utils";
import { cleanupAndUploadDatabases } from "./database-upload";
import { uploadDatabases } from "./database-upload";
import {
DependencyCacheUploadStatusReport,
uploadDependencyCaches,
@@ -35,7 +35,7 @@ import { EnvVar } from "./environment";
import { Feature, Features } from "./feature-flags";
import { KnownLanguage } from "./languages";
import { getActionsLogger, Logger } from "./logging";
import { cleanupAndUploadOverlayBaseDatabaseToCache } from "./overlay-database-utils";
import { uploadOverlayBaseDatabaseToCache } from "./overlay-database-utils";
import { getRepositoryNwo } from "./repository";
import * as statusReport from "./status-report";
import {
@@ -417,21 +417,12 @@ async function run() {
}
// Possibly upload the overlay-base database to actions cache.
// Note: Take care with the ordering of this call since databases may be cleaned up
// at the `overlay` level.
await cleanupAndUploadOverlayBaseDatabaseToCache(codeql, config, logger);
// If databases are to be uploaded, they will first be cleaned up at the overlay level.
await uploadOverlayBaseDatabaseToCache(codeql, config, logger);
// Possibly upload the database bundles for remote queries.
// Note: Take care with the ordering of this call since databases may be cleaned up
// at the `overlay` or `clear` level.
await cleanupAndUploadDatabases(
repositoryNwo,
codeql,
config,
apiDetails,
features,
logger,
);
// If databases are to be uploaded, they will first be cleaned up at the clear level.
await uploadDatabases(repositoryNwo, codeql, config, apiDetails, logger);
// Possibly upload the TRAP caches for later re-use
const trapCacheUploadStartTime = performance.now();
@@ -447,11 +438,14 @@ async function run() {
// Store dependency cache(s) if dependency caching is enabled.
if (shouldStoreCache(config.dependencyCachingEnabled)) {
dependencyCacheResults = await uploadDependencyCaches(
const minimizeJavaJars = await features.getValue(
Feature.JavaMinimizeDependencyJars,
codeql,
features,
);
dependencyCacheResults = await uploadDependencyCaches(
config,
logger,
minimizeJavaJars,
);
}

View File

@@ -38,26 +38,89 @@ export class CodeQLAnalysisError extends Error {
}
}
type KnownLanguageKey = keyof typeof KnownLanguage;
type RunQueriesDurationStatusReport = {
export interface QueriesStatusReport {
/**
* Time taken in ms to run queries for the language (or undefined if this language was not analyzed).
* Time taken in ms to run queries for actions (or undefined if this language was not analyzed).
*
* The "builtin" designation is now outdated with the move to CLI config parsing: this is the time
* taken to run _all_ the queries.
*/
[L in KnownLanguageKey as `analyze_builtin_queries_${L}_duration_ms`]?: number;
};
analyze_builtin_queries_actions_duration_ms?: number;
/**
* Time taken in ms to run queries for cpp (or undefined if this language was not analyzed).
*
* The "builtin" designation is now outdated with the move to CLI config parsing: this is the time
* taken to run _all_ the queries.
*/
analyze_builtin_queries_cpp_duration_ms?: number;
/**
* Time taken in ms to run queries for csharp (or undefined if this language was not analyzed).
*
* The "builtin" designation is now outdated with the move to CLI config parsing: this is the time
* taken to run _all_ the queries.
*/
analyze_builtin_queries_csharp_duration_ms?: number;
/**
* Time taken in ms to run queries for go (or undefined if this language was not analyzed).
*
* The "builtin" designation is now outdated with the move to CLI config parsing: this is the time
* taken to run _all_ the queries.
*/
analyze_builtin_queries_go_duration_ms?: number;
/**
* Time taken in ms to run queries for java (or undefined if this language was not analyzed).
*
* The "builtin" designation is now outdated with the move to CLI config parsing: this is the time
* taken to run _all_ the queries.
*/
analyze_builtin_queries_java_duration_ms?: number;
/**
* Time taken in ms to run queries for javascript (or undefined if this language was not analyzed).
*
* The "builtin" designation is now outdated with the move to CLI config parsing: this is the time
* taken to run _all_ the queries.
*/
analyze_builtin_queries_javascript_duration_ms?: number;
/**
* Time taken in ms to run queries for python (or undefined if this language was not analyzed).
*
* The "builtin" designation is now outdated with the move to CLI config parsing: this is the time
* taken to run _all_ the queries.
*/
analyze_builtin_queries_python_duration_ms?: number;
/**
* Time taken in ms to run queries for ruby (or undefined if this language was not analyzed).
*
* The "builtin" designation is now outdated with the move to CLI config parsing: this is the time
* taken to run _all_ the queries.
*/
analyze_builtin_queries_ruby_duration_ms?: number;
/** Time taken in ms to run queries for swift (or undefined if this language was not analyzed).
*
* The "builtin" designation is now outdated with the move to CLI config parsing: this is the time
* taken to run _all_ the queries.
*/
analyze_builtin_queries_swift_duration_ms?: number;
type InterpretResultsDurationStatusReport = {
/** Time taken in ms to interpret results for the language (or undefined if this language was not analyzed). */
[L in KnownLanguageKey as `interpret_results_${L}_duration_ms`]?: number;
};
/** Time taken in ms to interpret results for actions (or undefined if this language was not analyzed). */
interpret_results_actions_duration_ms?: number;
/** Time taken in ms to interpret results for cpp (or undefined if this language was not analyzed). */
interpret_results_cpp_duration_ms?: number;
/** Time taken in ms to interpret results for csharp (or undefined if this language was not analyzed). */
interpret_results_csharp_duration_ms?: number;
/** Time taken in ms to interpret results for go (or undefined if this language was not analyzed). */
interpret_results_go_duration_ms?: number;
/** Time taken in ms to interpret results for java (or undefined if this language was not analyzed). */
interpret_results_java_duration_ms?: number;
/** Time taken in ms to interpret results for javascript (or undefined if this language was not analyzed). */
interpret_results_javascript_duration_ms?: number;
/** Time taken in ms to interpret results for python (or undefined if this language was not analyzed). */
interpret_results_python_duration_ms?: number;
/** Time taken in ms to interpret results for ruby (or undefined if this language was not analyzed). */
interpret_results_ruby_duration_ms?: number;
/** Time taken in ms to interpret results for swift (or undefined if this language was not analyzed). */
interpret_results_swift_duration_ms?: number;
export interface QueriesStatusReport
extends RunQueriesDurationStatusReport,
InterpretResultsDurationStatusReport {
/**
* Whether the analysis is diff-informed (in the sense that the action generates a diff-range data
* extension for the analysis, regardless of whether the data extension is actually used by queries).

View File

@@ -18,6 +18,11 @@ import {
const GITHUB_ENTERPRISE_VERSION_HEADER = "x-github-enterprise-version";
export enum DisallowedAPIVersionReason {
ACTION_TOO_OLD,
ACTION_TOO_NEW,
}
export type GitHubApiCombinedDetails = GitHubApiDetails &
GitHubApiExternalRepoDetails;

View File

@@ -1,5 +1,3 @@
import * as crypto from "crypto";
import * as core from "@actions/core";
import { getOptionalInput, isDefaultSetup } from "./actions-util";
@@ -73,33 +71,6 @@ export function getCachingKind(input: string | undefined): CachingKind {
}
}
// The length to which `createCacheKeyHash` truncates hash strings.
export const cacheKeyHashLength = 16;
/**
* Creates a SHA-256 hash of the cache key components to ensure uniqueness
* while keeping the cache key length manageable.
*
* @param components Object containing all components that should influence cache key uniqueness
* @returns A short SHA-256 hash (first 16 characters) of the components
*/
export function createCacheKeyHash(components: Record<string, any>): string {
// From https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/JSON/stringify
//
// "Properties are visited using the same algorithm as Object.keys(), which
// has a well-defined order and is stable across implementations. For example,
// JSON.stringify on the same object will always produce the same string, and
// JSON.parse(JSON.stringify(obj)) would produce an object with the same key
// ordering as the original (assuming the object is completely
// JSON-serializable)."
const componentsJson = JSON.stringify(components);
return crypto
.createHash("sha256")
.update(componentsJson)
.digest("hex")
.substring(0, cacheKeyHashLength);
}
/** Determines whether dependency caching is enabled. */
export function getDependencyCachingEnabled(): CachingKind {
// If the workflow specified something always respect that

View File

@@ -159,7 +159,10 @@ type CliErrorConfiguration = {
* All of our caught CLI error messages that we handle specially: ie. if we
* would like to categorize an error as a configuration error or not.
*/
const cliErrorsConfig: Record<CliConfigErrorCategory, CliErrorConfiguration> = {
export const cliErrorsConfig: Record<
CliConfigErrorCategory,
CliErrorConfiguration
> = {
[CliConfigErrorCategory.AutobuildError]: {
cliErrorMessageCandidates: [
new RegExp("We were unable to automatically build your code"),

View File

@@ -35,7 +35,7 @@ import { ToolsDownloadStatusReport } from "./tools-download";
import { ToolsFeature, isSupportedToolsFeature } from "./tools-features";
import { shouldEnableIndirectTracing } from "./tracer-config";
import * as util from "./util";
import { BuildMode, CleanupLevel, getErrorMessage } from "./util";
import { BuildMode, getErrorMessage } from "./util";
type Options = Array<string | number | boolean>;
@@ -141,10 +141,7 @@ export interface CodeQL {
/**
* Clean up all the databases within a database cluster.
*/
databaseCleanupCluster(
config: Config,
cleanupLevel: CleanupLevel,
): Promise<void>;
databaseCleanupCluster(config: Config, cleanupLevel: string): Promise<void>;
/**
* Run 'codeql database bundle'.
*/
@@ -516,7 +513,7 @@ export async function getCodeQLForTesting(
* version requirement. Must be set to true outside tests.
* @returns A new CodeQL object
*/
async function getCodeQLForCmd(
export async function getCodeQLForCmd(
cmd: string,
checkVersion: boolean,
): Promise<CodeQL> {
@@ -881,7 +878,7 @@ async function getCodeQLForCmd(
},
async databaseCleanupCluster(
config: Config,
cleanupLevel: CleanupLevel,
cleanupLevel: string,
): Promise<void> {
const cacheCleanupFlag = (await util.codeQlVersionAtLeast(
this,
@@ -1074,11 +1071,8 @@ async function getCodeQLForCmd(
/**
* Gets the options for `path` of `options` as an array of extra option strings.
*
* @param paths The CLI command components to get extra options for.
* @param args Additional arguments for this function.
* @param args.ignoringOptions
* Options that should be ignored, for example because they have already
* been passed and it is an error to pass them more than once.
* @param ignoringOptions Options that should be ignored, for example because they have already
* been passed and it is an error to pass them more than once.
*/
function getExtraOptionsFromEnv(
paths: string[],
@@ -1160,9 +1154,8 @@ async function runCli(
/**
* Writes the code scanning configuration that is to be used by the CLI.
*
* @param config The CodeQL Action state to write.
* @param logger The logger to use.
*
* @param codeql The CodeQL object to use.
* @param config The CodeQL Action state to use.
* @returns The path to the generated user configuration file.
*/
async function writeCodeScanningConfigFile(
@@ -1225,7 +1218,7 @@ export async function getTrapCachingExtractorConfigArgsForLang(
*
* This will not exist if the configuration is being parsed in the Action.
*/
function getGeneratedCodeScanningConfigPath(config: Config): string {
export function getGeneratedCodeScanningConfigPath(config: Config): string {
return path.resolve(config.tempDir, "user-config.yaml");
}

View File

@@ -37,9 +37,7 @@ import {
ConfigurationError,
withTmpDir,
BuildMode,
DiskUsage,
} from "./util";
import * as util from "./util";
setupTests(test);
@@ -202,14 +200,16 @@ test("load code quality config", async (t) => {
);
// And the config we expect it to result in
const expectedConfig = createTestConfig({
const expectedConfig: configUtils.Config = {
version: actionsUtil.getActionVersion(),
analysisKinds: [AnalysisKind.CodeQuality],
languages: [KnownLanguage.actions],
buildMode: undefined,
originalUserInput: {},
// This gets set because we only have `AnalysisKind.CodeQuality`
computedConfig: {
"disable-default-queries": true,
queries: [{ uses: "code-quality" }],
"query-filters": [],
},
tempDir,
codeQLCmd: codeql.getPath(),
@@ -218,7 +218,14 @@ test("load code quality config", async (t) => {
debugMode: false,
debugArtifactName: "",
debugDatabaseName: "",
});
trapCaches: {},
trapCacheDownloadTime: 0,
dependencyCachingEnabled: CachingKind.None,
extraQueryExclusions: [],
overlayDatabaseMode: OverlayDatabaseMode.None,
useOverlayDatabaseCaching: false,
repositoryProperties: {},
};
t.deepEqual(config, expectedConfig);
});
@@ -248,7 +255,6 @@ test("initActionState doesn't throw if there are queries configured in the repos
const computedConfig: configUtils.UserConfig = {
"disable-default-queries": true,
queries: [{ uses: "code-quality" }],
"query-filters": [],
};
const expectedConfig = createTestConfig({
@@ -499,7 +505,9 @@ test("load non-empty input", async (t) => {
};
// And the config we expect it to parse to
const expectedConfig = createTestConfig({
const expectedConfig: configUtils.Config = {
version: actionsUtil.getActionVersion(),
analysisKinds: [AnalysisKind.CodeScanning],
languages: [KnownLanguage.javascript],
buildMode: BuildMode.None,
originalUserInput: userConfig,
@@ -511,7 +519,14 @@ test("load non-empty input", async (t) => {
debugMode: false,
debugArtifactName: "my-artifact",
debugDatabaseName: "my-db",
});
trapCaches: {},
trapCacheDownloadTime: 0,
dependencyCachingEnabled: CachingKind.None,
extraQueryExclusions: [],
overlayDatabaseMode: OverlayDatabaseMode.None,
useOverlayDatabaseCaching: false,
repositoryProperties: {},
};
const languagesInput = "javascript";
const configFilePath = createConfigFile(inputFileContents, tempDir);
@@ -856,62 +871,71 @@ const mockRepositoryNwo = parseRepositoryNwo("owner/repo");
expectedLanguages: ["javascript"],
},
].forEach((args) => {
test(`getLanguages: ${args.name}`, async (t) => {
const mockRequest = mockLanguagesInRepo(args.languagesInRepository);
const stubExtractorEntry = {
extractor_root: "",
};
const codeQL = createStubCodeQL({
betterResolveLanguages: (options) =>
Promise.resolve({
aliases: {
"c#": KnownLanguage.csharp,
c: KnownLanguage.cpp,
kotlin: KnownLanguage.java,
typescript: KnownLanguage.javascript,
},
extractors: {
cpp: [stubExtractorEntry],
csharp: [stubExtractorEntry],
java: [stubExtractorEntry],
javascript: [stubExtractorEntry],
python: [stubExtractorEntry],
...(options?.filterToLanguagesWithQueries
? {}
: {
html: [stubExtractorEntry],
}),
},
}),
for (const resolveSupportedLanguagesUsingCli of [true, false]) {
test(`getLanguages${resolveSupportedLanguagesUsingCli ? " (supported languages via CLI)" : ""}: ${args.name}`, async (t) => {
const features = createFeatures(
resolveSupportedLanguagesUsingCli
? [Feature.ResolveSupportedLanguagesUsingCli]
: [],
);
const mockRequest = mockLanguagesInRepo(args.languagesInRepository);
const stubExtractorEntry = {
extractor_root: "",
};
const codeQL = createStubCodeQL({
betterResolveLanguages: (options) =>
Promise.resolve({
aliases: {
"c#": KnownLanguage.csharp,
c: KnownLanguage.cpp,
kotlin: KnownLanguage.java,
typescript: KnownLanguage.javascript,
},
extractors: {
cpp: [stubExtractorEntry],
csharp: [stubExtractorEntry],
java: [stubExtractorEntry],
javascript: [stubExtractorEntry],
python: [stubExtractorEntry],
...(options?.filterToLanguagesWithQueries
? {}
: {
html: [stubExtractorEntry],
}),
},
}),
});
if (args.expectedLanguages) {
// happy path
const actualLanguages = await configUtils.getLanguages(
codeQL,
args.languagesInput,
mockRepositoryNwo,
".",
features,
mockLogger,
);
t.deepEqual(actualLanguages.sort(), args.expectedLanguages.sort());
} else {
// there is an error
await t.throwsAsync(
async () =>
await configUtils.getLanguages(
codeQL,
args.languagesInput,
mockRepositoryNwo,
".",
features,
mockLogger,
),
{ message: args.expectedError },
);
}
t.deepEqual(mockRequest.called, args.expectedApiCall);
});
if (args.expectedLanguages) {
// happy path
const actualLanguages = await configUtils.getLanguages(
codeQL,
args.languagesInput,
mockRepositoryNwo,
".",
mockLogger,
);
t.deepEqual(actualLanguages.sort(), args.expectedLanguages.sort());
} else {
// there is an error
await t.throwsAsync(
async () =>
await configUtils.getLanguages(
codeQL,
args.languagesInput,
mockRepositoryNwo,
".",
mockLogger,
),
{ message: args.expectedError },
);
}
t.deepEqual(mockRequest.called, args.expectedApiCall);
});
}
});
for (const { displayName, language, feature } of [
@@ -973,12 +997,12 @@ interface OverlayDatabaseModeTestSetup {
features: Feature[];
isPullRequest: boolean;
isDefaultBranch: boolean;
repositoryOwner: string;
buildMode: BuildMode | undefined;
languages: Language[];
codeqlVersion: string;
gitRoot: string | undefined;
codeScanningConfig: configUtils.UserConfig;
diskUsage: DiskUsage | undefined;
}
const defaultOverlayDatabaseModeTestSetup: OverlayDatabaseModeTestSetup = {
@@ -986,15 +1010,12 @@ const defaultOverlayDatabaseModeTestSetup: OverlayDatabaseModeTestSetup = {
features: [],
isPullRequest: false,
isDefaultBranch: false,
repositoryOwner: "github",
buildMode: BuildMode.None,
languages: [KnownLanguage.javascript],
codeqlVersion: CODEQL_OVERLAY_MINIMUM_VERSION,
gitRoot: "/some/git/root",
codeScanningConfig: {},
diskUsage: {
numAvailableBytes: 50_000_000_000,
numTotalBytes: 100_000_000_000,
},
};
const getOverlayDatabaseModeMacro = test.macro({
@@ -1027,8 +1048,6 @@ const getOverlayDatabaseModeMacro = test.macro({
setup.overlayDatabaseEnvVar;
}
sinon.stub(util, "checkDiskUsage").resolves(setup.diskUsage);
// Mock feature flags
const features = createFeatures(setup.features);
@@ -1037,6 +1056,12 @@ const getOverlayDatabaseModeMacro = test.macro({
.stub(actionsUtil, "isAnalyzingPullRequest")
.returns(setup.isPullRequest);
// Mock repository owner
const repository = {
owner: setup.repositoryOwner,
repo: "test-repo",
};
// Set up CodeQL mock
const codeql = mockCodeQLVersion(setup.codeqlVersion);
@@ -1059,6 +1084,7 @@ const getOverlayDatabaseModeMacro = test.macro({
const result = await configUtils.getOverlayDatabaseMode(
codeql,
repository,
features,
setup.languages,
tempDir, // sourceRoot
@@ -1186,45 +1212,6 @@ test(
},
);
test(
getOverlayDatabaseModeMacro,
"No overlay-base database on default branch if runner disk space is too low",
{
languages: [KnownLanguage.javascript],
features: [
Feature.OverlayAnalysis,
Feature.OverlayAnalysisCodeScanningJavascript,
],
isDefaultBranch: true,
diskUsage: {
numAvailableBytes: 1_000_000_000,
numTotalBytes: 100_000_000_000,
},
},
{
overlayDatabaseMode: OverlayDatabaseMode.None,
useOverlayDatabaseCaching: false,
},
);
test(
getOverlayDatabaseModeMacro,
"No overlay-base database on default branch if we can't determine runner disk space",
{
languages: [KnownLanguage.javascript],
features: [
Feature.OverlayAnalysis,
Feature.OverlayAnalysisCodeScanningJavascript,
],
isDefaultBranch: true,
diskUsage: undefined,
},
{
overlayDatabaseMode: OverlayDatabaseMode.None,
useOverlayDatabaseCaching: false,
},
);
test(
getOverlayDatabaseModeMacro,
"No overlay-base database on default branch when code-scanning feature enabled with disable-default-queries",
@@ -1395,45 +1382,6 @@ test(
},
);
test(
getOverlayDatabaseModeMacro,
"No overlay analysis on PR if runner disk space is too low",
{
languages: [KnownLanguage.javascript],
features: [
Feature.OverlayAnalysis,
Feature.OverlayAnalysisCodeScanningJavascript,
],
isPullRequest: true,
diskUsage: {
numAvailableBytes: 1_000_000_000,
numTotalBytes: 100_000_000_000,
},
},
{
overlayDatabaseMode: OverlayDatabaseMode.None,
useOverlayDatabaseCaching: false,
},
);
test(
getOverlayDatabaseModeMacro,
"No overlay analysis on PR if we can't determine runner disk space",
{
languages: [KnownLanguage.javascript],
features: [
Feature.OverlayAnalysis,
Feature.OverlayAnalysisCodeScanningJavascript,
],
isPullRequest: true,
diskUsage: undefined,
},
{
overlayDatabaseMode: OverlayDatabaseMode.None,
useOverlayDatabaseCaching: false,
},
);
test(
getOverlayDatabaseModeMacro,
"No overlay analysis on PR when code-scanning feature enabled with disable-default-queries",
@@ -1558,9 +1506,10 @@ test(
test(
getOverlayDatabaseModeMacro,
"Overlay PR analysis by env",
"Overlay PR analysis by env for dsp-testing",
{
overlayDatabaseEnvVar: "overlay",
repositoryOwner: "dsp-testing",
},
{
overlayDatabaseMode: OverlayDatabaseMode.Overlay,
@@ -1570,10 +1519,10 @@ test(
test(
getOverlayDatabaseModeMacro,
"Overlay PR analysis by env on a runner with low disk space",
"Overlay PR analysis by env for other-org",
{
overlayDatabaseEnvVar: "overlay",
diskUsage: { numAvailableBytes: 0, numTotalBytes: 100_000_000_000 },
repositoryOwner: "other-org",
},
{
overlayDatabaseMode: OverlayDatabaseMode.Overlay,
@@ -1583,11 +1532,12 @@ test(
test(
getOverlayDatabaseModeMacro,
"Overlay PR analysis by feature flag",
"Overlay PR analysis by feature flag for dsp-testing",
{
languages: [KnownLanguage.javascript],
features: [Feature.OverlayAnalysis, Feature.OverlayAnalysisJavascript],
isPullRequest: true,
repositoryOwner: "dsp-testing",
},
{
overlayDatabaseMode: OverlayDatabaseMode.Overlay,
@@ -1595,6 +1545,21 @@ test(
},
);
test(
getOverlayDatabaseModeMacro,
"No overlay PR analysis by feature flag for other-org",
{
languages: [KnownLanguage.javascript],
features: [Feature.OverlayAnalysis, Feature.OverlayAnalysisJavascript],
isPullRequest: true,
repositoryOwner: "other-org",
},
{
overlayDatabaseMode: OverlayDatabaseMode.None,
useOverlayDatabaseCaching: false,
},
);
test(
getOverlayDatabaseModeMacro,
"Fallback due to autobuild with traced language",

View File

@@ -34,7 +34,6 @@ import {
OverlayDatabaseMode,
} from "./overlay-database-utils";
import { RepositoryNwo } from "./repository";
import { ToolsFeature } from "./tools-features";
import { downloadTrapCaches } from "./trap-caching";
import {
GitHubVersion,
@@ -43,22 +42,10 @@ import {
codeQlVersionAtLeast,
cloneObject,
isDefined,
checkDiskUsage,
} from "./util";
export * from "./config/db-config";
/**
* The minimum available disk space (in MB) required to perform overlay analysis.
* If the available disk space on the runner is below the threshold when deciding
* whether to perform overlay analysis, then the action will not perform overlay
* analysis unless overlay analysis has been explicitly enabled via environment
* variable.
*/
const OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_MB = 20000;
const OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_BYTES =
OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_MB * 1_000_000;
export type RegistryConfigWithCredentials = RegistryConfigNoCredentials & {
// Token to use when downloading packs from this registry.
token: string;
@@ -160,9 +147,6 @@ export interface Config {
/** A value indicating how dependency caching should be used. */
dependencyCachingEnabled: CachingKind;
/** The keys of caches that we restored, if any. */
dependencyCachingRestoredKeys: string[];
/**
* Extra query exclusions to append to the config.
*/
@@ -191,12 +175,14 @@ export interface Config {
repositoryProperties: RepositoryProperties;
}
async function getSupportedLanguageMap(
export async function getSupportedLanguageMap(
codeql: CodeQL,
features: FeatureEnablement,
logger: Logger,
): Promise<Record<string, string>> {
const resolveSupportedLanguagesUsingCli = await codeql.supportsFeature(
ToolsFeature.BuiltinExtractorsSpecifyDefaultQueries,
const resolveSupportedLanguagesUsingCli = await features.getValue(
Feature.ResolveSupportedLanguagesUsingCli,
codeql,
);
const resolveResult = await codeql.betterResolveLanguages({
filterToLanguagesWithQueries: resolveSupportedLanguagesUsingCli,
@@ -254,7 +240,7 @@ export function hasActionsWorkflows(sourceRoot: string): boolean {
/**
* Gets the set of languages in the current repository.
*/
async function getRawLanguagesInRepo(
export async function getRawLanguagesInRepo(
repository: RepositoryNwo,
sourceRoot: string,
logger: Logger,
@@ -297,6 +283,7 @@ export async function getLanguages(
languagesInput: string | undefined,
repository: RepositoryNwo,
sourceRoot: string,
features: FeatureEnablement,
logger: Logger,
): Promise<Language[]> {
// Obtain languages without filtering them.
@@ -307,7 +294,7 @@ export async function getLanguages(
logger,
);
const languageMap = await getSupportedLanguageMap(codeql, logger);
const languageMap = await getSupportedLanguageMap(codeql, features, logger);
const languagesSet = new Set<Language>();
const unknownLanguages: string[] = [];
@@ -363,7 +350,7 @@ export function getRawLanguagesNoAutodetect(
* @returns A tuple containing a list of languages in this repository that might be
* analyzable and whether or not this list was determined automatically.
*/
async function getRawLanguages(
export async function getRawLanguages(
languagesInput: string | undefined,
repository: RepositoryNwo,
sourceRoot: string,
@@ -444,6 +431,7 @@ export async function initActionState(
languagesInput,
repository,
sourceRoot,
features,
logger,
);
@@ -511,7 +499,6 @@ export async function initActionState(
trapCaches,
trapCacheDownloadTime,
dependencyCachingEnabled: getCachingKind(dependencyCachingEnabled),
dependencyCachingRestoredKeys: [],
extraQueryExclusions: [],
overlayDatabaseMode: OverlayDatabaseMode.None,
useOverlayDatabaseCaching: false,
@@ -595,11 +582,17 @@ const OVERLAY_ANALYSIS_CODE_SCANNING_FEATURES: Record<Language, Feature> = {
};
async function isOverlayAnalysisFeatureEnabled(
repository: RepositoryNwo,
features: FeatureEnablement,
codeql: CodeQL,
languages: Language[],
codeScanningConfig: UserConfig,
): Promise<boolean> {
// TODO: Remove the repository owner check once support for overlay analysis
// stabilizes, and no more backward-incompatible changes are expected.
if (!["github", "dsp-testing"].includes(repository.owner)) {
return false;
}
if (!(await features.getValue(Feature.OverlayAnalysis, codeql))) {
return false;
}
@@ -657,6 +650,7 @@ async function isOverlayAnalysisFeatureEnabled(
*/
export async function getOverlayDatabaseMode(
codeql: CodeQL,
repository: RepositoryNwo,
features: FeatureEnablement,
languages: Language[],
sourceRoot: string,
@@ -685,43 +679,27 @@ export async function getOverlayDatabaseMode(
);
} else if (
await isOverlayAnalysisFeatureEnabled(
repository,
features,
codeql,
languages,
codeScanningConfig,
)
) {
const diskUsage = await checkDiskUsage(logger);
if (
diskUsage === undefined ||
diskUsage.numAvailableBytes < OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_BYTES
) {
const diskSpaceMb =
diskUsage === undefined
? 0
: Math.round(diskUsage.numAvailableBytes / 1_000_000);
overlayDatabaseMode = OverlayDatabaseMode.None;
useOverlayDatabaseCaching = false;
if (isAnalyzingPullRequest()) {
overlayDatabaseMode = OverlayDatabaseMode.Overlay;
useOverlayDatabaseCaching = true;
logger.info(
`Setting overlay database mode to ${overlayDatabaseMode} ` +
`due to insufficient disk space (${diskSpaceMb} MB).`,
"with caching because we are analyzing a pull request.",
);
} else if (await isAnalyzingDefaultBranch()) {
overlayDatabaseMode = OverlayDatabaseMode.OverlayBase;
useOverlayDatabaseCaching = true;
logger.info(
`Setting overlay database mode to ${overlayDatabaseMode} ` +
"with caching because we are analyzing the default branch.",
);
} else {
if (isAnalyzingPullRequest()) {
overlayDatabaseMode = OverlayDatabaseMode.Overlay;
useOverlayDatabaseCaching = true;
logger.info(
`Setting overlay database mode to ${overlayDatabaseMode} ` +
"with caching because we are analyzing a pull request.",
);
} else if (await isAnalyzingDefaultBranch()) {
overlayDatabaseMode = OverlayDatabaseMode.OverlayBase;
useOverlayDatabaseCaching = true;
logger.info(
`Setting overlay database mode to ${overlayDatabaseMode} ` +
"with caching because we are analyzing the default branch.",
);
}
}
}
@@ -798,8 +776,7 @@ function userConfigFromActionPath(tempDir: string): string {
function hasQueryCustomisation(userConfig: UserConfig): boolean {
return (
isDefined(userConfig["disable-default-queries"]) ||
isDefined(userConfig.queries) ||
isDefined(userConfig["query-filters"])
isDefined(userConfig.queries)
);
}
@@ -861,7 +838,6 @@ export async function initConfig(
// Set the query customisation options for Code Quality only analysis.
config.computedConfig["disable-default-queries"] = true;
config.computedConfig.queries = queries;
config.computedConfig["query-filters"] = [];
}
// The choice of overlay database mode depends on the selection of languages
@@ -871,6 +847,7 @@ export async function initConfig(
const { overlayDatabaseMode, useOverlayDatabaseCaching } =
await getOverlayDatabaseMode(
inputs.codeql,
inputs.repository,
inputs.features,
config.languages,
inputs.sourceRoot,
@@ -1057,6 +1034,7 @@ export async function getConfig(
* pack.
*
* @param registriesInput The value of the `registries` input.
* @param codeQL a codeQL object, used only for checking the version of CodeQL.
* @param tempDir a temporary directory to store the generated qlconfig.yml file.
* @param logger a logger object.
* @returns The path to the generated `qlconfig.yml` file and the auth tokens to
@@ -1259,7 +1237,7 @@ export function isCodeQualityEnabled(config: Config): boolean {
* @returns Returns `AnalysisKind.CodeScanning` if `AnalysisKind.CodeScanning` is enabled;
* otherwise `AnalysisKind.CodeQuality`.
*/
function getPrimaryAnalysisKind(config: Config): AnalysisKind {
export function getPrimaryAnalysisKind(config: Config): AnalysisKind {
return isCodeScanningEnabled(config)
? AnalysisKind.CodeScanning
: AnalysisKind.CodeQuality;

View File

@@ -160,6 +160,7 @@ const PACK_IDENTIFIER_PATTERN = (function () {
* Version and path are optional.
*
* @param packStr the package specification to verify.
* @param configFile Config file to use for error reporting
*/
export function parsePacksSpecification(packStr: string): Pack {
if (typeof packStr !== "string") {

View File

@@ -10,12 +10,11 @@ import { GitHubApiDetails } from "./api-client";
import * as apiClient from "./api-client";
import { createStubCodeQL } from "./codeql";
import { Config } from "./config-utils";
import { cleanupAndUploadDatabases } from "./database-upload";
import { uploadDatabases } from "./database-upload";
import * as gitUtils from "./git-utils";
import { KnownLanguage } from "./languages";
import { RepositoryNwo } from "./repository";
import {
createFeatures,
createTestConfig,
getRecordingLogger,
LoggedMessage,
@@ -92,12 +91,11 @@ test("Abort database upload if 'upload-database' input set to false", async (t)
sinon.stub(gitUtils, "isAnalyzingDefaultBranch").resolves(true);
const loggedMessages = [];
await cleanupAndUploadDatabases(
await uploadDatabases(
testRepoName,
getCodeQL(),
getTestConfig(tmpDir),
testApiDetails,
createFeatures([]),
getRecordingLogger(loggedMessages),
);
t.assert(
@@ -123,7 +121,7 @@ test("Abort database upload if 'analysis-kinds: code-scanning' is not enabled",
await mockHttpRequests(201);
const loggedMessages = [];
await cleanupAndUploadDatabases(
await uploadDatabases(
testRepoName,
getCodeQL(),
{
@@ -131,7 +129,6 @@ test("Abort database upload if 'analysis-kinds: code-scanning' is not enabled",
analysisKinds: [AnalysisKind.CodeQuality],
},
testApiDetails,
createFeatures([]),
getRecordingLogger(loggedMessages),
);
t.assert(
@@ -158,12 +155,11 @@ test("Abort database upload if running against GHES", async (t) => {
config.gitHubVersion = { type: GitHubVariant.GHES, version: "3.0" };
const loggedMessages = [];
await cleanupAndUploadDatabases(
await uploadDatabases(
testRepoName,
getCodeQL(),
config,
testApiDetails,
createFeatures([]),
getRecordingLogger(loggedMessages),
);
t.assert(
@@ -187,12 +183,11 @@ test("Abort database upload if not analyzing default branch", async (t) => {
sinon.stub(gitUtils, "isAnalyzingDefaultBranch").resolves(false);
const loggedMessages = [];
await cleanupAndUploadDatabases(
await uploadDatabases(
testRepoName,
getCodeQL(),
getTestConfig(tmpDir),
testApiDetails,
createFeatures([]),
getRecordingLogger(loggedMessages),
);
t.assert(
@@ -217,12 +212,11 @@ test("Don't crash if uploading a database fails", async (t) => {
await mockHttpRequests(500);
const loggedMessages = [] as LoggedMessage[];
await cleanupAndUploadDatabases(
await uploadDatabases(
testRepoName,
getCodeQL(),
getTestConfig(tmpDir),
testApiDetails,
createFeatures([]),
getRecordingLogger(loggedMessages),
);
@@ -249,12 +243,11 @@ test("Successfully uploading a database to github.com", async (t) => {
await mockHttpRequests(201);
const loggedMessages = [] as LoggedMessage[];
await cleanupAndUploadDatabases(
await uploadDatabases(
testRepoName,
getCodeQL(),
getTestConfig(tmpDir),
testApiDetails,
createFeatures([]),
getRecordingLogger(loggedMessages),
);
t.assert(
@@ -279,7 +272,7 @@ test("Successfully uploading a database to GHEC-DR", async (t) => {
const databaseUploadSpy = await mockHttpRequests(201);
const loggedMessages = [] as LoggedMessage[];
await cleanupAndUploadDatabases(
await uploadDatabases(
testRepoName,
getCodeQL(),
getTestConfig(tmpDir),
@@ -288,7 +281,6 @@ test("Successfully uploading a database to GHEC-DR", async (t) => {
url: "https://tenant.ghe.com",
apiURL: undefined,
},
createFeatures([]),
getRecordingLogger(loggedMessages),
);
t.assert(

View File

@@ -5,20 +5,17 @@ import { AnalysisKind } from "./analyses";
import { getApiClient, GitHubApiDetails } from "./api-client";
import { type CodeQL } from "./codeql";
import { Config } from "./config-utils";
import { Feature, FeatureEnablement } from "./feature-flags";
import * as gitUtils from "./git-utils";
import { Logger, withGroupAsync } from "./logging";
import { OverlayDatabaseMode } from "./overlay-database-utils";
import { RepositoryNwo } from "./repository";
import * as util from "./util";
import { bundleDb, CleanupLevel, parseGitHubUrl } from "./util";
import { bundleDb, parseGitHubUrl } from "./util";
export async function cleanupAndUploadDatabases(
export async function uploadDatabases(
repositoryNwo: RepositoryNwo,
codeql: CodeQL,
config: Config,
apiDetails: GitHubApiDetails,
features: FeatureEnablement,
logger: Logger,
): Promise<void> {
if (actionsUtil.getRequiredInput("upload-database") !== "true") {
@@ -53,16 +50,10 @@ export async function cleanupAndUploadDatabases(
return;
}
const cleanupLevel =
config.overlayDatabaseMode === OverlayDatabaseMode.OverlayBase &&
(await features.getValue(Feature.UploadOverlayDbToApi))
? CleanupLevel.Overlay
: CleanupLevel.Clear;
// Clean up the database, since intermediate results may still be written to the
// database if there is high RAM pressure.
await withGroupAsync("Cleaning up databases", async () => {
await codeql.databaseCleanupCluster(config, cleanupLevel);
await codeql.databaseCleanupCluster(config, "clear");
});
const client = getApiClient();

View File

@@ -1,6 +1,6 @@
{
"bundleVersion": "codeql-bundle-v2.23.5",
"cliVersion": "2.23.5",
"priorBundleVersion": "codeql-bundle-v2.23.3",
"priorCliVersion": "2.23.3"
"bundleVersion": "codeql-bundle-v2.23.3",
"cliVersion": "2.23.3",
"priorBundleVersion": "codeql-bundle-v2.23.2",
"priorCliVersion": "2.23.2"
}

View File

@@ -1,627 +0,0 @@
import * as fs from "fs";
import path from "path";
import * as actionsCache from "@actions/cache";
import * as glob from "@actions/glob";
import test from "ava";
import * as sinon from "sinon";
import { cacheKeyHashLength } from "./caching-utils";
import * as cachingUtils from "./caching-utils";
import { createStubCodeQL } from "./codeql";
import {
CacheConfig,
checkHashPatterns,
getCsharpHashPatterns,
getFeaturePrefix,
makePatternCheck,
internal,
CSHARP_BASE_PATTERNS,
CSHARP_EXTRA_PATTERNS,
downloadDependencyCaches,
CacheHitKind,
cacheKey,
uploadDependencyCaches,
CacheStoreResult,
} from "./dependency-caching";
import { Feature } from "./feature-flags";
import { KnownLanguage } from "./languages";
import {
setupTests,
createFeatures,
getRecordingLogger,
checkExpectedLogMessages,
LoggedMessage,
createTestConfig,
} from "./testing-utils";
import { withTmpDir } from "./util";
setupTests(test);
function makeAbsolutePatterns(tmpDir: string, patterns: string[]): string[] {
return patterns.map((pattern) => path.join(tmpDir, pattern));
}
test("makePatternCheck - returns undefined if no patterns match", async (t) => {
await withTmpDir(async (tmpDir) => {
fs.writeFileSync(path.join(tmpDir, "test.java"), "");
const result = await makePatternCheck(
makeAbsolutePatterns(tmpDir, ["**/*.cs"]),
);
t.is(result, undefined);
});
});
test("makePatternCheck - returns all patterns if any pattern matches", async (t) => {
await withTmpDir(async (tmpDir) => {
fs.writeFileSync(path.join(tmpDir, "test.java"), "");
const patterns = makeAbsolutePatterns(tmpDir, ["**/*.cs", "**/*.java"]);
const result = await makePatternCheck(patterns);
t.deepEqual(result, patterns);
});
});
test("getCsharpHashPatterns - returns base patterns if any pattern matches", async (t) => {
const codeql = createStubCodeQL({});
const features = createFeatures([]);
const makePatternCheckStub = sinon.stub(internal, "makePatternCheck");
makePatternCheckStub
.withArgs(CSHARP_BASE_PATTERNS)
.resolves(CSHARP_BASE_PATTERNS);
makePatternCheckStub.withArgs(CSHARP_EXTRA_PATTERNS).rejects();
await t.notThrowsAsync(async () => {
const result = await getCsharpHashPatterns(codeql, features);
t.deepEqual(result, CSHARP_BASE_PATTERNS);
});
});
test("getCsharpHashPatterns - returns base patterns if any base pattern matches and CsharpNewCacheKey is enabled", async (t) => {
const codeql = createStubCodeQL({});
const features = createFeatures([Feature.CsharpNewCacheKey]);
const makePatternCheckStub = sinon.stub(internal, "makePatternCheck");
makePatternCheckStub
.withArgs(CSHARP_BASE_PATTERNS)
.resolves(CSHARP_BASE_PATTERNS);
makePatternCheckStub
.withArgs(CSHARP_EXTRA_PATTERNS)
.resolves(CSHARP_EXTRA_PATTERNS);
await t.notThrowsAsync(async () => {
const result = await getCsharpHashPatterns(codeql, features);
t.deepEqual(result, CSHARP_BASE_PATTERNS);
});
});
test("getCsharpHashPatterns - returns extra patterns if any extra pattern matches and CsharpNewCacheKey is enabled", async (t) => {
const codeql = createStubCodeQL({});
const features = createFeatures([Feature.CsharpNewCacheKey]);
const makePatternCheckStub = sinon.stub(internal, "makePatternCheck");
makePatternCheckStub.withArgs(CSHARP_BASE_PATTERNS).resolves(undefined);
makePatternCheckStub
.withArgs(CSHARP_EXTRA_PATTERNS)
.resolves(CSHARP_EXTRA_PATTERNS);
await t.notThrowsAsync(async () => {
const result = await getCsharpHashPatterns(codeql, features);
t.deepEqual(result, CSHARP_EXTRA_PATTERNS);
});
});
test("getCsharpHashPatterns - returns undefined if neither base nor extra patterns match", async (t) => {
const codeql = createStubCodeQL({});
const features = createFeatures([Feature.CsharpNewCacheKey]);
const makePatternCheckStub = sinon.stub(internal, "makePatternCheck");
makePatternCheckStub.withArgs(CSHARP_BASE_PATTERNS).resolves(undefined);
makePatternCheckStub.withArgs(CSHARP_EXTRA_PATTERNS).resolves(undefined);
await t.notThrowsAsync(async () => {
const result = await getCsharpHashPatterns(codeql, features);
t.deepEqual(result, undefined);
});
});
test("checkHashPatterns - logs when no patterns match", async (t) => {
const codeql = createStubCodeQL({});
const features = createFeatures([]);
const messages: LoggedMessage[] = [];
const config: CacheConfig = {
getDependencyPaths: () => [],
getHashPatterns: async () => undefined,
};
const result = await checkHashPatterns(
codeql,
features,
KnownLanguage.csharp,
config,
"download",
getRecordingLogger(messages),
);
t.is(result, undefined);
checkExpectedLogMessages(t, messages, [
"Skipping download of dependency cache",
]);
});
test("checkHashPatterns - returns patterns when patterns match", async (t) => {
await withTmpDir(async (tmpDir) => {
const codeql = createStubCodeQL({});
const features = createFeatures([]);
const messages: LoggedMessage[] = [];
const patterns = makeAbsolutePatterns(tmpDir, ["**/*.cs", "**/*.java"]);
fs.writeFileSync(path.join(tmpDir, "test.java"), "");
const config: CacheConfig = {
getDependencyPaths: () => [],
getHashPatterns: async () => makePatternCheck(patterns),
};
const result = await checkHashPatterns(
codeql,
features,
KnownLanguage.csharp,
config,
"upload",
getRecordingLogger(messages),
);
t.deepEqual(result, patterns);
t.deepEqual(messages, []);
});
});
type RestoreCacheFunc = (
paths: string[],
primaryKey: string,
restoreKeys: string[] | undefined,
) => Promise<string | undefined>;
/**
* Constructs a function that `actionsCache.restoreCache` can be stubbed with.
*
* @param mockCacheKeys The keys of caches that we want to exist in the Actions cache.
*
* @returns Returns a function that `actionsCache.restoreCache` can be stubbed with.
*/
function makeMockCacheCheck(mockCacheKeys: string[]): RestoreCacheFunc {
return async (
_paths: string[],
primaryKey: string,
restoreKeys: string[] | undefined,
) => {
// The behaviour here mirrors what the real `restoreCache` would do:
// - Starting with the primary restore key, check all caches for a match:
// even for the primary restore key, this only has to be a prefix match.
// - If the primary restore key doesn't prefix-match any cache, then proceed
// in the same way for each restore key in turn.
for (const restoreKey of [primaryKey, ...(restoreKeys || [])]) {
for (const mockCacheKey of mockCacheKeys) {
if (mockCacheKey.startsWith(restoreKey)) {
return mockCacheKey;
}
}
}
// Only if no restore key matches any cache key prefix, there is no matching
// cache and we return `undefined`.
return undefined;
};
}
test("downloadDependencyCaches - does not restore caches with feature keys if no features are enabled", async (t) => {
process.env["RUNNER_OS"] = "Linux";
const codeql = createStubCodeQL({});
const messages: LoggedMessage[] = [];
const logger = getRecordingLogger(messages);
sinon.stub(glob, "hashFiles").resolves("abcdef");
const keyWithFeature = await cacheKey(
codeql,
createFeatures([Feature.CsharpNewCacheKey]),
KnownLanguage.csharp,
// Patterns don't matter here because we have stubbed `hashFiles` to always return a specific hash above.
[],
);
const restoreCacheStub = sinon
.stub(actionsCache, "restoreCache")
.callsFake(makeMockCacheCheck([keyWithFeature]));
const makePatternCheckStub = sinon.stub(internal, "makePatternCheck");
makePatternCheckStub
.withArgs(CSHARP_BASE_PATTERNS)
.resolves(CSHARP_BASE_PATTERNS);
makePatternCheckStub.withArgs(CSHARP_EXTRA_PATTERNS).resolves(undefined);
const result = await downloadDependencyCaches(
codeql,
createFeatures([]),
[KnownLanguage.csharp],
logger,
);
const statusReport = result.statusReport;
t.is(statusReport.length, 1);
t.is(statusReport[0].language, KnownLanguage.csharp);
t.is(statusReport[0].hit_kind, CacheHitKind.Miss);
t.deepEqual(result.restoredKeys, []);
t.assert(restoreCacheStub.calledOnce);
});
test("downloadDependencyCaches - restores caches with feature keys if features are enabled", async (t) => {
process.env["RUNNER_OS"] = "Linux";
const codeql = createStubCodeQL({});
const messages: LoggedMessage[] = [];
const logger = getRecordingLogger(messages);
const features = createFeatures([Feature.CsharpNewCacheKey]);
const mockHash = "abcdef";
sinon.stub(glob, "hashFiles").resolves(mockHash);
const keyWithFeature = await cacheKey(
codeql,
features,
KnownLanguage.csharp,
// Patterns don't matter here because we have stubbed `hashFiles` to always return a specific hash above.
[],
);
const restoreCacheStub = sinon
.stub(actionsCache, "restoreCache")
.callsFake(makeMockCacheCheck([keyWithFeature]));
const makePatternCheckStub = sinon.stub(internal, "makePatternCheck");
makePatternCheckStub
.withArgs(CSHARP_BASE_PATTERNS)
.resolves(CSHARP_BASE_PATTERNS);
makePatternCheckStub.withArgs(CSHARP_EXTRA_PATTERNS).resolves(undefined);
const result = await downloadDependencyCaches(
codeql,
features,
[KnownLanguage.csharp],
logger,
);
// Check that the status report for telemetry indicates that one cache was restored with an exact match.
const statusReport = result.statusReport;
t.is(statusReport.length, 1);
t.is(statusReport[0].language, KnownLanguage.csharp);
t.is(statusReport[0].hit_kind, CacheHitKind.Exact);
// Check that the restored key has been returned.
const restoredKeys = result.restoredKeys;
t.is(restoredKeys.length, 1);
t.assert(
restoredKeys[0].endsWith(mockHash),
"Expected restored key to end with hash returned by `hashFiles`",
);
// `restoreCache` should have been called exactly once.
t.assert(restoreCacheStub.calledOnce);
});
test("downloadDependencyCaches - restores caches with feature keys if features are enabled for partial matches", async (t) => {
process.env["RUNNER_OS"] = "Linux";
const codeql = createStubCodeQL({});
const messages: LoggedMessage[] = [];
const logger = getRecordingLogger(messages);
const features = createFeatures([Feature.CsharpNewCacheKey]);
// We expect two calls to `hashFiles`: the first by the call to `cacheKey` below,
// and the second by `downloadDependencyCaches`. We use the result of the first
// call as part of the cache key that identifies a mock, existing cache. The result
// of the second call is for the primary restore key, which we don't want to match
// the first key so that we can test the restore keys logic.
const restoredHash = "abcdef";
const hashFilesStub = sinon.stub(glob, "hashFiles");
hashFilesStub.onFirstCall().resolves(restoredHash);
hashFilesStub.onSecondCall().resolves("123456");
const keyWithFeature = await cacheKey(
codeql,
features,
KnownLanguage.csharp,
// Patterns don't matter here because we have stubbed `hashFiles` to always return a specific hash above.
[],
);
const restoreCacheStub = sinon
.stub(actionsCache, "restoreCache")
.callsFake(makeMockCacheCheck([keyWithFeature]));
const makePatternCheckStub = sinon.stub(internal, "makePatternCheck");
makePatternCheckStub
.withArgs(CSHARP_BASE_PATTERNS)
.resolves(CSHARP_BASE_PATTERNS);
makePatternCheckStub.withArgs(CSHARP_EXTRA_PATTERNS).resolves(undefined);
const result = await downloadDependencyCaches(
codeql,
features,
[KnownLanguage.csharp],
logger,
);
// Check that the status report for telemetry indicates that one cache was restored with a partial match.
const statusReport = result.statusReport;
t.is(statusReport.length, 1);
t.is(statusReport[0].language, KnownLanguage.csharp);
t.is(statusReport[0].hit_kind, CacheHitKind.Partial);
// Check that the restored key has been returned.
const restoredKeys = result.restoredKeys;
t.is(restoredKeys.length, 1);
t.assert(
restoredKeys[0].endsWith(restoredHash),
"Expected restored key to end with hash returned by `hashFiles`",
);
t.assert(restoreCacheStub.calledOnce);
});
test("uploadDependencyCaches - skips upload for a language with no cache config", async (t) => {
const codeql = createStubCodeQL({});
const messages: LoggedMessage[] = [];
const logger = getRecordingLogger(messages);
const features = createFeatures([]);
const config = createTestConfig({
languages: [KnownLanguage.actions],
});
const result = await uploadDependencyCaches(codeql, features, config, logger);
t.is(result.length, 0);
checkExpectedLogMessages(t, messages, [
"Skipping upload of dependency cache for actions",
]);
});
test("uploadDependencyCaches - skips upload if no files for the hash exist", async (t) => {
const codeql = createStubCodeQL({});
const messages: LoggedMessage[] = [];
const logger = getRecordingLogger(messages);
const features = createFeatures([]);
const config = createTestConfig({
languages: [KnownLanguage.go],
});
const makePatternCheckStub = sinon.stub(internal, "makePatternCheck");
makePatternCheckStub.resolves(undefined);
const result = await uploadDependencyCaches(codeql, features, config, logger);
t.is(result.length, 1);
t.is(result[0].language, KnownLanguage.go);
t.is(result[0].result, CacheStoreResult.NoHash);
});
test("uploadDependencyCaches - skips upload if we know the cache already exists", async (t) => {
process.env["RUNNER_OS"] = "Linux";
const codeql = createStubCodeQL({});
const messages: LoggedMessage[] = [];
const logger = getRecordingLogger(messages);
const features = createFeatures([]);
const mockHash = "abcdef";
sinon.stub(glob, "hashFiles").resolves(mockHash);
const makePatternCheckStub = sinon.stub(internal, "makePatternCheck");
makePatternCheckStub
.withArgs(CSHARP_BASE_PATTERNS)
.resolves(CSHARP_BASE_PATTERNS);
const primaryCacheKey = await cacheKey(
codeql,
features,
KnownLanguage.csharp,
CSHARP_BASE_PATTERNS,
);
const config = createTestConfig({
languages: [KnownLanguage.csharp],
dependencyCachingRestoredKeys: [primaryCacheKey],
});
const result = await uploadDependencyCaches(codeql, features, config, logger);
t.is(result.length, 1);
t.is(result[0].language, KnownLanguage.csharp);
t.is(result[0].result, CacheStoreResult.Duplicate);
});
test("uploadDependencyCaches - skips upload if cache size is 0", async (t) => {
process.env["RUNNER_OS"] = "Linux";
const codeql = createStubCodeQL({});
const messages: LoggedMessage[] = [];
const logger = getRecordingLogger(messages);
const features = createFeatures([]);
const mockHash = "abcdef";
sinon.stub(glob, "hashFiles").resolves(mockHash);
const makePatternCheckStub = sinon.stub(internal, "makePatternCheck");
makePatternCheckStub
.withArgs(CSHARP_BASE_PATTERNS)
.resolves(CSHARP_BASE_PATTERNS);
sinon.stub(cachingUtils, "getTotalCacheSize").resolves(0);
const config = createTestConfig({
languages: [KnownLanguage.csharp],
});
const result = await uploadDependencyCaches(codeql, features, config, logger);
t.is(result.length, 1);
t.is(result[0].language, KnownLanguage.csharp);
t.is(result[0].result, CacheStoreResult.Empty);
checkExpectedLogMessages(t, messages, [
"Skipping upload of dependency cache",
]);
});
test("uploadDependencyCaches - uploads caches when all requirements are met", async (t) => {
process.env["RUNNER_OS"] = "Linux";
const codeql = createStubCodeQL({});
const messages: LoggedMessage[] = [];
const logger = getRecordingLogger(messages);
const features = createFeatures([]);
const mockHash = "abcdef";
sinon.stub(glob, "hashFiles").resolves(mockHash);
const makePatternCheckStub = sinon.stub(internal, "makePatternCheck");
makePatternCheckStub
.withArgs(CSHARP_BASE_PATTERNS)
.resolves(CSHARP_BASE_PATTERNS);
sinon.stub(cachingUtils, "getTotalCacheSize").resolves(1024);
sinon.stub(actionsCache, "saveCache").resolves();
const config = createTestConfig({
languages: [KnownLanguage.csharp],
});
const result = await uploadDependencyCaches(codeql, features, config, logger);
t.is(result.length, 1);
t.is(result[0].language, KnownLanguage.csharp);
t.is(result[0].result, CacheStoreResult.Stored);
t.is(result[0].upload_size_bytes, 1024);
checkExpectedLogMessages(t, messages, ["Uploading cache of size"]);
});
test("uploadDependencyCaches - catches `ReserveCacheError` exceptions", async (t) => {
process.env["RUNNER_OS"] = "Linux";
const codeql = createStubCodeQL({});
const messages: LoggedMessage[] = [];
const logger = getRecordingLogger(messages);
const features = createFeatures([]);
const mockHash = "abcdef";
sinon.stub(glob, "hashFiles").resolves(mockHash);
const makePatternCheckStub = sinon.stub(internal, "makePatternCheck");
makePatternCheckStub
.withArgs(CSHARP_BASE_PATTERNS)
.resolves(CSHARP_BASE_PATTERNS);
sinon.stub(cachingUtils, "getTotalCacheSize").resolves(1024);
sinon
.stub(actionsCache, "saveCache")
.throws(new actionsCache.ReserveCacheError("Already in use"));
const config = createTestConfig({
languages: [KnownLanguage.csharp],
});
await t.notThrowsAsync(async () => {
const result = await uploadDependencyCaches(
codeql,
features,
config,
logger,
);
t.is(result.length, 1);
t.is(result[0].language, KnownLanguage.csharp);
t.is(result[0].result, CacheStoreResult.Duplicate);
checkExpectedLogMessages(t, messages, ["Not uploading cache for"]);
});
});
test("uploadDependencyCaches - throws other exceptions", async (t) => {
process.env["RUNNER_OS"] = "Linux";
const codeql = createStubCodeQL({});
const messages: LoggedMessage[] = [];
const logger = getRecordingLogger(messages);
const features = createFeatures([]);
const mockHash = "abcdef";
sinon.stub(glob, "hashFiles").resolves(mockHash);
const makePatternCheckStub = sinon.stub(internal, "makePatternCheck");
makePatternCheckStub
.withArgs(CSHARP_BASE_PATTERNS)
.resolves(CSHARP_BASE_PATTERNS);
sinon.stub(cachingUtils, "getTotalCacheSize").resolves(1024);
sinon.stub(actionsCache, "saveCache").throws();
const config = createTestConfig({
languages: [KnownLanguage.csharp],
});
await t.throwsAsync(async () => {
await uploadDependencyCaches(codeql, features, config, logger);
});
});
test("getFeaturePrefix - returns empty string if no features are enabled", async (t) => {
const codeql = createStubCodeQL({});
const features = createFeatures([]);
for (const knownLanguage of Object.values(KnownLanguage)) {
const result = await getFeaturePrefix(codeql, features, knownLanguage);
t.deepEqual(result, "", `Expected no feature prefix for ${knownLanguage}`);
}
});
test("getFeaturePrefix - Java - returns 'minify-' if JavaMinimizeDependencyJars is enabled", async (t) => {
const codeql = createStubCodeQL({});
const features = createFeatures([Feature.JavaMinimizeDependencyJars]);
const result = await getFeaturePrefix(codeql, features, KnownLanguage.java);
t.deepEqual(result, "minify-");
});
test("getFeaturePrefix - non-Java - returns '' if JavaMinimizeDependencyJars is enabled", async (t) => {
const codeql = createStubCodeQL({});
const features = createFeatures([Feature.JavaMinimizeDependencyJars]);
for (const knownLanguage of Object.values(KnownLanguage)) {
// Skip Java since we expect a result for it, which is tested in the previous test.
if (knownLanguage === KnownLanguage.java) {
continue;
}
const result = await getFeaturePrefix(codeql, features, knownLanguage);
t.deepEqual(result, "", `Expected no feature prefix for ${knownLanguage}`);
}
});
test("getFeaturePrefix - C# - returns prefix if CsharpNewCacheKey is enabled", async (t) => {
const codeql = createStubCodeQL({});
const features = createFeatures([Feature.CsharpNewCacheKey]);
const result = await getFeaturePrefix(codeql, features, KnownLanguage.csharp);
t.notDeepEqual(result, "");
t.assert(result.endsWith("-"));
// Check the length of the prefix, which should correspond to `cacheKeyHashLength` + 1 for the trailing `-`.
t.is(result.length, cacheKeyHashLength + 1);
});
test("getFeaturePrefix - non-C# - returns '' if CsharpNewCacheKey is enabled", async (t) => {
const codeql = createStubCodeQL({});
const features = createFeatures([Feature.CsharpNewCacheKey]);
for (const knownLanguage of Object.values(KnownLanguage)) {
// Skip C# since we expect a result for it, which is tested in the previous test.
if (knownLanguage === KnownLanguage.csharp) {
continue;
}
const result = await getFeaturePrefix(codeql, features, knownLanguage);
t.deepEqual(result, "", `Expected no feature prefix for ${knownLanguage}`);
}
});

View File

@@ -6,11 +6,9 @@ import * as glob from "@actions/glob";
import { getTemporaryDirectory } from "./actions-util";
import { listActionsCaches } from "./api-client";
import { createCacheKeyHash, getTotalCacheSize } from "./caching-utils";
import { CodeQL } from "./codeql";
import { getTotalCacheSize } from "./caching-utils";
import { Config } from "./config-utils";
import { EnvVar } from "./environment";
import { Feature, FeatureEnablement } from "./feature-flags";
import { KnownLanguage, Language } from "./languages";
import { Logger } from "./logging";
import { getErrorMessage, getRequiredEnvParam } from "./util";
@@ -18,21 +16,15 @@ import { getErrorMessage, getRequiredEnvParam } from "./util";
/**
* Caching configuration for a particular language.
*/
export interface CacheConfig {
/** Gets the paths of directories on the runner that should be included in the cache. */
getDependencyPaths: () => string[];
interface CacheConfig {
/** The paths of directories on the runner that should be included in the cache. */
paths: string[];
/**
* Gets an array of glob patterns for the paths of files whose contents affect which dependencies are used
* by a project. This function also checks whether there are any matching files and returns
* `undefined` if no files match.
*
* The glob patterns are intended to be used for cache keys, where we find all files which match these
* patterns, calculate a hash for their contents, and use that hash as part of the cache key.
* Patterns for the paths of files whose contents affect which dependencies are used
* by a project. We find all files which match these patterns, calculate a hash for
* their contents, and use that hash as part of the cache key.
*/
getHashPatterns: (
codeql: CodeQL,
features: FeatureEnablement,
) => Promise<string[] | undefined>;
hash: string[];
}
const CODEQL_DEPENDENCY_CACHE_PREFIX = "codeql-dependencies";
@@ -47,105 +39,21 @@ export function getJavaTempDependencyDir(): string {
return join(getTemporaryDirectory(), "codeql_java", "repository");
}
/**
* Returns an array of paths of directories on the runner that should be included in a dependency cache
* for a Java analysis. It is important that this is a function, because we call `getTemporaryDirectory`
* which would otherwise fail in tests if we haven't had a chance to initialise `RUNNER_TEMP`.
*
* @returns The paths of directories on the runner that should be included in a dependency cache
* for a Java analysis.
*/
function getJavaDependencyDirs(): string[] {
return [
// Maven
join(os.homedir(), ".m2", "repository"),
// Gradle
join(os.homedir(), ".gradle", "caches"),
// CodeQL Java build-mode: none
getJavaTempDependencyDir(),
];
}
/**
* Checks that there are files which match `patterns`. If there are matching files for any of the patterns,
* this function returns all `patterns`. Otherwise, `undefined` is returned.
*
* @param patterns The glob patterns to find matching files for.
* @returns The array of glob patterns if there are matching files, or `undefined` otherwise.
*/
export async function makePatternCheck(
patterns: string[],
): Promise<string[] | undefined> {
const globber = await makeGlobber(patterns);
if ((await globber.glob()).length === 0) {
return undefined;
}
return patterns;
}
/** These files contain accurate information about dependencies, including the exact versions
* that the relevant package manager has determined for the project. Using these gives us
* stable hashes unless the dependencies change.
*/
export const CSHARP_BASE_PATTERNS = [
// NuGet
"**/packages.lock.json",
// Paket
"**/paket.lock",
];
/** These are less accurate for use in cache key calculations, because they:
*
* - Don't contain the exact versions used. They may only contain version ranges or none at all.
* - They contain information unrelated to dependencies, which we don't care about.
*
* As a result, the hash we compute from these files may change, even if
* the dependencies haven't changed.
*/
export const CSHARP_EXTRA_PATTERNS = [
"**/*.csproj",
"**/packages.config",
"**/nuget.config",
];
/**
* Returns the list of glob patterns that should be used to calculate the cache key hash
* for a C# dependency cache. This will try to use `CSHARP_BASE_PATTERNS` whenever possible.
* As a fallback, it will also use `CSHARP_EXTRA_PATTERNS` if the corresponding FF is enabled.
*
* @param codeql The CodeQL instance to use.
* @param features Information about which FFs are enabled.
* @returns A list of glob patterns to use for hashing.
*/
export async function getCsharpHashPatterns(
codeql: CodeQL,
features: FeatureEnablement,
): Promise<string[] | undefined> {
const basePatterns = await internal.makePatternCheck(CSHARP_BASE_PATTERNS);
if (basePatterns !== undefined) {
return basePatterns;
}
if (await features.getValue(Feature.CsharpNewCacheKey, codeql)) {
return internal.makePatternCheck(CSHARP_EXTRA_PATTERNS);
}
// If we get to this point, we didn't find any files with `CSHARP_BASE_PATTERNS`,
// and `Feature.CsharpNewCacheKey` is not enabled.
return undefined;
}
/**
* Default caching configurations per language.
*/
const defaultCacheConfigs: { [language: string]: CacheConfig } = {
java: {
getDependencyPaths: getJavaDependencyDirs,
getHashPatterns: async () =>
internal.makePatternCheck([
function getDefaultCacheConfig(): { [language: string]: CacheConfig } {
return {
java: {
paths: [
// Maven
join(os.homedir(), ".m2", "repository"),
// Gradle
join(os.homedir(), ".gradle", "caches"),
// CodeQL Java build-mode: none
getJavaTempDependencyDir(),
],
hash: [
// Maven
"**/pom.xml",
// Gradle
@@ -155,17 +63,23 @@ const defaultCacheConfigs: { [language: string]: CacheConfig } = {
"buildSrc/**/Dependencies.kt",
"gradle/*.versions.toml",
"**/versions.properties",
]),
},
csharp: {
getDependencyPaths: () => [join(os.homedir(), ".nuget", "packages")],
getHashPatterns: getCsharpHashPatterns,
},
go: {
getDependencyPaths: () => [join(os.homedir(), "go", "pkg", "mod")],
getHashPatterns: async () => internal.makePatternCheck(["**/go.sum"]),
},
};
],
},
csharp: {
paths: [join(os.homedir(), ".nuget", "packages")],
hash: [
// NuGet
"**/packages.lock.json",
// Paket
"**/paket.lock",
],
},
go: {
paths: [join(os.homedir(), "go", "pkg", "mod")],
hash: ["**/go.sum"],
},
};
}
async function makeGlobber(patterns: string[]): Promise<glob.Globber> {
return glob.create(patterns.join("\n"));
@@ -193,66 +107,23 @@ export interface DependencyCacheRestoreStatus {
/** An array of `DependencyCacheRestoreStatus` objects for each analysed language with a caching configuration. */
export type DependencyCacheRestoreStatusReport = DependencyCacheRestoreStatus[];
/** Represents the results of `downloadDependencyCaches`. */
export interface DownloadDependencyCachesResult {
/** The status report for telemetry */
statusReport: DependencyCacheRestoreStatusReport;
/** An array of cache keys that we have restored and therefore know to exist. */
restoredKeys: string[];
}
/**
* A wrapper around `cacheConfig.getHashPatterns` which logs when there are no files to calculate
* a hash for the cache key from.
*
* @param codeql The CodeQL instance to use.
* @param features Information about which FFs are enabled.
* @param language The language the `CacheConfig` is for. For use in the log message.
* @param cacheConfig The caching configuration to call `getHashPatterns` on.
* @param checkType Whether we are checking the patterns for a download or upload.
* @param logger The logger to write the log message to if there is an error.
* @returns An array of glob patterns to use for hashing files, or `undefined` if there are no matching files.
*/
export async function checkHashPatterns(
codeql: CodeQL,
features: FeatureEnablement,
language: Language,
cacheConfig: CacheConfig,
checkType: "download" | "upload",
logger: Logger,
): Promise<string[] | undefined> {
const patterns = await cacheConfig.getHashPatterns(codeql, features);
if (patterns === undefined) {
logger.info(
`Skipping ${checkType} of dependency cache for ${language} as we cannot calculate a hash for the cache key.`,
);
}
return patterns;
}
/**
* Attempts to restore dependency caches for the languages being analyzed.
*
* @param codeql The CodeQL instance to use.
* @param features Information about which FFs are enabled.
* @param languages The languages being analyzed.
* @param logger A logger to record some informational messages to.
*
* @param minimizeJavaJars Whether the Java extractor should rewrite downloaded JARs to minimize their size.
* @returns An array of `DependencyCacheRestoreStatus` objects for each analysed language with a caching configuration.
*/
export async function downloadDependencyCaches(
codeql: CodeQL,
features: FeatureEnablement,
languages: Language[],
logger: Logger,
): Promise<DownloadDependencyCachesResult> {
minimizeJavaJars: boolean,
): Promise<DependencyCacheRestoreStatusReport> {
const status: DependencyCacheRestoreStatusReport = [];
const restoredKeys: string[] = [];
for (const language of languages) {
const cacheConfig = defaultCacheConfigs[language];
const cacheConfig = getDefaultCacheConfig()[language];
if (cacheConfig === undefined) {
logger.info(
@@ -263,22 +134,19 @@ export async function downloadDependencyCaches(
// Check that we can find files to calculate the hash for the cache key from, so we don't end up
// with an empty string.
const patterns = await checkHashPatterns(
codeql,
features,
language,
cacheConfig,
"download",
logger,
);
if (patterns === undefined) {
const globber = await makeGlobber(cacheConfig.hash);
if ((await globber.glob()).length === 0) {
status.push({ language, hit_kind: CacheHitKind.NoHash });
logger.info(
`Skipping download of dependency cache for ${language} as we cannot calculate a hash for the cache key.`,
);
continue;
}
const primaryKey = await cacheKey(codeql, features, language, patterns);
const primaryKey = await cacheKey(language, cacheConfig, minimizeJavaJars);
const restoreKeys: string[] = [
await cachePrefix(codeql, features, language),
await cachePrefix(language, minimizeJavaJars),
];
logger.info(
@@ -289,7 +157,7 @@ export async function downloadDependencyCaches(
const start = performance.now();
const hitKey = await actionsCache.restoreCache(
cacheConfig.getDependencyPaths(),
cacheConfig.paths,
primaryKey,
restoreKeys,
);
@@ -297,27 +165,16 @@ export async function downloadDependencyCaches(
if (hitKey !== undefined) {
logger.info(`Cache hit on key ${hitKey} for ${language}.`);
// We have a partial cache hit, unless the key of the restored cache matches the
// primary restore key.
let hit_kind = CacheHitKind.Partial;
if (hitKey === primaryKey) {
hit_kind = CacheHitKind.Exact;
}
status.push({
language,
hit_kind,
download_duration_ms,
});
restoredKeys.push(hitKey);
const hit_kind =
hitKey === primaryKey ? CacheHitKind.Exact : CacheHitKind.Partial;
status.push({ language, hit_kind, download_duration_ms });
} else {
status.push({ language, hit_kind: CacheHitKind.Miss });
logger.info(`No suitable cache found for ${language}.`);
}
}
return { statusReport: status, restoredKeys };
return status;
}
/** Enumerates possible outcomes for storing caches. */
@@ -346,22 +203,20 @@ export type DependencyCacheUploadStatusReport = DependencyCacheUploadStatus[];
/**
* Attempts to store caches for the languages that were analyzed.
*
* @param codeql The CodeQL instance to use.
* @param features Information about which FFs are enabled.
* @param config The configuration for this workflow.
* @param logger A logger to record some informational messages to.
* @param minimizeJavaJars Whether the Java extractor should rewrite downloaded JARs to minimize their size.
*
* @returns An array of `DependencyCacheUploadStatus` objects for each analysed language with a caching configuration.
*/
export async function uploadDependencyCaches(
codeql: CodeQL,
features: FeatureEnablement,
config: Config,
logger: Logger,
minimizeJavaJars: boolean,
): Promise<DependencyCacheUploadStatusReport> {
const status: DependencyCacheUploadStatusReport = [];
for (const language of config.languages) {
const cacheConfig = defaultCacheConfigs[language];
const cacheConfig = getDefaultCacheConfig()[language];
if (cacheConfig === undefined) {
logger.info(
@@ -372,28 +227,13 @@ export async function uploadDependencyCaches(
// Check that we can find files to calculate the hash for the cache key from, so we don't end up
// with an empty string.
const patterns = await checkHashPatterns(
codeql,
features,
language,
cacheConfig,
"upload",
logger,
);
if (patterns === undefined) {
const globber = await makeGlobber(cacheConfig.hash);
if ((await globber.glob()).length === 0) {
status.push({ language, result: CacheStoreResult.NoHash });
continue;
}
// Now that we have verified that there are suitable files, compute the hash for the cache key.
const key = await cacheKey(codeql, features, language, patterns);
// Check that we haven't previously restored this exact key. If a cache with this key
// already exists in the Actions Cache, performing the next steps is pointless as the cache
// will not get overwritten. We can therefore skip the expensive work of measuring the size
// of the cache contents and attempting to upload it if we know that the cache already exists.
if (config.dependencyCachingRestoredKeys.includes(key)) {
status.push({ language, result: CacheStoreResult.Duplicate });
logger.info(
`Skipping upload of dependency cache for ${language} as we cannot calculate a hash for the cache key.`,
);
continue;
}
@@ -407,11 +247,7 @@ export async function uploadDependencyCaches(
// use the cache quota that we compete with. In that case, we do not wish to use up all of the quota
// with the dependency caches. For this, we could use the Cache API to check whether other workflows
// are using the quota and how full it is.
const size = await getTotalCacheSize(
cacheConfig.getDependencyPaths(),
logger,
true,
);
const size = await getTotalCacheSize(cacheConfig.paths, logger, true);
// Skip uploading an empty cache.
if (size === 0) {
@@ -422,13 +258,15 @@ export async function uploadDependencyCaches(
continue;
}
const key = await cacheKey(language, cacheConfig, minimizeJavaJars);
logger.info(
`Uploading cache of size ${size} for ${language} with key ${key}...`,
);
try {
const start = performance.now();
await actionsCache.saveCache(cacheConfig.getDependencyPaths(), key);
await actionsCache.saveCache(cacheConfig.paths, key);
const upload_duration_ms = Math.round(performance.now() - start);
status.push({
@@ -461,86 +299,31 @@ export async function uploadDependencyCaches(
/**
* Computes a cache key for the specified language.
*
* @param codeql The CodeQL instance to use.
* @param features Information about which FFs are enabled.
* @param language The language being analyzed.
* @param patterns The file patterns to hash.
*
* @param cacheConfig The cache configuration for the language.
* @param minimizeJavaJars Whether the Java extractor should rewrite downloaded JARs to minimize their size.
* @returns A cache key capturing information about the project(s) being analyzed in the specified language.
*/
export async function cacheKey(
codeql: CodeQL,
features: FeatureEnablement,
async function cacheKey(
language: Language,
patterns: string[],
cacheConfig: CacheConfig,
minimizeJavaJars: boolean = false,
): Promise<string> {
const hash = await glob.hashFiles(patterns.join("\n"));
return `${await cachePrefix(codeql, features, language)}${hash}`;
}
/**
* If experimental features which the cache contents depend on are enabled for the current language,
* this function returns a prefix that uniquely identifies the set of enabled features. The purpose of
* this is to avoid restoring caches whose contents depended on experimental features, if those
* experimental features are later disabled.
*
* @param codeql The CodeQL instance.
* @param features Information about enabled features.
* @param language The language we are creating the key for.
*
* @returns A cache key prefix identifying the enabled, experimental features that the cache depends on.
*/
export async function getFeaturePrefix(
codeql: CodeQL,
features: FeatureEnablement,
language: Language,
): Promise<string> {
const enabledFeatures: Feature[] = [];
const addFeatureIfEnabled = async (feature: Feature) => {
if (await features.getValue(feature, codeql)) {
enabledFeatures.push(feature);
}
};
if (language === KnownLanguage.java) {
// To ensure a safe rollout of JAR minimization, we change the key when the feature is enabled.
const minimizeJavaJars = await features.getValue(
Feature.JavaMinimizeDependencyJars,
codeql,
);
// To maintain backwards compatibility with this, we return "minify-" instead of a hash.
if (minimizeJavaJars) {
return "minify-";
}
} else if (language === KnownLanguage.csharp) {
await addFeatureIfEnabled(Feature.CsharpNewCacheKey);
}
// If any features that affect the cache are enabled, return a feature prefix by
// computing a hash of the feature array.
if (enabledFeatures.length > 0) {
return `${createCacheKeyHash(enabledFeatures)}-`;
}
// No feature prefix.
return "";
const hash = await glob.hashFiles(cacheConfig.hash.join("\n"));
return `${await cachePrefix(language, minimizeJavaJars)}${hash}`;
}
/**
* Constructs a prefix for the cache key, comprised of a CodeQL-specific prefix, a version number that
* can be changed to invalidate old caches, the runner's operating system, and the specified language name.
*
* @param codeql The CodeQL instance to use.
* @param features Information about which FFs are enabled.
* @param language The language being analyzed.
* @param minimizeJavaJars Whether the Java extractor should rewrite downloaded JARs to minimize their size.
* @returns The prefix that identifies what a cache is for.
*/
async function cachePrefix(
codeql: CodeQL,
features: FeatureEnablement,
language: Language,
minimizeJavaJars: boolean,
): Promise<string> {
const runnerOs = getRequiredEnvParam("RUNNER_OS");
const customPrefix = process.env[EnvVar.DEPENDENCY_CACHING_PREFIX];
@@ -550,18 +333,12 @@ async function cachePrefix(
prefix = `${prefix}-${customPrefix}`;
}
// Calculate the feature prefix for the cache, if any. This is a hash that identifies
// experimental features that affect the cache contents.
const featurePrefix = await getFeaturePrefix(codeql, features, language);
// Assemble the cache key. For backwards compatibility with the JAR minification experiment's existing
// feature prefix usage, we add that feature prefix at the start. Other feature prefixes are inserted
// after the general CodeQL dependency cache prefix.
if (featurePrefix === "minify-") {
return `${featurePrefix}${prefix}-${CODEQL_DEPENDENCY_CACHE_VERSION}-${runnerOs}-${language}-`;
} else {
return `${prefix}-${featurePrefix}${CODEQL_DEPENDENCY_CACHE_VERSION}-${runnerOs}-${language}-`;
// To ensure a safe rollout of JAR minimization, we change the key when the feature is enabled.
if (language === KnownLanguage.java && minimizeJavaJars) {
prefix = `minify-${prefix}`;
}
return `${prefix}-${CODEQL_DEPENDENCY_CACHE_VERSION}-${runnerOs}-${language}-`;
}
/** Represents information about our overall cache usage for CodeQL dependency caches. */
@@ -594,7 +371,3 @@ export async function getDependencyCacheUsage(
return undefined;
}
export const internal = {
makePatternCheck,
};

View File

@@ -20,6 +20,12 @@ export enum EnvVar {
/** Whether the CodeQL Action has invoked the Go autobuilder. */
DID_AUTOBUILD_GOLANG = "CODEQL_ACTION_DID_AUTOBUILD_GOLANG",
/**
* Whether to disable the SARIF post-processing in the Action that removes duplicate locations from
* notifications in the `run[].invocations[].toolExecutionNotifications` SARIF property.
*/
DISABLE_DUPLICATE_LOCATION_FIX = "CODEQL_ACTION_DISABLE_DUPLICATE_LOCATION_FIX",
/**
* Whether the CodeQL Action is using its own deprecated and non-standard way of scanning for
* multiple languages.
@@ -50,12 +56,20 @@ export enum EnvVar {
/** Whether the error for a deprecated version of the CodeQL Action was logged. */
LOG_VERSION_DEPRECATION = "CODEQL_ACTION_DID_LOG_VERSION_DEPRECATION",
/**
* For macOS. Result of `csrutil status` to determine whether System Integrity
* Protection is enabled.
*/
IS_SIP_ENABLED = "CODEQL_ACTION_IS_SIP_ENABLED",
/** UUID representing the current job run. */
JOB_RUN_UUID = "JOB_RUN_UUID",
/** Status for the entire job, submitted to the status report in `init-post` */
JOB_STATUS = "CODEQL_ACTION_JOB_STATUS",
ODASA_TRACER_CONFIGURATION = "ODASA_TRACER_CONFIGURATION",
/** The value of the `output` input for the analyze action. */
SARIF_RESULTS_OUTPUT_DIR = "CODEQL_ACTION_SARIF_RESULTS_OUTPUT_DIR",

View File

@@ -47,7 +47,6 @@ export enum Feature {
AnalyzeUseNewUpload = "analyze_use_new_upload",
CleanupTrapCaches = "cleanup_trap_caches",
CppDependencyInstallation = "cpp_dependency_installation_enabled",
CsharpNewCacheKey = "csharp_new_cache_key",
DiffInformedQueries = "diff_informed_queries",
DisableCsharpBuildless = "disable_csharp_buildless",
DisableJavaBuildlessEnabled = "disable_java_buildless_enabled",
@@ -77,7 +76,7 @@ export enum Feature {
OverlayAnalysisSwift = "overlay_analysis_swift",
PythonDefaultIsToNotExtractStdlib = "python_default_is_to_not_extract_stdlib",
QaTelemetryEnabled = "qa_telemetry_enabled",
UploadOverlayDbToApi = "upload_overlay_db_to_api",
ResolveSupportedLanguagesUsingCli = "resolve_supported_languages_using_cli",
UseRepositoryProperties = "use_repository_properties",
ValidateDbConfig = "validate_db_config",
}
@@ -134,11 +133,6 @@ export const featureConfig: Record<
legacyApi: true,
minimumVersion: "2.15.0",
},
[Feature.CsharpNewCacheKey]: {
defaultValue: false,
envVar: "CODEQL_ACTION_CSHARP_NEW_CACHE_KEY",
minimumVersion: undefined,
},
[Feature.DiffInformedQueries]: {
defaultValue: true,
envVar: "CODEQL_ACTION_DIFF_INFORMED_QUERIES",
@@ -167,10 +161,11 @@ export const featureConfig: Record<
legacyApi: true,
minimumVersion: undefined,
},
[Feature.JavaMinimizeDependencyJars]: {
[Feature.ResolveSupportedLanguagesUsingCli]: {
defaultValue: false,
envVar: "CODEQL_ACTION_JAVA_MINIMIZE_DEPENDENCY_JARS",
minimumVersion: "2.23.0",
envVar: "CODEQL_ACTION_RESOLVE_SUPPORTED_LANGUAGES_USING_CLI",
minimumVersion: undefined,
toolsFeature: ToolsFeature.BuiltinExtractorsSpecifyDefaultQueries,
},
[Feature.OverlayAnalysis]: {
defaultValue: false,
@@ -283,21 +278,21 @@ export const featureConfig: Record<
minimumVersion: undefined,
toolsFeature: ToolsFeature.PythonDefaultIsToNotExtractStdlib,
},
[Feature.UseRepositoryProperties]: {
defaultValue: false,
envVar: "CODEQL_ACTION_USE_REPOSITORY_PROPERTIES",
minimumVersion: undefined,
},
[Feature.QaTelemetryEnabled]: {
defaultValue: false,
envVar: "CODEQL_ACTION_QA_TELEMETRY",
legacyApi: true,
minimumVersion: undefined,
},
[Feature.UploadOverlayDbToApi]: {
[Feature.JavaMinimizeDependencyJars]: {
defaultValue: false,
envVar: "CODEQL_ACTION_UPLOAD_OVERLAY_DB_TO_API",
minimumVersion: undefined,
},
[Feature.UseRepositoryProperties]: {
defaultValue: false,
envVar: "CODEQL_ACTION_USE_REPOSITORY_PROPERTIES",
minimumVersion: undefined,
envVar: "CODEQL_ACTION_JAVA_MINIMIZE_DEPENDENCY_JARS",
minimumVersion: "2.23.0",
},
[Feature.ValidateDbConfig]: {
defaultValue: false,

View File

@@ -122,6 +122,67 @@ export const determineBaseBranchHeadCommitOid = async function (
}
};
/**
* Deepen the git history of HEAD by one level. Errors are logged.
*
* This function uses the `checkout_path` to determine the repository path and
* works only when called from `analyze` or `upload-sarif`.
*/
export const deepenGitHistory = async function () {
try {
await runGitCommand(
getOptionalInput("checkout_path"),
[
"fetch",
"origin",
"HEAD",
"--no-tags",
"--no-recurse-submodules",
"--deepen=1",
],
"Cannot deepen the shallow repository.",
);
} catch {
// Errors are already logged by runGitCommand()
}
};
/**
* Fetch the given remote branch. Errors are logged.
*
* This function uses the `checkout_path` to determine the repository path and
* works only when called from `analyze` or `upload-sarif`.
*/
export const gitFetch = async function (branch: string, extraFlags: string[]) {
try {
await runGitCommand(
getOptionalInput("checkout_path"),
["fetch", "--no-tags", ...extraFlags, "origin", `${branch}:${branch}`],
`Cannot fetch ${branch}.`,
);
} catch {
// Errors are already logged by runGitCommand()
}
};
/**
* Repack the git repository, using with the given flags. Errors are logged.
*
* This function uses the `checkout_path` to determine the repository path and
* works only when called from `analyze` or `upload-sarif`.
*/
export const gitRepack = async function (flags: string[]) {
try {
await runGitCommand(
getOptionalInput("checkout_path"),
["repack", ...flags],
"Cannot repack the repository.",
);
} catch {
// Errors are already logged by runGitCommand()
}
};
/**
* Decode, if necessary, a file path produced by Git. See
* https://git-scm.com/docs/git-config#Documentation/git-config.txt-corequotePath

View File

@@ -371,7 +371,7 @@ async function run() {
}
let overlayBaseDatabaseStats: OverlayBaseDatabaseDownloadStats | undefined;
let dependencyCachingStatus: DependencyCacheRestoreStatusReport | undefined;
let dependencyCachingResults: DependencyCacheRestoreStatusReport | undefined;
try {
if (
config.overlayDatabaseMode === OverlayDatabaseMode.Overlay &&
@@ -578,16 +578,16 @@ async function run() {
}
// Restore dependency cache(s), if they exist.
const minimizeJavaJars = await features.getValue(
Feature.JavaMinimizeDependencyJars,
codeql,
);
if (shouldRestoreCache(config.dependencyCachingEnabled)) {
const dependencyCachingResult = await downloadDependencyCaches(
codeql,
features,
dependencyCachingResults = await downloadDependencyCaches(
config.languages,
logger,
minimizeJavaJars,
);
dependencyCachingStatus = dependencyCachingResult.statusReport;
config.dependencyCachingRestoredKeys =
dependencyCachingResult.restoredKeys;
}
// Suppress warnings about disabled Python library extraction.
@@ -648,7 +648,7 @@ async function run() {
`${EnvVar.JAVA_EXTRACTOR_MINIMIZE_DEPENDENCY_JARS} is already set to '${process.env[EnvVar.JAVA_EXTRACTOR_MINIMIZE_DEPENDENCY_JARS]}', so the Action will not override it.`,
);
} else if (
(await features.getValue(Feature.JavaMinimizeDependencyJars, codeql)) &&
minimizeJavaJars &&
config.dependencyCachingEnabled &&
config.buildMode === BuildMode.None &&
config.languages.includes(KnownLanguage.java)
@@ -735,7 +735,7 @@ async function run() {
toolsSource,
toolsVersion,
overlayBaseDatabaseStats,
dependencyCachingStatus,
dependencyCachingResults,
logger,
error,
);
@@ -758,7 +758,7 @@ async function run() {
toolsSource,
toolsVersion,
overlayBaseDatabaseStats,
dependencyCachingStatus,
dependencyCachingResults,
logger,
);
}

View File

@@ -1,3 +1,4 @@
import * as crypto from "crypto";
import * as fs from "fs";
import * as path from "path";
@@ -10,13 +11,11 @@ import {
getWorkflowRunID,
} from "./actions-util";
import { getAutomationID } from "./api-client";
import { createCacheKeyHash } from "./caching-utils";
import { type CodeQL } from "./codeql";
import { type Config } from "./config-utils";
import { getCommitOid, getFileOidsUnderPath } from "./git-utils";
import { Logger, withGroupAsync } from "./logging";
import {
CleanupLevel,
getErrorMessage,
isInTestMode,
tryGetFolderBytes,
@@ -29,7 +28,7 @@ export enum OverlayDatabaseMode {
None = "none",
}
export const CODEQL_OVERLAY_MINIMUM_VERSION = "2.23.5";
export const CODEQL_OVERLAY_MINIMUM_VERSION = "2.22.4";
/**
* The maximum (uncompressed) size of the overlay base database that we will
@@ -176,7 +175,7 @@ const MAX_CACHE_OPERATION_MS = 600_000;
* @param warningPrefix Prefix for the check failure warning message
* @returns True if the verification succeeded, false otherwise
*/
function checkOverlayBaseDatabase(
export function checkOverlayBaseDatabase(
config: Config,
logger: Logger,
warningPrefix: string,
@@ -205,7 +204,7 @@ function checkOverlayBaseDatabase(
* @returns A promise that resolves to true if the upload was performed and
* successfully completed, or false otherwise
*/
export async function cleanupAndUploadOverlayBaseDatabaseToCache(
export async function uploadOverlayBaseDatabaseToCache(
codeql: CodeQL,
config: Config,
logger: Logger,
@@ -243,7 +242,7 @@ export async function cleanupAndUploadOverlayBaseDatabaseToCache(
// Clean up the database using the overlay cleanup level.
await withGroupAsync("Cleaning up databases", async () => {
await codeql.databaseCleanupCluster(config, CleanupLevel.Overlay);
await codeql.databaseCleanupCluster(config, "overlay");
});
const dbLocation = config.dbLocation;
@@ -515,3 +514,27 @@ export async function getCacheRestoreKeyPrefix(
// easier to debug and understand the cache key structure.
return `${CACHE_PREFIX}-${CACHE_VERSION}-${componentsHash}-${languages}-${codeQlVersion}-`;
}
/**
* Creates a SHA-256 hash of the cache key components to ensure uniqueness
* while keeping the cache key length manageable.
*
* @param components Object containing all components that should influence cache key uniqueness
* @returns A short SHA-256 hash (first 16 characters) of the components
*/
function createCacheKeyHash(components: Record<string, any>): string {
// From https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/JSON/stringify
//
// "Properties are visited using the same algorithm as Object.keys(), which
// has a well-defined order and is stable across implementations. For example,
// JSON.stringify on the same object will always produce the same string, and
// JSON.parse(JSON.stringify(obj)) would produce an object with the same key
// ordering as the original (assuming the object is completely
// JSON-serializable)."
const componentsJson = JSON.stringify(components);
return crypto
.createHash("sha256")
.update(componentsJson)
.digest("hex")
.substring(0, 16);
}

View File

@@ -34,7 +34,7 @@ export enum ToolsSource {
Download = "DOWNLOAD",
}
const CODEQL_DEFAULT_ACTION_REPOSITORY = "github/codeql-action";
export const CODEQL_DEFAULT_ACTION_REPOSITORY = "github/codeql-action";
const CODEQL_NIGHTLIES_REPOSITORY_OWNER = "dsp-testing";
const CODEQL_NIGHTLIES_REPOSITORY_NAME = "codeql-cli-nightlies";
@@ -180,6 +180,17 @@ export function tryGetTagNameFromUrl(
return match[1];
}
export function tryGetBundleVersionFromUrl(
url: string,
logger: Logger,
): string | undefined {
const tagName = tryGetTagNameFromUrl(url, logger);
if (tagName === undefined) {
return undefined;
}
return tryGetBundleVersionFromTagName(tagName, logger);
}
export function convertToSemVer(version: string, logger: Logger): string {
if (!semver.valid(version)) {
logger.debug(
@@ -569,7 +580,7 @@ export async function getCodeQLSource(
* Gets a fallback version number to use when looking for CodeQL in the toolcache if we didn't find
* the `x.y.z` version. This is to support old versions of the toolcache.
*/
async function tryGetFallbackToolcacheVersion(
export async function tryGetFallbackToolcacheVersion(
cliVersion: string | undefined,
tagName: string,
logger: Logger,
@@ -718,7 +729,7 @@ function getCanonicalToolcacheVersion(
return cliVersion;
}
interface SetupCodeQLResult {
export interface SetupCodeQLResult {
codeqlFolder: string;
toolsDownloadStatusReport?: ToolsDownloadStatusReport;
toolsSource: ToolsSource;
@@ -739,7 +750,7 @@ export async function setupCodeQLBundle(
defaultCliVersion: CodeQLDefaultVersionInfo,
features: FeatureEnablement,
logger: Logger,
): Promise<SetupCodeQLResult> {
) {
if (!(await util.isBinaryAccessible("tar", logger))) {
throw new util.ConfigurationError(
"Could not find tar in PATH, so unable to extract CodeQL bundle.",

View File

@@ -8,7 +8,7 @@ import { ConfigurationError, getErrorMessage, isDefined } from "./util";
export const UPDATEJOB_PROXY = "update-job-proxy";
export const UPDATEJOB_PROXY_VERSION = "v2.0.20250624110901";
const UPDATEJOB_PROXY_URL_PREFIX =
export const UPDATEJOB_PROXY_URL_PREFIX =
"https://github.com/github/codeql-action/releases/download/codeql-bundle-v2.22.0/";
export type Credential = {
@@ -202,7 +202,7 @@ export function getFallbackUrl(proxyPackage: string): string {
*
* @returns The response from the GitHub API.
*/
async function getLinkedRelease() {
export async function getLinkedRelease() {
return getApiClient().rest.repos.getReleaseByTag({
owner: "github",
repo: "codeql-action",

View File

@@ -54,7 +54,7 @@ export enum ActionName {
* considered to be a third party analysis and is treated differently when calculating SLOs. To ensure
* misconfigured workflows are not treated as third party, only the upload-sarif action can return false.
*/
function isFirstPartyAnalysis(actionName: ActionName): boolean {
export function isFirstPartyAnalysis(actionName: ActionName): boolean {
if (actionName !== ActionName.UploadSarif) {
return true;
}
@@ -252,7 +252,7 @@ export interface EventReport {
*
* @param actionName The name of the action, e.g. 'init', 'finish', 'upload-sarif'
* @param status The status. Must be 'success', 'failure', or 'starting'
* @param actionStartedAt The time this action started executing.
* @param startedAt The time this action started executing.
* @param cause Cause of failure (only supply if status is 'failure')
* @param exception Exception (only supply if status is 'failure')
* @returns undefined if an exception was thrown.

View File

@@ -392,7 +392,6 @@ export function createTestConfig(overrides: Partial<Config>): Config {
trapCaches: {},
trapCacheDownloadTime: 0,
dependencyCachingEnabled: CachingKind.None,
dependencyCachingRestoredKeys: [],
extraQueryExclusions: [],
overlayDatabaseMode: OverlayDatabaseMode.None,
useOverlayDatabaseCaching: false,

View File

@@ -17,7 +17,7 @@ import { cleanUpPath, getErrorMessage, getRequiredEnvParam } from "./util";
/**
* High watermark to use when streaming the download and extraction of the CodeQL tools.
*/
const STREAMING_HIGH_WATERMARK_BYTES = 4 * 1024 * 1024; // 4 MiB
export const STREAMING_HIGH_WATERMARK_BYTES = 4 * 1024 * 1024; // 4 MiB
/**
* The name of the tool cache directory for the CodeQL tools.

View File

@@ -76,7 +76,7 @@ export async function endTracingForCluster(
}
}
async function getTracerConfigForCluster(
export async function getTracerConfigForCluster(
config: Config,
): Promise<TracerConfig> {
const tracingEnvVariables = JSON.parse(

View File

@@ -412,7 +412,7 @@ export function findSarifFilesInDir(
return sarifFiles;
}
function getSarifFilePaths(
export function getSarifFilePaths(
sarifPath: string,
isSarif: (name: string) => boolean,
) {
@@ -738,7 +738,7 @@ export async function postProcessSarifFiles(
* @param logger The logger to use.
* @param pathInput The input provided for `post-processed-sarif-path`.
* @param uploadTarget The upload target.
* @param postProcessingResults The results of post-processing SARIF files.
* @param processingResults The results of post-processing SARIF files.
*/
export async function writePostProcessedFiles(
logger: Logger,

View File

@@ -447,21 +447,13 @@ const CHECK_ACTION_VERSION_TESTS: Array<[string, util.GitHubVersion, boolean]> =
["2.2.1", { type: util.GitHubVariant.DOTCOM }, true],
["2.2.1", { type: util.GitHubVariant.GHE_DOTCOM }, true],
["2.2.1", { type: util.GitHubVariant.GHES, version: "3.10" }, false],
["2.2.1", { type: util.GitHubVariant.GHES, version: "3.11" }, false],
["2.2.1", { type: util.GitHubVariant.GHES, version: "3.12" }, false],
["3.2.1", { type: util.GitHubVariant.DOTCOM }, true],
["3.2.1", { type: util.GitHubVariant.GHE_DOTCOM }, true],
["2.2.1", { type: util.GitHubVariant.GHES, version: "3.11" }, true],
["2.2.1", { type: util.GitHubVariant.GHES, version: "3.12" }, true],
["3.2.1", { type: util.GitHubVariant.DOTCOM }, false],
["3.2.1", { type: util.GitHubVariant.GHE_DOTCOM }, false],
["3.2.1", { type: util.GitHubVariant.GHES, version: "3.10" }, false],
["3.2.1", { type: util.GitHubVariant.GHES, version: "3.11" }, false],
["3.2.1", { type: util.GitHubVariant.GHES, version: "3.12" }, false],
["3.2.1", { type: util.GitHubVariant.GHES, version: "3.19" }, false],
["3.2.1", { type: util.GitHubVariant.GHES, version: "3.20" }, true],
["3.2.1", { type: util.GitHubVariant.GHES, version: "3.21" }, true],
["4.2.1", { type: util.GitHubVariant.DOTCOM }, false],
["4.2.1", { type: util.GitHubVariant.GHE_DOTCOM }, false],
["4.2.1", { type: util.GitHubVariant.GHES, version: "3.19" }, false],
["4.2.1", { type: util.GitHubVariant.GHES, version: "3.20" }, false],
["4.2.1", { type: util.GitHubVariant.GHES, version: "3.21" }, false],
];
for (const [
@@ -476,7 +468,7 @@ for (const [
githubVersion,
)}`;
test(`checkActionVersion ${reportErrorDescription} for ${versionsDescription}`, async (t) => {
const warningSpy = sinon.spy(core, "warning");
const warningSpy = sinon.spy(core, "error");
const versionStub = sinon
.stub(api, "getGitHubVersion")
.resolves(githubVersion);
@@ -488,7 +480,9 @@ for (const [
if (shouldReportError) {
t.true(
warningSpy.calledOnceWithExactly(
sinon.match("CodeQL Action v3 will be deprecated in December 2026."),
sinon.match(
"CodeQL Action major versions v1 and v2 have been deprecated.",
),
),
);
} else {

View File

@@ -4,6 +4,7 @@ import * as os from "os";
import * as path from "path";
import * as core from "@actions/core";
import * as exec from "@actions/exec/lib/exec";
import * as io from "@actions/io";
import getFolderSize from "get-folder-size";
import * as yaml from "js-yaml";
@@ -1025,6 +1026,34 @@ export function fixInvalidNotifications(
return newSarif;
}
/**
* Removes duplicates from the sarif file.
*
* When `CODEQL_ACTION_DISABLE_DUPLICATE_LOCATION_FIX` is set to true, this will
* simply rename the input file to the output file. Otherwise, it will parse the
* input file as JSON, remove duplicate locations from the SARIF notification
* objects, and write the result to the output file.
*
* For context, see documentation of:
* `CODEQL_ACTION_DISABLE_DUPLICATE_LOCATION_FIX`. */
export function fixInvalidNotificationsInFile(
inputPath: string,
outputPath: string,
logger: Logger,
): void {
if (process.env[EnvVar.DISABLE_DUPLICATE_LOCATION_FIX] === "true") {
logger.info(
"SARIF notification object duplicate location fix disabled by the " +
`${EnvVar.DISABLE_DUPLICATE_LOCATION_FIX} environment variable.`,
);
fs.renameSync(inputPath, outputPath);
} else {
let sarif = JSON.parse(fs.readFileSync(inputPath, "utf8")) as SarifFile;
sarif = fixInvalidNotifications(sarif, logger);
fs.writeFileSync(outputPath, JSON.stringify(sarif));
}
}
export function wrapError(error: unknown): Error {
return error instanceof Error ? error : new Error(String(error));
}
@@ -1085,38 +1114,38 @@ export async function checkDiskUsage(
}
/**
* Prompt the customer to upgrade to CodeQL Action v4, if appropriate.
* Prompt the customer to upgrade to CodeQL Action v3, if appropriate.
*
* Check whether a customer is running v3. If they are, and we can determine that the GitHub
* instance supports v4, then log an error prompting the customer to upgrade to v4.
* Check whether a customer is running v1 or v2. If they are, and we can determine that the GitHub
* instance supports v3, then log an error prompting the customer to upgrade to v3.
*/
export function checkActionVersion(
version: string,
githubVersion: GitHubVersion,
) {
if (
!semver.satisfies(version, ">=4") && // do not log error if the customer is already running v4
!semver.satisfies(version, ">=3") && // do not log error if the customer is already running v3
!process.env[EnvVar.LOG_VERSION_DEPRECATION] // do not log error if we have already
) {
// Only error for versions of GHES that are compatible with CodeQL Action version 4.
// Only error for versions of GHES that are compatible with CodeQL Action version 3.
//
// GHES 3.20 is the first version to ship with the v4 tag and this warning message code.
// Therefore, users who are seeing this warning message code are running on GHES 3.20 or newer,
// and should update to CodeQL Action v4.
// GHES 3.11 shipped without the v3 tag, but it also shipped without this warning message code.
// Therefore users who are seeing this warning message code have pulled in a new version of the
// Action, and with it the v3 tag.
if (
githubVersion.type === GitHubVariant.DOTCOM ||
githubVersion.type === GitHubVariant.GHE_DOTCOM ||
(githubVersion.type === GitHubVariant.GHES &&
semver.satisfies(
semver.coerce(githubVersion.version) ?? "0.0.0",
">=3.20",
">=3.11",
))
) {
core.warning(
"CodeQL Action v3 will be deprecated in December 2026. " +
"Please update all occurrences of the CodeQL Action in your workflow files to v4. " +
core.error(
"CodeQL Action major versions v1 and v2 have been deprecated. " +
"Please update all occurrences of the CodeQL Action in your workflow files to v3. " +
"For more information, see " +
"https://github.blog/changelog/2025-10-28-upcoming-deprecation-of-codeql-action-v3/",
"https://github.blog/changelog/2025-01-10-code-scanning-codeql-action-v2-is-now-deprecated/",
);
// set LOG_VERSION_DEPRECATION env var to prevent the warning from being logged multiple times
core.exportVariable(EnvVar.LOG_VERSION_DEPRECATION, "true");
@@ -1168,6 +1197,49 @@ export function cloneObject<T>(obj: T): T {
return JSON.parse(JSON.stringify(obj)) as T;
}
// The first time this function is called, it runs `csrutil status` to determine
// whether System Integrity Protection is enabled; and saves the result in an
// environment variable. Afterwards, simply return the value of the environment
// variable.
export async function checkSipEnablement(
logger: Logger,
): Promise<boolean | undefined> {
if (
process.env[EnvVar.IS_SIP_ENABLED] !== undefined &&
["true", "false"].includes(process.env[EnvVar.IS_SIP_ENABLED])
) {
return process.env[EnvVar.IS_SIP_ENABLED] === "true";
}
try {
const sipStatusOutput = await exec.getExecOutput("csrutil status");
if (sipStatusOutput.exitCode === 0) {
if (
sipStatusOutput.stdout.includes(
"System Integrity Protection status: enabled.",
)
) {
core.exportVariable(EnvVar.IS_SIP_ENABLED, "true");
return true;
}
if (
sipStatusOutput.stdout.includes(
"System Integrity Protection status: disabled.",
)
) {
core.exportVariable(EnvVar.IS_SIP_ENABLED, "false");
return false;
}
}
return undefined;
} catch (e) {
logger.warning(
`Failed to determine if System Integrity Protection was enabled: ${e}`,
);
return undefined;
}
}
export async function cleanUpPath(file: string, name: string, logger: Logger) {
logger.debug(`Cleaning up ${name}.`);
try {
@@ -1219,6 +1291,17 @@ export function isDefined<T>(value: T | null | undefined): value is T {
return value !== undefined && value !== null;
}
/** Like `Object.keys`, but typed so that the elements of the resulting array have the
* same type as the keys of the input object. Note that this may not be sound if the input
* object has been cast to `T` from a subtype of `T` and contains additional keys that
* are not represented by `keyof T`.
*/
export function unsafeKeysInvariant<T extends Record<string, any>>(
object: T,
): Array<keyof T> {
return Object.keys(object) as Array<keyof T>;
}
/** Like `Object.entries`, but typed so that the key elements of the result have the
* same type as the keys of the input object. Note that this may not be sound if the input
* object has been cast to `T` from a subtype of `T` and contains additional keys that
@@ -1231,8 +1314,3 @@ export function unsafeEntriesInvariant<T extends Record<string, any>>(
([_, val]) => val !== undefined,
) as Array<[keyof T, Exclude<T[keyof T], undefined>]>;
}
export enum CleanupLevel {
Clear = "clear",
Overlay = "overlay",
}

View File

@@ -1,6 +0,0 @@
{
"sdk": {
"version": "9.0.307",
"rollForward": "latestFeature"
}
}