Compare commits

...

23 Commits

Author SHA1 Message Date
Henry Mercer
935969c6f7 Merge pull request #1013 from github/henrymercer/ml-powered-query-pack-v0.2.0
Run version `~0.2.0` of the ML-powered query pack on v2.8.4+ of the CLI
2022-03-31 16:25:07 +01:00
Henry Mercer
e26813cf98 Run version ~0.2.0 of the ML-powered query pack for v2.8.4+ of the CLI 2022-03-31 14:58:41 +01:00
Henry Mercer
2c03704a6c Allow the version of the ML-powered pack to depend on the CLI version 2022-03-31 14:58:29 +01:00
Henry Mercer
dd6b592e3e Simplify ML-powered query status report definition
We now limit the cardinality of the ML-powered JS queries status report
field server-side. With no need for a limit on the cardinality of the
status report client-side, we can simplify how we produce it.
2022-03-31 14:55:32 +01:00
Henry Mercer
a90d8bf711 Merge pull request #1011 from github/henrymercer/ml-powered-queries-pr-check
Add a PR check to validate that ML-powered queries are run correctly
2022-03-31 11:13:26 +01:00
Henry Mercer
dc0338e493 Use latest major version of actions/upload-artifact 2022-03-31 10:11:33 +01:00
Henry Mercer
57096fe795 Add a PR check to validate that ML-powered queries are run correctly 2022-03-31 10:11:30 +01:00
Henry Mercer
b0ddf36abe Merge pull request #1012 from github/henrymercer/update-actions-major-versions
Update major versions of Actions in README and workflows
2022-03-30 21:06:16 +01:00
Henry Mercer
1ea2f2d7f1 Merge branch 'main' into henrymercer/update-actions-major-versions 2022-03-30 20:00:06 +01:00
Henry Mercer
9dcc141f12 Merge pull request #1010 from github/henrymercer/stop-running-ml-powered-queries-on-windows
Stop running ML-powered queries on Windows
2022-03-30 19:57:03 +01:00
Henry Mercer
ea751a9fae Update other Actions from v2 to v3 2022-03-30 19:46:09 +01:00
Henry Mercer
a2949f47b3 Update actions/checkout from v2 to v3 2022-03-30 19:46:09 +01:00
Henry Mercer
7871f0d5e1 Update CodeQL Action from v1 to v2 in README 2022-03-30 19:46:09 +01:00
Henry Mercer
e6f3e049b4 Add descriptions to each test 2022-03-30 18:17:06 +01:00
Henry Mercer
e83a1d469e Stop running ML-powered queries on Windows 2022-03-30 18:05:12 +01:00
Edoardo Pirovano
894faced79 Merge pull request #1008 from github/edoardo/no-fail-12.12
Avoid failure if `@types/node` is already 12.12
2022-03-30 17:52:20 +01:00
Edoardo Pirovano
a9095cefc9 Avoid failure if @types/node is already 12.12 2022-03-30 16:58:25 +01:00
Henry Mercer
4d339ae3ec Merge pull request #1009 from github/henrymercer/run-pr-checks-on-v2-branch
Run all PR checks on the `v2` branch
2022-03-30 16:53:09 +01:00
Henry Mercer
381ea36211 Delete unused workflows 2022-03-30 16:00:52 +01:00
Henry Mercer
e769c2dd6e Run all PR checks on v2 branch 2022-03-30 15:59:09 +01:00
Edoardo Pirovano
bae3a3acab Merge pull request #1005 from github/mergeback/v2.1.6-to-main-28eead24
Mergeback v2.1.6 refs/heads/v2 into main
2022-03-30 14:27:22 +01:00
github-actions[bot]
bcd5c027de Update checked-in dependencies 2022-03-30 11:55:38 +00:00
github-actions[bot]
ee3341a9d8 Update changelog and version after v2.1.6 2022-03-30 11:13:37 +00:00
70 changed files with 687 additions and 335 deletions

View File

@@ -12,6 +12,7 @@ on:
branches: branches:
- main - main
- v1 - v1
- v2
pull_request: pull_request:
types: types:
- opened - opened
@@ -69,7 +70,7 @@ jobs:
runs-on: ${{ matrix.os }} runs-on: ${{ matrix.os }}
steps: steps:
- name: Check out repository - name: Check out repository
uses: actions/checkout@v2 uses: actions/checkout@v3
- name: Prepare test - name: Prepare test
id: prepare-test id: prepare-test
uses: ./.github/prepare-test uses: ./.github/prepare-test

View File

@@ -12,6 +12,7 @@ on:
branches: branches:
- main - main
- v1 - v1
- v2
pull_request: pull_request:
types: types:
- opened - opened
@@ -53,7 +54,7 @@ jobs:
runs-on: ${{ matrix.os }} runs-on: ${{ matrix.os }}
steps: steps:
- name: Check out repository - name: Check out repository
uses: actions/checkout@v2 uses: actions/checkout@v3
- name: Prepare test - name: Prepare test
id: prepare-test id: prepare-test
uses: ./.github/prepare-test uses: ./.github/prepare-test
@@ -70,7 +71,7 @@ jobs:
run: ./build.sh run: ./build.sh
- uses: ./../action/analyze - uses: ./../action/analyze
id: analysis id: analysis
- uses: actions/download-artifact@v2 - uses: actions/download-artifact@v3
with: with:
name: my-debug-artifacts-${{ matrix.os }}-${{ matrix.version }} name: my-debug-artifacts-${{ matrix.os }}-${{ matrix.version }}
- shell: bash - shell: bash

View File

@@ -12,6 +12,7 @@ on:
branches: branches:
- main - main
- v1 - v1
- v2
pull_request: pull_request:
types: types:
- opened - opened
@@ -31,7 +32,7 @@ jobs:
runs-on: ${{ matrix.os }} runs-on: ${{ matrix.os }}
steps: steps:
- name: Check out repository - name: Check out repository
uses: actions/checkout@v2 uses: actions/checkout@v3
- name: Prepare test - name: Prepare test
id: prepare-test id: prepare-test
uses: ./.github/prepare-test uses: ./.github/prepare-test

View File

@@ -12,6 +12,7 @@ on:
branches: branches:
- main - main
- v1 - v1
- v2
pull_request: pull_request:
types: types:
- opened - opened
@@ -69,13 +70,13 @@ jobs:
runs-on: ${{ matrix.os }} runs-on: ${{ matrix.os }}
steps: steps:
- name: Check out repository - name: Check out repository
uses: actions/checkout@v2 uses: actions/checkout@v3
- name: Prepare test - name: Prepare test
id: prepare-test id: prepare-test
uses: ./.github/prepare-test uses: ./.github/prepare-test
with: with:
version: ${{ matrix.version }} version: ${{ matrix.version }}
- uses: actions/setup-go@v2 - uses: actions/setup-go@v3
with: with:
go-version: ^1.13.1 go-version: ^1.13.1
- uses: ./../action/init - uses: ./../action/init

View File

@@ -12,6 +12,7 @@ on:
branches: branches:
- main - main
- v1 - v1
- v2
pull_request: pull_request:
types: types:
- opened - opened
@@ -53,13 +54,13 @@ jobs:
runs-on: ${{ matrix.os }} runs-on: ${{ matrix.os }}
steps: steps:
- name: Check out repository - name: Check out repository
uses: actions/checkout@v2 uses: actions/checkout@v3
- name: Prepare test - name: Prepare test
id: prepare-test id: prepare-test
uses: ./.github/prepare-test uses: ./.github/prepare-test
with: with:
version: ${{ matrix.version }} version: ${{ matrix.version }}
- uses: actions/setup-go@v2 - uses: actions/setup-go@v3
with: with:
go-version: ^1.13.1 go-version: ^1.13.1
- uses: ./../action/init - uses: ./../action/init

View File

@@ -12,6 +12,7 @@ on:
branches: branches:
- main - main
- v1 - v1
- v2
pull_request: pull_request:
types: types:
- opened - opened
@@ -69,13 +70,13 @@ jobs:
runs-on: ${{ matrix.os }} runs-on: ${{ matrix.os }}
steps: steps:
- name: Check out repository - name: Check out repository
uses: actions/checkout@v2 uses: actions/checkout@v3
- name: Prepare test - name: Prepare test
id: prepare-test id: prepare-test
uses: ./.github/prepare-test uses: ./.github/prepare-test
with: with:
version: ${{ matrix.version }} version: ${{ matrix.version }}
- uses: actions/setup-go@v2 - uses: actions/setup-go@v3
with: with:
go-version: ^1.13.1 go-version: ^1.13.1
- uses: ./../action/init - uses: ./../action/init

View File

@@ -12,6 +12,7 @@ on:
branches: branches:
- main - main
- v1 - v1
- v2
pull_request: pull_request:
types: types:
- opened - opened
@@ -35,7 +36,7 @@ jobs:
runs-on: ${{ matrix.os }} runs-on: ${{ matrix.os }}
steps: steps:
- name: Check out repository - name: Check out repository
uses: actions/checkout@v2 uses: actions/checkout@v3
- name: Prepare test - name: Prepare test
id: prepare-test id: prepare-test
uses: ./.github/prepare-test uses: ./.github/prepare-test

119
.github/workflows/__ml-powered-queries.yml generated vendored Normal file
View File

@@ -0,0 +1,119 @@
# Warning: This file is generated automatically, and should not be modified.
# Instead, please modify the template in the pr-checks directory and run:
# pip install ruamel.yaml && python3 sync.py
# to regenerate this file.
name: PR Check - ML-powered queries
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GO111MODULE: auto
on:
push:
branches:
- main
- v1
- v2
pull_request:
types:
- opened
- synchronize
- reopened
- ready_for_review
workflow_dispatch: {}
jobs:
ml-powered-queries:
strategy:
matrix:
include:
- os: ubuntu-latest
version: stable-20220120
- os: macos-latest
version: stable-20220120
- os: windows-latest
version: stable-20220120
- os: ubuntu-latest
version: cached
- os: macos-latest
version: cached
- os: windows-latest
version: cached
- os: ubuntu-latest
version: latest
- os: macos-latest
version: latest
- os: windows-latest
version: latest
- os: ubuntu-latest
version: nightly-latest
- os: macos-latest
version: nightly-latest
- os: windows-latest
version: nightly-latest
name: ML-powered queries
timeout-minutes: 45
runs-on: ${{ matrix.os }}
steps:
- name: Check out repository
uses: actions/checkout@v3
- name: Prepare test
id: prepare-test
uses: ./.github/prepare-test
with:
version: ${{ matrix.version }}
- uses: ./../action/init
with:
languages: javascript
queries: security-extended
source-root: ./../action/tests/ml-powered-queries-repo
tools: ${{ steps.prepare-test.outputs.tools-url }}
- uses: ./../action/analyze
with:
output: ${{ runner.temp }}/results
upload-database: false
env:
TEST_MODE: true
- name: Upload SARIF
uses: actions/upload-artifact@v3
with:
name: ml-powered-queries-${{ matrix.os }}-${{ matrix.version }}.sarif.json
path: ${{ runner.temp }}/results/javascript.sarif
retention-days: 7
- name: Check results
env:
IS_WINDOWS: ${{ matrix.os == 'windows-latest' }}
shell: bash
run: |
cd "$RUNNER_TEMP/results"
# We should run at least the ML-powered queries in `expected_rules`.
expected_rules="js/ml-powered/nosql-injection js/ml-powered/path-injection js/ml-powered/sql-injection js/ml-powered/xss"
for rule in ${expected_rules}; do
found_rule=$(jq --arg rule "${rule}" '[.runs[0].tool.extensions[].rules | select(. != null) |
flatten | .[].id] | any(. == $rule)' javascript.sarif)
echo "Did find rule '${rule}': ${found_rule}"
if [[ "${found_rule}" != "true" && "${IS_WINDOWS}" != "true" ]]; then
echo "Expected SARIF output to contain rule '${rule}', but found no such rule."
exit 1
elif [[ "${found_rule}" == "true" && "${IS_WINDOWS}" == "true" ]]; then
echo "Found rule '${rule}' in the SARIF output which shouldn't have been part of the analysis."
exit 1
fi
done
# We should have at least one alert from an ML-powered query.
num_alerts=$(jq '[.runs[0].results[] |
select(.properties.score != null and (.rule.id | startswith("js/ml-powered/")))] | length' \
javascript.sarif)
echo "Found ${num_alerts} alerts from ML-powered queries.";
if [[ "${num_alerts}" -eq 0 && "${IS_WINDOWS}" != "true" ]]; then
echo "Expected to find at least one alert from an ML-powered query but found ${num_alerts}."
exit 1
elif [[ "${num_alerts}" -ne 0 && "${IS_WINDOWS}" == "true" ]]; then
echo "Expected not to find any alerts from an ML-powered query but found ${num_alerts}."
exit 1
fi
env:
INTERNAL_CODEQL_ACTION_DEBUG_LOC: true

View File

@@ -12,6 +12,7 @@ on:
branches: branches:
- main - main
- v1 - v1
- v2
pull_request: pull_request:
types: types:
- opened - opened
@@ -53,7 +54,7 @@ jobs:
runs-on: ${{ matrix.os }} runs-on: ${{ matrix.os }}
steps: steps:
- name: Check out repository - name: Check out repository
uses: actions/checkout@v2 uses: actions/checkout@v3
- name: Prepare test - name: Prepare test
id: prepare-test id: prepare-test
uses: ./.github/prepare-test uses: ./.github/prepare-test

View File

@@ -12,6 +12,7 @@ on:
branches: branches:
- main - main
- v1 - v1
- v2
pull_request: pull_request:
types: types:
- opened - opened
@@ -33,7 +34,7 @@ jobs:
runs-on: ${{ matrix.os }} runs-on: ${{ matrix.os }}
steps: steps:
- name: Check out repository - name: Check out repository
uses: actions/checkout@v2 uses: actions/checkout@v3
- name: Prepare test - name: Prepare test
id: prepare-test id: prepare-test
uses: ./.github/prepare-test uses: ./.github/prepare-test

View File

@@ -12,6 +12,7 @@ on:
branches: branches:
- main - main
- v1 - v1
- v2
pull_request: pull_request:
types: types:
- opened - opened
@@ -33,7 +34,7 @@ jobs:
runs-on: ${{ matrix.os }} runs-on: ${{ matrix.os }}
steps: steps:
- name: Check out repository - name: Check out repository
uses: actions/checkout@v2 uses: actions/checkout@v3
- name: Prepare test - name: Prepare test
id: prepare-test id: prepare-test
uses: ./.github/prepare-test uses: ./.github/prepare-test

View File

@@ -12,6 +12,7 @@ on:
branches: branches:
- main - main
- v1 - v1
- v2
pull_request: pull_request:
types: types:
- opened - opened
@@ -33,7 +34,7 @@ jobs:
runs-on: ${{ matrix.os }} runs-on: ${{ matrix.os }}
steps: steps:
- name: Check out repository - name: Check out repository
uses: actions/checkout@v2 uses: actions/checkout@v3
- name: Prepare test - name: Prepare test
id: prepare-test id: prepare-test
uses: ./.github/prepare-test uses: ./.github/prepare-test

View File

@@ -12,6 +12,7 @@ on:
branches: branches:
- main - main
- v1 - v1
- v2
pull_request: pull_request:
types: types:
- opened - opened
@@ -69,7 +70,7 @@ jobs:
runs-on: ${{ matrix.os }} runs-on: ${{ matrix.os }}
steps: steps:
- name: Check out repository - name: Check out repository
uses: actions/checkout@v2 uses: actions/checkout@v3
- name: Prepare test - name: Prepare test
id: prepare-test id: prepare-test
uses: ./.github/prepare-test uses: ./.github/prepare-test

View File

@@ -12,6 +12,7 @@ on:
branches: branches:
- main - main
- v1 - v1
- v2
pull_request: pull_request:
types: types:
- opened - opened
@@ -41,7 +42,7 @@ jobs:
runs-on: ${{ matrix.os }} runs-on: ${{ matrix.os }}
steps: steps:
- name: Check out repository - name: Check out repository
uses: actions/checkout@v2 uses: actions/checkout@v3
- name: Prepare test - name: Prepare test
id: prepare-test id: prepare-test
uses: ./.github/prepare-test uses: ./.github/prepare-test

View File

@@ -12,6 +12,7 @@ on:
branches: branches:
- main - main
- v1 - v1
- v2
pull_request: pull_request:
types: types:
- opened - opened
@@ -33,7 +34,7 @@ jobs:
runs-on: ${{ matrix.os }} runs-on: ${{ matrix.os }}
steps: steps:
- name: Check out repository - name: Check out repository
uses: actions/checkout@v2 uses: actions/checkout@v3
- name: Prepare test - name: Prepare test
id: prepare-test id: prepare-test
uses: ./.github/prepare-test uses: ./.github/prepare-test

View File

@@ -12,6 +12,7 @@ on:
branches: branches:
- main - main
- v1 - v1
- v2
pull_request: pull_request:
types: types:
- opened - opened
@@ -31,7 +32,7 @@ jobs:
runs-on: ${{ matrix.os }} runs-on: ${{ matrix.os }}
steps: steps:
- name: Check out repository - name: Check out repository
uses: actions/checkout@v2 uses: actions/checkout@v3
- name: Prepare test - name: Prepare test
id: prepare-test id: prepare-test
uses: ./.github/prepare-test uses: ./.github/prepare-test

3
.github/workflows/__test-proxy.yml generated vendored
View File

@@ -12,6 +12,7 @@ on:
branches: branches:
- main - main
- v1 - v1
- v2
pull_request: pull_request:
types: types:
- opened - opened
@@ -31,7 +32,7 @@ jobs:
runs-on: ${{ matrix.os }} runs-on: ${{ matrix.os }}
steps: steps:
- name: Check out repository - name: Check out repository
uses: actions/checkout@v2 uses: actions/checkout@v3
- name: Prepare test - name: Prepare test
id: prepare-test id: prepare-test
uses: ./.github/prepare-test uses: ./.github/prepare-test

3
.github/workflows/__test-ruby.yml generated vendored
View File

@@ -12,6 +12,7 @@ on:
branches: branches:
- main - main
- v1 - v1
- v2
pull_request: pull_request:
types: types:
- opened - opened
@@ -41,7 +42,7 @@ jobs:
runs-on: ${{ matrix.os }} runs-on: ${{ matrix.os }}
steps: steps:
- name: Check out repository - name: Check out repository
uses: actions/checkout@v2 uses: actions/checkout@v3
- name: Prepare test - name: Prepare test
id: prepare-test id: prepare-test
uses: ./.github/prepare-test uses: ./.github/prepare-test

View File

@@ -12,6 +12,7 @@ on:
branches: branches:
- main - main
- v1 - v1
- v2
pull_request: pull_request:
types: types:
- opened - opened
@@ -41,7 +42,7 @@ jobs:
runs-on: ${{ matrix.os }} runs-on: ${{ matrix.os }}
steps: steps:
- name: Check out repository - name: Check out repository
uses: actions/checkout@v2 uses: actions/checkout@v3
- name: Prepare test - name: Prepare test
id: prepare-test id: prepare-test
uses: ./.github/prepare-test uses: ./.github/prepare-test

View File

@@ -12,6 +12,7 @@ on:
branches: branches:
- main - main
- v1 - v1
- v2
pull_request: pull_request:
types: types:
- opened - opened
@@ -69,7 +70,7 @@ jobs:
runs-on: ${{ matrix.os }} runs-on: ${{ matrix.os }}
steps: steps:
- name: Check out repository - name: Check out repository
uses: actions/checkout@v2 uses: actions/checkout@v3
- name: Prepare test - name: Prepare test
id: prepare-test id: prepare-test
uses: ./.github/prepare-test uses: ./.github/prepare-test

View File

@@ -12,6 +12,7 @@ on:
branches: branches:
- main - main
- v1 - v1
- v2
pull_request: pull_request:
types: types:
- opened - opened
@@ -69,13 +70,13 @@ jobs:
runs-on: ${{ matrix.os }} runs-on: ${{ matrix.os }}
steps: steps:
- name: Check out repository - name: Check out repository
uses: actions/checkout@v2 uses: actions/checkout@v3
- name: Prepare test - name: Prepare test
id: prepare-test id: prepare-test
uses: ./.github/prepare-test uses: ./.github/prepare-test
with: with:
version: ${{ matrix.version }} version: ${{ matrix.version }}
- uses: actions/checkout@v2 - uses: actions/checkout@v3
with: with:
ref: 474bbf07f9247ffe1856c6a0f94aeeb10e7afee6 ref: 474bbf07f9247ffe1856c6a0f94aeeb10e7afee6
path: x/y/z/some-path path: x/y/z/some-path

View File

@@ -15,7 +15,7 @@ jobs:
steps: steps:
- name: Checkout CodeQL Action - name: Checkout CodeQL Action
uses: actions/checkout@v2 uses: actions/checkout@v3
- name: Check Expected Release Files - name: Check Expected Release Files
run: | run: |
bundle_version="$(cat "./src/defaults.json" | jq -r ".bundleVersion")" bundle_version="$(cat "./src/defaults.json" | jq -r ".bundleVersion")"

View File

@@ -20,7 +20,7 @@ jobs:
security-events: write security-events: write
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v3
- name: Init with default CodeQL bundle from the VM image - name: Init with default CodeQL bundle from the VM image
id: init-default id: init-default
uses: ./init uses: ./init
@@ -75,7 +75,7 @@ jobs:
security-events: write security-events: write
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v3
- uses: ./init - uses: ./init
id: init id: init
with: with:

View File

@@ -34,8 +34,8 @@ jobs:
GITHUB_CONTEXT: '${{ toJson(github) }}' GITHUB_CONTEXT: '${{ toJson(github) }}'
run: echo "$GITHUB_CONTEXT" run: echo "$GITHUB_CONTEXT"
- uses: actions/checkout@v2 - uses: actions/checkout@v3
- uses: actions/setup-node@v2 - uses: actions/setup-node@v3
- name: Update git config - name: Update git config
run: | run: |

View File

@@ -2,7 +2,7 @@ name: PR Checks (Basic Checks and Runner)
on: on:
push: push:
branches: [main, v1] branches: [main, v1, v2]
pull_request: pull_request:
# Run checks on reopened draft PRs to support triggering PR checks on draft PRs that were opened # Run checks on reopened draft PRs to support triggering PR checks on draft PRs that were opened
# by other workflows. # by other workflows.
@@ -16,7 +16,7 @@ jobs:
timeout-minutes: 45 timeout-minutes: 45
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v3
- name: Run Lint - name: Run Lint
run: npm run-script lint run: npm run-script lint
@@ -30,7 +30,7 @@ jobs:
node-types-version: [12.12, current] node-types-version: [12.12, current]
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v3
- name: Update version of @types/node - name: Update version of @types/node
if: matrix.node-types-version != 'current' if: matrix.node-types-version != 'current'
@@ -46,11 +46,13 @@ jobs:
# `npm install` on Linux. # `npm install` on Linux.
npm install npm install
git config --global user.email "github-actions@github.com" if [ ! -z "$(git status --porcelain)" ]; then
git config --global user.name "github-actions[bot]" git config --global user.email "github-actions@github.com"
# The period in `git add --all .` ensures that we stage deleted files too. git config --global user.name "github-actions[bot]"
git add --all . # The period in `git add --all .` ensures that we stage deleted files too.
git commit -m "Use @types/node=${NODE_TYPES_VERSION}" git add --all .
git commit -m "Use @types/node=${NODE_TYPES_VERSION}"
fi
- name: Check generated JS - name: Check generated JS
run: .github/workflows/script/check-js.sh run: .github/workflows/script/check-js.sh
@@ -61,7 +63,7 @@ jobs:
timeout-minutes: 45 timeout-minutes: 45
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v3
- name: Check node modules up to date - name: Check node modules up to date
run: .github/workflows/script/check-node-modules.sh run: .github/workflows/script/check-node-modules.sh
@@ -71,9 +73,9 @@ jobs:
timeout-minutes: 45 timeout-minutes: 45
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v3
- name: Set up Python - name: Set up Python
uses: actions/setup-python@v2 uses: actions/setup-python@v3
with: with:
python-version: 3.8 python-version: 3.8
- name: Install dependencies - name: Install dependencies
@@ -93,7 +95,7 @@ jobs:
timeout-minutes: 45 timeout-minutes: 45
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v3
- name: npm run-script test - name: npm run-script test
run: npm run-script test run: npm run-script test
@@ -104,7 +106,7 @@ jobs:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v3
- name: Build runner - name: Build runner
run: | run: |
@@ -133,7 +135,7 @@ jobs:
runs-on: windows-latest runs-on: windows-latest
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v3
- name: Build runner - name: Build runner
run: | run: |
@@ -158,7 +160,7 @@ jobs:
runs-on: macos-latest runs-on: macos-latest
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v3
- name: Build runner - name: Build runner
run: | run: |
@@ -183,7 +185,7 @@ jobs:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v3
- name: Move codeql-action - name: Move codeql-action
shell: bash shell: bash
@@ -223,7 +225,7 @@ jobs:
runs-on: windows-2019 runs-on: windows-2019
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v3
- name: Move codeql-action - name: Move codeql-action
shell: bash shell: bash
@@ -251,7 +253,7 @@ jobs:
& $Env:CODEQL_RUNNER dotnet build /p:UseSharedCompilation=false & $Env:CODEQL_RUNNER dotnet build /p:UseSharedCompilation=false
- name: Upload tracer logs - name: Upload tracer logs
uses: actions/upload-artifact@v2 uses: actions/upload-artifact@v3
with: with:
name: tracer-logs name: tracer-logs
path: ./codeql-runner/compound-build-tracer.log path: ./codeql-runner/compound-build-tracer.log
@@ -269,7 +271,7 @@ jobs:
runs-on: macos-latest runs-on: macos-latest
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v3
- name: Move codeql-action - name: Move codeql-action
shell: bash shell: bash
@@ -308,7 +310,7 @@ jobs:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v3
- name: Move codeql-action - name: Move codeql-action
shell: bash shell: bash
@@ -347,7 +349,7 @@ jobs:
runs-on: windows-2019 runs-on: windows-2019
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v3
- name: Move codeql-action - name: Move codeql-action
shell: bash shell: bash
@@ -385,7 +387,7 @@ jobs:
timeout-minutes: 45 timeout-minutes: 45
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v3
- name: Move codeql-action - name: Move codeql-action
shell: bash shell: bash
@@ -425,7 +427,7 @@ jobs:
if: ${{ github.event_name != 'pull_request' || github.event.pull_request.base.repo.id == github.event.pull_request.head.repo.id }} if: ${{ github.event_name != 'pull_request' || github.event.pull_request.base.repo.id == github.event.pull_request.head.repo.id }}
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v3
- name: Build runner - name: Build runner
run: | run: |
@@ -446,7 +448,7 @@ jobs:
timeout-minutes: 45 timeout-minutes: 45
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v3
- name: Build runner - name: Build runner
run: | run: |

View File

@@ -2,7 +2,7 @@ name: Test Python Package Installation on Linux and Mac
on: on:
push: push:
branches: [main, v1] branches: [main, v1, v2]
pull_request: pull_request:
# Run checks on reopened draft PRs to support triggering PR checks on draft PRs that were opened # Run checks on reopened draft PRs to support triggering PR checks on draft PRs that were opened
# by other workflows. # by other workflows.
@@ -25,7 +25,7 @@ jobs:
steps: steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@v2 - uses: actions/checkout@v3
- name: Initialize CodeQL - name: Initialize CodeQL
uses: ./init uses: ./init
@@ -71,7 +71,7 @@ jobs:
steps: steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@v2 - uses: actions/checkout@v3
- name: Initialize CodeQL - name: Initialize CodeQL
uses: ./init uses: ./init
@@ -122,9 +122,9 @@ jobs:
steps: steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@v2 - uses: actions/checkout@v3
- uses: actions/setup-python@v2 - uses: actions/setup-python@v3
with: with:
python-version: ${{ matrix.python_version }} python-version: ${{ matrix.python_version }}

View File

@@ -1,55 +0,0 @@
name: Release runner
on:
workflow_dispatch:
inputs:
bundle-tag:
description: 'Tag of the bundle release (e.g., "codeql-bundle-20200826")'
required: false
jobs:
release-runner:
timeout-minutes: 45
runs-on: ubuntu-latest
env:
RELEASE_TAG: "${{ github.event.inputs.bundle-tag }}"
strategy:
matrix:
extension: ["linux", "macos", "win.exe"]
steps:
- uses: actions/checkout@v2
- name: Build runner
run: |
cd runner
npm install
npm run build-runner
- uses: actions/upload-artifact@v2
with:
name: codeql-runner-${{matrix.extension}}
path: runner/dist/codeql-runner-${{matrix.extension}}
- name: Resolve Upload URL for the release
if: ${{ github.event.inputs.bundle-tag != null }}
id: save_url
run: |
UPLOAD_URL=$(curl -sS \
"https://api.github.com/repos/${GITHUB_REPOSITORY}/releases/tags/${RELEASE_TAG}" \
-H "Accept: application/json" \
-H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" | jq .upload_url | sed s/\"//g)
echo ${UPLOAD_URL}
echo "::set-output name=upload_url::${UPLOAD_URL}"
- name: Upload Platform Package
if: ${{ github.event.inputs.bundle-tag != null }}
uses: actions/upload-release-asset@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.save_url.outputs.upload_url }}
asset_path: runner/dist/codeql-runner-${{matrix.extension}}
asset_name: codeql-runner-${{matrix.extension}}
asset_content_type: application/octet-stream

View File

@@ -1,74 +0,0 @@
#
# Split the CodeQL Bundle into platform bundles
#
# Instructions:
# 1. Upload the new codeql-bundle (codeql-bundle.tar.gz) as an asset of the
# release (codeql-bundle-20200826)
# 2. Take note of the CLI Release used by the bundle (e.g., v2.2.5)
# 3. Manually launch this workflow file (via the Actions UI) specifying
# - The CLI Release (e.g., v2.2.5)
# - The release tag (e.g., codeql-bundle-20200826)
# 4. If everything succeeds you should see 3 new assets.
#
name: Split Bundle
on:
workflow_dispatch:
inputs:
cli-release:
description: 'CodeQL CLI Release (e.g., "v2.2.5")'
required: true
bundle-tag:
description: 'Tag of the bundle release (e.g., "codeql-bundle-20200826")'
required: true
jobs:
build:
runs-on: ubuntu-latest
timeout-minutes: 45
env:
CLI_RELEASE: "${{ github.event.inputs.cli-release }}"
RELEASE_TAG: "${{ github.event.inputs.bundle-tag }}"
strategy:
fail-fast: false
matrix:
platform: ["linux64", "osx64", "win64"]
steps:
- name: Resolve Upload URL for the release
id: save_url
run: |
UPLOAD_URL=$(curl -sS \
"https://api.github.com/repos/${GITHUB_REPOSITORY}/releases/tags/${RELEASE_TAG}" \
-H "Accept: application/json" \
-H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" | jq .upload_url | sed s/\"//g)
echo ${UPLOAD_URL}
echo "::set-output name=upload_url::${UPLOAD_URL}"
- name: Download CodeQL CLI and Bundle
run: |
wget --no-verbose "https://github.com/${GITHUB_REPOSITORY}/releases/download/${RELEASE_TAG}/codeql-bundle.tar.gz"
wget --no-verbose "https://github.com/github/codeql-cli-binaries/releases/download/${CLI_RELEASE}/codeql-${{matrix.platform}}.zip"
- name: Create Platform Package
# Replace the codeql-binaries with the platform specific ones
run: |
gunzip codeql-bundle.tar.gz
tar -f codeql-bundle.tar --delete codeql
unzip -q codeql-${{matrix.platform}}.zip
tar -f codeql-bundle.tar --append codeql
gzip codeql-bundle.tar
mv codeql-bundle.tar.gz codeql-bundle-${{matrix.platform}}.tar.gz
du -sh codeql-bundle-${{matrix.platform}}.tar.gz
- name: Upload Platform Package
uses: actions/upload-release-asset@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.save_url.outputs.upload_url }}
asset_path: ./codeql-bundle-${{matrix.platform}}.tar.gz
asset_name: codeql-bundle-${{matrix.platform}}.tar.gz
asset_content_type: application/tar+gzip

View File

@@ -11,7 +11,7 @@ jobs:
if: contains(github.event.pull_request.labels.*.name, 'Update dependencies') && (github.event.pull_request.head.repo.full_name == 'github/codeql-action') if: contains(github.event.pull_request.labels.*.name, 'Update dependencies') && (github.event.pull_request.head.repo.full_name == 'github/codeql-action')
steps: steps:
- name: Checkout repository - name: Checkout repository
uses: actions/checkout@v2 uses: actions/checkout@v3
- name: Remove PR label - name: Remove PR label
env: env:

View File

@@ -23,13 +23,13 @@ jobs:
GITHUB_CONTEXT: '${{ toJson(github) }}' GITHUB_CONTEXT: '${{ toJson(github) }}'
run: echo "$GITHUB_CONTEXT" run: echo "$GITHUB_CONTEXT"
- uses: actions/checkout@v2 - uses: actions/checkout@v3
with: with:
# Need full history so we calculate diffs # Need full history so we calculate diffs
fetch-depth: 0 fetch-depth: 0
- name: Set up Python - name: Set up Python
uses: actions/setup-python@v2 uses: actions/setup-python@v3
with: with:
python-version: 3.8 python-version: 3.8

View File

@@ -13,13 +13,13 @@ jobs:
steps: steps:
- name: Setup Python - name: Setup Python
uses: actions/setup-python@v2 uses: actions/setup-python@v3
with: with:
python-version: "3.7" python-version: "3.7"
- name: Checkout CodeQL Action - name: Checkout CodeQL Action
uses: actions/checkout@v2 uses: actions/checkout@v3
- name: Checkout Enterprise Releases - name: Checkout Enterprise Releases
uses: actions/checkout@v2 uses: actions/checkout@v3
with: with:
repository: github/enterprise-releases repository: github/enterprise-releases
ssh-key: ${{ secrets.ENTERPRISE_RELEASES_SSH_KEY }} ssh-key: ${{ secrets.ENTERPRISE_RELEASES_SSH_KEY }}

View File

@@ -1,5 +1,9 @@
# CodeQL Action Changelog # CodeQL Action Changelog
## [UNRELEASED]
No user facing changes.
## 2.1.6 - 30 Mar 2022 ## 2.1.6 - 30 Mar 2022
- [v2+ only] The CodeQL Action now runs on Node.js v16. [#1000](https://github.com/github/codeql-action/pull/1000) - [v2+ only] The CodeQL Action now runs on Node.js v16. [#1000](https://github.com/github/codeql-action/pull/1000)

View File

@@ -52,11 +52,11 @@ jobs:
steps: steps:
- name: Checkout repository - name: Checkout repository
uses: actions/checkout@v2 uses: actions/checkout@v3
# Initializes the CodeQL tools for scanning. # Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL - name: Initialize CodeQL
uses: github/codeql-action/init@v1 uses: github/codeql-action/init@v2
# Override language selection by uncommenting this and choosing your languages # Override language selection by uncommenting this and choosing your languages
# with: # with:
# languages: go, javascript, csharp, python, cpp, java # languages: go, javascript, csharp, python, cpp, java
@@ -64,7 +64,7 @@ jobs:
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
# If this step fails, then you should remove it and run the build manually (see below). # If this step fails, then you should remove it and run the build manually (see below).
- name: Autobuild - name: Autobuild
uses: github/codeql-action/autobuild@v1 uses: github/codeql-action/autobuild@v2
# Command-line programs to run using the OS shell. # Command-line programs to run using the OS shell.
# 📚 https://git.io/JvXDl # 📚 https://git.io/JvXDl
@@ -78,14 +78,14 @@ jobs:
# make release # make release
- name: Perform CodeQL Analysis - name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v1 uses: github/codeql-action/analyze@v2
``` ```
If you prefer to integrate this within an existing CI workflow, it should end up looking something like this: If you prefer to integrate this within an existing CI workflow, it should end up looking something like this:
```yaml ```yaml
- name: Initialize CodeQL - name: Initialize CodeQL
uses: github/codeql-action/init@v1 uses: github/codeql-action/init@v2
with: with:
languages: go, javascript languages: go, javascript
@@ -95,7 +95,7 @@ If you prefer to integrate this within an existing CI workflow, it should end up
make release make release
- name: Perform CodeQL Analysis - name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v1 uses: github/codeql-action/analyze@v2
``` ```
### Configuration file ### Configuration file
@@ -103,7 +103,7 @@ If you prefer to integrate this within an existing CI workflow, it should end up
Use the `config-file` parameter of the `init` action to enable the configuration file. The value of `config-file` is the path to the configuration file you want to use. This example loads the configuration file `./.github/codeql/codeql-config.yml`. Use the `config-file` parameter of the `init` action to enable the configuration file. The value of `config-file` is the path to the configuration file you want to use. This example loads the configuration file `./.github/codeql/codeql-config.yml`.
```yaml ```yaml
- uses: github/codeql-action/init@v1 - uses: github/codeql-action/init@v2
with: with:
config-file: ./.github/codeql/codeql-config.yml config-file: ./.github/codeql/codeql-config.yml
``` ```
@@ -111,7 +111,7 @@ Use the `config-file` parameter of the `init` action to enable the configuration
The configuration file can be located in a different repository. This is useful if you want to share the same configuration across multiple repositories. If the configuration file is in a private repository you can also specify an `external-repository-token` option. This should be a personal access token that has read access to any repositories containing referenced config files and queries. The configuration file can be located in a different repository. This is useful if you want to share the same configuration across multiple repositories. If the configuration file is in a private repository you can also specify an `external-repository-token` option. This should be a personal access token that has read access to any repositories containing referenced config files and queries.
```yaml ```yaml
- uses: github/codeql-action/init@v1 - uses: github/codeql-action/init@v2
with: with:
config-file: owner/repo/codeql-config.yml@branch config-file: owner/repo/codeql-config.yml@branch
external-repository-token: ${{ secrets.EXTERNAL_REPOSITORY_TOKEN }} external-repository-token: ${{ secrets.EXTERNAL_REPOSITORY_TOKEN }}
@@ -122,7 +122,7 @@ For information on how to write a configuration file, see "[Using a custom confi
If you only want to customise the queries used, you can specify them in your workflow instead of creating a config file, using the `queries` property of the `init` action: If you only want to customise the queries used, you can specify them in your workflow instead of creating a config file, using the `queries` property of the `init` action:
```yaml ```yaml
- uses: github/codeql-action/init@v1 - uses: github/codeql-action/init@v2
with: with:
queries: <local-or-remote-query>,<another-query> queries: <local-or-remote-query>,<another-query>
``` ```
@@ -130,7 +130,7 @@ If you only want to customise the queries used, you can specify them in your wor
By default, this will override any queries specified in a config file. If you wish to use both sets of queries, prefix the list of queries in the workflow with `+`: By default, this will override any queries specified in a config file. If you wish to use both sets of queries, prefix the list of queries in the workflow with `+`:
```yaml ```yaml
- uses: github/codeql-action/init@v1 - uses: github/codeql-action/init@v2
with: with:
queries: +<local-or-remote-query>,<another-query> queries: +<local-or-remote-query>,<another-query>
``` ```

2
lib/actions-util.js generated
View File

@@ -455,7 +455,7 @@ async function getRef() {
return ref; return ref;
} }
const head = await (0, exports.getCommitOid)(checkoutPath, "HEAD"); const head = await (0, exports.getCommitOid)(checkoutPath, "HEAD");
// in actions/checkout@v2 we can check if git rev-parse HEAD == GITHUB_SHA // in actions/checkout@v2+ we can check if git rev-parse HEAD == GITHUB_SHA
// in actions/checkout@v1 this may not be true as it checks out the repository // in actions/checkout@v1 this may not be true as it checks out the repository
// using GITHUB_REF. There is a subtle race condition where // using GITHUB_REF. There is a subtle race condition where
// git rev-parse GITHUB_REF != GITHUB_SHA, so we must check // git rev-parse GITHUB_REF != GITHUB_SHA, so we must check

File diff suppressed because one or more lines are too long

5
lib/codeql.js generated
View File

@@ -474,9 +474,8 @@ async function getCodeQLForCmd(cmd, checkVersion) {
if (config.injectedMlQueries) { if (config.injectedMlQueries) {
// We need to inject the ML queries into the original user input before // We need to inject the ML queries into the original user input before
// we pass this on to the CLI, to make sure these get run. // we pass this on to the CLI, to make sure these get run.
let packString = util_1.ML_POWERED_JS_QUERIES_PACK.packName; const pack = await util.getMlPoweredJsQueriesPack(codeql);
if (util_1.ML_POWERED_JS_QUERIES_PACK.version) const packString = pack.packName + (pack.version ? `@${pack.version}` : "");
packString = `${packString}@${util_1.ML_POWERED_JS_QUERIES_PACK.version}`;
if (augmentedConfig.packs === undefined) if (augmentedConfig.packs === undefined)
augmentedConfig.packs = []; augmentedConfig.packs = [];
if (Array.isArray(augmentedConfig.packs)) { if (Array.isArray(augmentedConfig.packs)) {

File diff suppressed because one or more lines are too long

9
lib/config-utils.js generated
View File

@@ -130,15 +130,18 @@ async function addBuiltinSuiteQueries(languages, codeQL, resultMap, packs, suite
// If we're running the JavaScript security-extended analysis (or a superset of it), the repo is // If we're running the JavaScript security-extended analysis (or a superset of it), the repo is
// opted into the ML-powered queries beta, and a user hasn't already added the ML-powered query // opted into the ML-powered queries beta, and a user hasn't already added the ML-powered query
// pack, then add the ML-powered query pack so that we run ML-powered queries. // pack, then add the ML-powered query pack so that we run ML-powered queries.
if (languages.includes("javascript") && if (
// Disable ML-powered queries on Windows
process.platform !== "win32" &&
languages.includes("javascript") &&
(found === "security-extended" || found === "security-and-quality") && (found === "security-extended" || found === "security-and-quality") &&
!((_a = packs.javascript) === null || _a === void 0 ? void 0 : _a.some((pack) => pack.packName === util_1.ML_POWERED_JS_QUERIES_PACK.packName)) && !((_a = packs.javascript) === null || _a === void 0 ? void 0 : _a.some((pack) => pack.packName === util_1.ML_POWERED_JS_QUERIES_PACK_NAME)) &&
(await featureFlags.getValue(feature_flags_1.FeatureFlag.MlPoweredQueriesEnabled)) && (await featureFlags.getValue(feature_flags_1.FeatureFlag.MlPoweredQueriesEnabled)) &&
(await (0, util_1.codeQlVersionAbove)(codeQL, codeql_1.CODEQL_VERSION_ML_POWERED_QUERIES))) { (await (0, util_1.codeQlVersionAbove)(codeQL, codeql_1.CODEQL_VERSION_ML_POWERED_QUERIES))) {
if (!packs.javascript) { if (!packs.javascript) {
packs.javascript = []; packs.javascript = [];
} }
packs.javascript.push(util_1.ML_POWERED_JS_QUERIES_PACK); packs.javascript.push(await (0, util_1.getMlPoweredJsQueriesPack)(codeQL));
injectedMlQueries = true; injectedMlQueries = true;
} }
const suites = languages.map((l) => `${l}-${suiteName}.qls`); const suites = languages.map((l) => `${l}-${suiteName}.qls`);

File diff suppressed because one or more lines are too long

View File

@@ -911,11 +911,20 @@ const mlPoweredQueriesMacro = ava_1.default.macro({
? `${expectedVersionString} are` ? `${expectedVersionString} are`
: "aren't"} loaded for packs: ${packsInput}, queries: ${queriesInput} using CLI v${codeQLVersion} when feature flag is ${isMlPoweredQueriesFlagEnabled ? "enabled" : "disabled"}`, : "aren't"} loaded for packs: ${packsInput}, queries: ${queriesInput} using CLI v${codeQLVersion} when feature flag is ${isMlPoweredQueriesFlagEnabled ? "enabled" : "disabled"}`,
}); });
// macro, isMlPoweredQueriesFlagEnabled, packsInput, queriesInput, versionString // macro, codeQLVersion, isMlPoweredQueriesFlagEnabled, packsInput, queriesInput, expectedVersionString
// Test that ML-powered queries aren't run on v2.7.4 of the CLI.
(0, ava_1.default)(mlPoweredQueriesMacro, "2.7.4", true, undefined, "security-extended", undefined); (0, ava_1.default)(mlPoweredQueriesMacro, "2.7.4", true, undefined, "security-extended", undefined);
// Test that ML-powered queries aren't run when the feature flag is off.
(0, ava_1.default)(mlPoweredQueriesMacro, "2.7.5", false, undefined, "security-extended", undefined); (0, ava_1.default)(mlPoweredQueriesMacro, "2.7.5", false, undefined, "security-extended", undefined);
// Test that ML-powered queries aren't run when the user hasn't specified that we should run the
// `security-extended` or `security-and-quality` query suite.
(0, ava_1.default)(mlPoweredQueriesMacro, "2.7.5", true, undefined, undefined, undefined); (0, ava_1.default)(mlPoweredQueriesMacro, "2.7.5", true, undefined, undefined, undefined);
(0, ava_1.default)(mlPoweredQueriesMacro, "2.7.5", true, undefined, "security-extended", "~0.1.0"); // Test that ML-powered queries are run on non-Windows platforms running `security-extended`.
(0, ava_1.default)(mlPoweredQueriesMacro, "2.7.5", true, undefined, "security-and-quality", "~0.1.0"); (0, ava_1.default)(mlPoweredQueriesMacro, "2.7.5", true, undefined, "security-extended", process.platform === "win32" ? undefined : "~0.1.0");
(0, ava_1.default)(mlPoweredQueriesMacro, "2.7.5", true, "codeql/javascript-experimental-atm-queries@0.0.1", "security-and-quality", "0.0.1"); // Test that ML-powered queries are run on non-Windows platforms running `security-and-quality`.
(0, ava_1.default)(mlPoweredQueriesMacro, "2.7.5", true, undefined, "security-and-quality", process.platform === "win32" ? undefined : "~0.1.0");
// Test that we don't inject an ML-powered query pack if the user has already specified one.
(0, ava_1.default)(mlPoweredQueriesMacro, "2.7.5", true, "codeql/javascript-experimental-atm-queries@0.0.1", "security-and-quality", process.platform === "win32" ? undefined : "0.0.1");
// Test that the ~0.2.0 version of ML-powered queries is run on v2.8.4 of the CLI.
(0, ava_1.default)(mlPoweredQueriesMacro, "2.8.4", true, undefined, "security-extended", process.platform === "win32" ? undefined : "~0.2.0");
//# sourceMappingURL=config-utils.test.js.map //# sourceMappingURL=config-utils.test.js.map

File diff suppressed because one or more lines are too long

55
lib/util.js generated
View File

@@ -22,7 +22,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod }; return (mod && mod.__esModule) ? mod : { "default": mod };
}; };
Object.defineProperty(exports, "__esModule", { value: true }); Object.defineProperty(exports, "__esModule", { value: true });
exports.getMlPoweredJsQueriesStatus = exports.ML_POWERED_JS_QUERIES_PACK = exports.isGoodVersion = exports.delay = exports.bundleDb = exports.codeQlVersionAbove = exports.getCachedCodeQlVersion = exports.cacheCodeQlVersion = exports.isGitHubGhesVersionBelow = exports.isHTTPError = exports.UserError = exports.HTTPError = exports.getRequiredEnvParam = exports.isActions = exports.getMode = exports.enrichEnvironment = exports.initializeEnvironment = exports.Mode = exports.assertNever = exports.getGitHubAuth = exports.apiVersionInRange = exports.DisallowedAPIVersionReason = exports.checkGitHubVersionInRange = exports.getGitHubVersion = exports.GitHubVariant = exports.parseGitHubUrl = exports.getCodeQLDatabasePath = exports.getThreadsFlag = exports.getThreadsFlagValue = exports.getAddSnippetsFlag = exports.getMemoryFlag = exports.getMemoryFlagValue = exports.withTmpDir = exports.getToolNames = exports.getExtraOptionsEnvParam = exports.DEFAULT_DEBUG_DATABASE_NAME = exports.DEFAULT_DEBUG_ARTIFACT_NAME = exports.GITHUB_DOTCOM_URL = void 0; exports.getMlPoweredJsQueriesStatus = exports.getMlPoweredJsQueriesPack = exports.ML_POWERED_JS_QUERIES_PACK_NAME = exports.isGoodVersion = exports.delay = exports.bundleDb = exports.codeQlVersionAbove = exports.getCachedCodeQlVersion = exports.cacheCodeQlVersion = exports.isGitHubGhesVersionBelow = exports.isHTTPError = exports.UserError = exports.HTTPError = exports.getRequiredEnvParam = exports.isActions = exports.getMode = exports.enrichEnvironment = exports.initializeEnvironment = exports.Mode = exports.assertNever = exports.getGitHubAuth = exports.apiVersionInRange = exports.DisallowedAPIVersionReason = exports.checkGitHubVersionInRange = exports.getGitHubVersion = exports.GitHubVariant = exports.parseGitHubUrl = exports.getCodeQLDatabasePath = exports.getThreadsFlag = exports.getThreadsFlagValue = exports.getAddSnippetsFlag = exports.getMemoryFlag = exports.getMemoryFlagValue = exports.withTmpDir = exports.getToolNames = exports.getExtraOptionsEnvParam = exports.DEFAULT_DEBUG_DATABASE_NAME = exports.DEFAULT_DEBUG_ARTIFACT_NAME = exports.GITHUB_DOTCOM_URL = void 0;
const fs = __importStar(require("fs")); const fs = __importStar(require("fs"));
const os = __importStar(require("os")); const os = __importStar(require("os"));
const path = __importStar(require("path")); const path = __importStar(require("path"));
@@ -545,24 +545,26 @@ function isGoodVersion(versionSpec) {
return !BROKEN_VERSIONS.includes(versionSpec); return !BROKEN_VERSIONS.includes(versionSpec);
} }
exports.isGoodVersion = isGoodVersion; exports.isGoodVersion = isGoodVersion;
exports.ML_POWERED_JS_QUERIES_PACK_NAME = "codeql/javascript-experimental-atm-queries";
/** /**
* The ML-powered JS query pack to add to the analysis if a repo is opted into the ML-powered * Gets the ML-powered JS query pack to add to the analysis if a repo is opted into the ML-powered
* queries beta. * queries beta.
*/ */
exports.ML_POWERED_JS_QUERIES_PACK = { async function getMlPoweredJsQueriesPack(codeQL) {
packName: "codeql/javascript-experimental-atm-queries", if (await codeQlVersionAbove(codeQL, "2.8.4")) {
version: "~0.1.0", return { packName: exports.ML_POWERED_JS_QUERIES_PACK_NAME, version: "~0.2.0" };
}; }
return { packName: exports.ML_POWERED_JS_QUERIES_PACK_NAME, version: "~0.1.0" };
}
exports.getMlPoweredJsQueriesPack = getMlPoweredJsQueriesPack;
/** /**
* Get information about ML-powered JS queries to populate status reports with. * Get information about ML-powered JS queries to populate status reports with.
* *
* This will be: * This will be:
* *
* - The version string if the analysis is using the ML-powered query pack that will be added to the * - The version string if the analysis is using a single version of the ML-powered query pack.
* analysis if the repo is opted into the ML-powered queries beta, i.e. * - "latest" if the version string of the ML-powered query pack is undefined. This is unlikely to
* {@link ML_POWERED_JS_QUERIES_PACK.version}. If the version string * occur in practice (see comment below).
* {@link ML_POWERED_JS_QUERIES_PACK.version} is undefined, then the status report string will be
* "latest", however this shouldn't occur in practice (see comment below).
* - "false" if the analysis won't run any ML-powered JS queries. * - "false" if the analysis won't run any ML-powered JS queries.
* - "other" in all other cases. * - "other" in all other cases.
* *
@@ -572,30 +574,25 @@ exports.ML_POWERED_JS_QUERIES_PACK = {
* version of the CodeQL Action. For instance, we might want to compare the `~0.1.0` and `~0.0.2` * version of the CodeQL Action. For instance, we might want to compare the `~0.1.0` and `~0.0.2`
* version strings. * version strings.
* *
* We restrict the set of strings we report here by excluding other version strings and combinations
* of version strings. We do this to limit the cardinality of the ML-powered JS queries status
* report field, since some platforms that ingest this status report bill based on the cardinality
* of its fields.
*
* This function lives here rather than in `init-action.ts` so it's easier to test, since tests for * This function lives here rather than in `init-action.ts` so it's easier to test, since tests for
* `init-action.ts` would each need to live in their own file. See `analyze-action-env.ts` for an * `init-action.ts` would each need to live in their own file. See `analyze-action-env.ts` for an
* explanation as to why this is. * explanation as to why this is.
*/ */
function getMlPoweredJsQueriesStatus(config) { function getMlPoweredJsQueriesStatus(config) {
const mlPoweredJsQueryPacks = (config.packs.javascript || []).filter((pack) => pack.packName === exports.ML_POWERED_JS_QUERIES_PACK.packName); const mlPoweredJsQueryPacks = (config.packs.javascript || []).filter((pack) => pack.packName === exports.ML_POWERED_JS_QUERIES_PACK_NAME);
if (mlPoweredJsQueryPacks.length === 0) { switch (mlPoweredJsQueryPacks.length) {
return "false"; case 1:
// We should always specify an explicit version string in `getMlPoweredJsQueriesPack`,
// otherwise we won't be able to make changes to the pack unless those changes are compatible
// with each version of the CodeQL Action. Therefore in practice we should only hit the
// `latest` case here when customers have explicitly added the ML-powered query pack to their
// CodeQL config.
return mlPoweredJsQueryPacks[0].version || "latest";
case 0:
return "false";
default:
return "other";
} }
const firstVersionString = mlPoweredJsQueryPacks[0].version;
if (mlPoweredJsQueryPacks.length === 1 &&
exports.ML_POWERED_JS_QUERIES_PACK.version === firstVersionString) {
// We should always specify an explicit version string in `ML_POWERED_JS_QUERIES_PACK`,
// otherwise we won't be able to make changes to the pack unless those changes are compatible
// with each version of the CodeQL Action. Therefore in practice, we should never hit the
// `latest` case here.
return exports.ML_POWERED_JS_QUERIES_PACK.version || "latest";
}
return "other";
} }
exports.getMlPoweredJsQueriesStatus = getMlPoweredJsQueriesStatus; exports.getMlPoweredJsQueriesStatus = getMlPoweredJsQueriesStatus;
//# sourceMappingURL=util.js.map //# sourceMappingURL=util.js.map

File diff suppressed because one or more lines are too long

43
lib/util.test.js generated
View File

@@ -205,32 +205,43 @@ async function mockStdInForAuthExpectError(t, mockLogger, ...text) {
await t.throwsAsync(async () => util.getGitHubAuth(mockLogger, undefined, true, stdin)); await t.throwsAsync(async () => util.getGitHubAuth(mockLogger, undefined, true, stdin));
} }
const ML_POWERED_JS_STATUS_TESTS = [ const ML_POWERED_JS_STATUS_TESTS = [
// If no packs are loaded, status is false.
[[], "false"], [[], "false"],
// If another pack is loaded but not the ML-powered query pack, status is false.
[[{ packName: "someOtherPack" }], "false"], [[{ packName: "someOtherPack" }], "false"],
// If the ML-powered query pack is loaded with a specific version, status is that version.
[ [
[{ packName: "someOtherPack" }, util.ML_POWERED_JS_QUERIES_PACK], [{ packName: util.ML_POWERED_JS_QUERIES_PACK_NAME, version: "~0.1.0" }],
util.ML_POWERED_JS_QUERIES_PACK.version, "~0.1.0",
],
[[util.ML_POWERED_JS_QUERIES_PACK], util.ML_POWERED_JS_QUERIES_PACK.version],
[[{ packName: util.ML_POWERED_JS_QUERIES_PACK.packName }], "other"],
[
[{ packName: util.ML_POWERED_JS_QUERIES_PACK.packName, version: "~0.0.1" }],
"other",
],
[
[
{ packName: util.ML_POWERED_JS_QUERIES_PACK.packName, version: "0.0.1" },
{ packName: util.ML_POWERED_JS_QUERIES_PACK.packName, version: "0.0.2" },
],
"other",
], ],
// If the ML-powered query pack is loaded with a specific version and another pack is loaded, the
// status is the version of the ML-powered query pack.
[ [
[ [
{ packName: "someOtherPack" }, { packName: "someOtherPack" },
{ packName: util.ML_POWERED_JS_QUERIES_PACK.packName }, { packName: util.ML_POWERED_JS_QUERIES_PACK_NAME, version: "~0.1.0" },
],
"~0.1.0",
],
// If the ML-powered query pack is loaded without a version, the status is "latest".
[[{ packName: util.ML_POWERED_JS_QUERIES_PACK_NAME }], "latest"],
// If the ML-powered query pack is loaded with two different versions, the status is "other".
[
[
{ packName: util.ML_POWERED_JS_QUERIES_PACK_NAME, version: "0.0.1" },
{ packName: util.ML_POWERED_JS_QUERIES_PACK_NAME, version: "0.0.2" },
], ],
"other", "other",
], ],
// If the ML-powered query pack is loaded with no specific version, and another pack is loaded,
// the status is "latest".
[
[
{ packName: "someOtherPack" },
{ packName: util.ML_POWERED_JS_QUERIES_PACK_NAME },
],
"latest",
],
]; ];
for (const [packs, expectedStatus] of ML_POWERED_JS_STATUS_TESTS) { for (const [packs, expectedStatus] of ML_POWERED_JS_STATUS_TESTS) {
const packDescriptions = `[${packs const packDescriptions = `[${packs

File diff suppressed because one or more lines are too long

2
node_modules/.package-lock.json generated vendored
View File

@@ -1,6 +1,6 @@
{ {
"name": "codeql", "name": "codeql",
"version": "2.1.6", "version": "2.1.7",
"lockfileVersion": 2, "lockfileVersion": 2,
"requires": true, "requires": true,
"packages": { "packages": {

4
package-lock.json generated
View File

@@ -1,12 +1,12 @@
{ {
"name": "codeql", "name": "codeql",
"version": "2.1.6", "version": "2.1.7",
"lockfileVersion": 2, "lockfileVersion": 2,
"requires": true, "requires": true,
"packages": { "packages": {
"": { "": {
"name": "codeql", "name": "codeql",
"version": "2.1.6", "version": "2.1.7",
"license": "MIT", "license": "MIT",
"dependencies": { "dependencies": {
"@actions/artifact": "^1.0.0", "@actions/artifact": "^1.0.0",

View File

@@ -1,6 +1,6 @@
{ {
"name": "codeql", "name": "codeql",
"version": "2.1.6", "version": "2.1.7",
"private": true, "private": true,
"description": "CodeQL action", "description": "CodeQL action",
"scripts": { "scripts": {

View File

@@ -13,7 +13,7 @@ steps:
run: ./build.sh run: ./build.sh
- uses: ./../action/analyze - uses: ./../action/analyze
id: analysis id: analysis
- uses: actions/download-artifact@v2 - uses: actions/download-artifact@v3
with: with:
name: my-debug-artifacts-${{ matrix.os }}-${{ matrix.version }} name: my-debug-artifacts-${{ matrix.os }}-${{ matrix.version }}
- shell: bash - shell: bash

View File

@@ -1,7 +1,7 @@
name: "Go: Custom queries" name: "Go: Custom queries"
description: "Checks that Go works in conjunction with a config file specifying custom queries" description: "Checks that Go works in conjunction with a config file specifying custom queries"
steps: steps:
- uses: actions/setup-go@v2 - uses: actions/setup-go@v3
with: with:
go-version: "^1.13.1" go-version: "^1.13.1"
- uses: ./../action/init - uses: ./../action/init

View File

@@ -4,7 +4,7 @@ os: ["ubuntu-latest", "macos-latest"]
env: env:
CODEQL_EXTRACTOR_GO_BUILD_TRACING: "true" CODEQL_EXTRACTOR_GO_BUILD_TRACING: "true"
steps: steps:
- uses: actions/setup-go@v2 - uses: actions/setup-go@v3
with: with:
go-version: "^1.13.1" go-version: "^1.13.1"
- uses: ./../action/init - uses: ./../action/init

View File

@@ -3,7 +3,7 @@ description: "Checks that Go tracing works"
env: env:
CODEQL_EXTRACTOR_GO_BUILD_TRACING: "true" CODEQL_EXTRACTOR_GO_BUILD_TRACING: "true"
steps: steps:
- uses: actions/setup-go@v2 - uses: actions/setup-go@v3
with: with:
go-version: "^1.13.1" go-version: "^1.13.1"
- uses: ./../action/init - uses: ./../action/init

View File

@@ -0,0 +1,67 @@
name: "ML-powered queries"
description: "Tests that ML-powered queries are run with the security-extended suite and that they produce alerts on a test DB"
versions: [
# Latest release in 2.7.x series
"stable-20220120",
"cached",
"latest",
"nightly-latest",
]
# Test on all three platforms since ML-powered queries use native code
os: ["ubuntu-latest", "macos-latest", "windows-latest"]
steps:
- uses: ./../action/init
with:
languages: javascript
queries: security-extended
source-root: ./../action/tests/ml-powered-queries-repo
tools: ${{ steps.prepare-test.outputs.tools-url }}
- uses: ./../action/analyze
with:
output: "${{ runner.temp }}/results"
upload-database: false
env:
TEST_MODE: true
- name: Upload SARIF
uses: actions/upload-artifact@v3
with:
name: ml-powered-queries-${{ matrix.os }}-${{ matrix.version }}.sarif.json
path: "${{ runner.temp }}/results/javascript.sarif"
retention-days: 7
- name: Check results
env:
IS_WINDOWS: ${{ matrix.os == 'windows-latest' }}
shell: bash
run: |
cd "$RUNNER_TEMP/results"
# We should run at least the ML-powered queries in `expected_rules`.
expected_rules="js/ml-powered/nosql-injection js/ml-powered/path-injection js/ml-powered/sql-injection js/ml-powered/xss"
for rule in ${expected_rules}; do
found_rule=$(jq --arg rule "${rule}" '[.runs[0].tool.extensions[].rules | select(. != null) |
flatten | .[].id] | any(. == $rule)' javascript.sarif)
echo "Did find rule '${rule}': ${found_rule}"
if [[ "${found_rule}" != "true" && "${IS_WINDOWS}" != "true" ]]; then
echo "Expected SARIF output to contain rule '${rule}', but found no such rule."
exit 1
elif [[ "${found_rule}" == "true" && "${IS_WINDOWS}" == "true" ]]; then
echo "Found rule '${rule}' in the SARIF output which shouldn't have been part of the analysis."
exit 1
fi
done
# We should have at least one alert from an ML-powered query.
num_alerts=$(jq '[.runs[0].results[] |
select(.properties.score != null and (.rule.id | startswith("js/ml-powered/")))] | length' \
javascript.sarif)
echo "Found ${num_alerts} alerts from ML-powered queries.";
if [[ "${num_alerts}" -eq 0 && "${IS_WINDOWS}" != "true" ]]; then
echo "Expected to find at least one alert from an ML-powered query but found ${num_alerts}."
exit 1
elif [[ "${num_alerts}" -ne 0 && "${IS_WINDOWS}" == "true" ]]; then
echo "Expected not to find any alerts from an ML-powered query but found ${num_alerts}."
exit 1
fi

View File

@@ -7,7 +7,7 @@ os: [ubuntu-latest, macos-latest, windows-2019]
steps: steps:
# Check out the actions repo again, but at a different location. # Check out the actions repo again, but at a different location.
# choose an arbitrary SHA so that we can later test that the commit_oid is not from main # choose an arbitrary SHA so that we can later test that the commit_oid is not from main
- uses: actions/checkout@v2 - uses: actions/checkout@v3
with: with:
ref: 474bbf07f9247ffe1856c6a0f94aeeb10e7afee6 ref: 474bbf07f9247ffe1856c6a0f94aeeb10e7afee6
path: x/y/z/some-path path: x/y/z/some-path

View File

@@ -49,7 +49,7 @@ for file in os.listdir('checks'):
steps = [ steps = [
{ {
'name': 'Check out repository', 'name': 'Check out repository',
'uses': 'actions/checkout@v2' 'uses': 'actions/checkout@v3'
}, },
{ {
'name': 'Prepare test', 'name': 'Prepare test',
@@ -108,7 +108,7 @@ for file in os.listdir('checks'):
}, },
'on': { 'on': {
'push': { 'push': {
'branches': ['main', 'v1'] 'branches': ['main', 'v1', 'v2']
}, },
'pull_request': { 'pull_request': {
'types': ["opened", "synchronize", "reopened", "ready_for_review"] 'types': ["opened", "synchronize", "reopened", "ready_for_review"]

View File

@@ -544,7 +544,7 @@ export async function getRef(): Promise<string> {
const head = await getCommitOid(checkoutPath, "HEAD"); const head = await getCommitOid(checkoutPath, "HEAD");
// in actions/checkout@v2 we can check if git rev-parse HEAD == GITHUB_SHA // in actions/checkout@v2+ we can check if git rev-parse HEAD == GITHUB_SHA
// in actions/checkout@v1 this may not be true as it checks out the repository // in actions/checkout@v1 this may not be true as it checks out the repository
// using GITHUB_REF. There is a subtle race condition where // using GITHUB_REF. There is a subtle race condition where
// git rev-parse GITHUB_REF != GITHUB_SHA, so we must check // git rev-parse GITHUB_REF != GITHUB_SHA, so we must check

View File

@@ -18,7 +18,7 @@ import { Logger } from "./logging";
import * as toolcache from "./toolcache"; import * as toolcache from "./toolcache";
import { toolrunnerErrorCatcher } from "./toolrunner-error-catcher"; import { toolrunnerErrorCatcher } from "./toolrunner-error-catcher";
import * as util from "./util"; import * as util from "./util";
import { isGoodVersion, ML_POWERED_JS_QUERIES_PACK } from "./util"; import { isGoodVersion } from "./util";
type Options = Array<string | number | boolean>; type Options = Array<string | number | boolean>;
@@ -741,9 +741,10 @@ async function getCodeQLForCmd(
if (config.injectedMlQueries) { if (config.injectedMlQueries) {
// We need to inject the ML queries into the original user input before // We need to inject the ML queries into the original user input before
// we pass this on to the CLI, to make sure these get run. // we pass this on to the CLI, to make sure these get run.
let packString = ML_POWERED_JS_QUERIES_PACK.packName; const pack = await util.getMlPoweredJsQueriesPack(codeql);
if (ML_POWERED_JS_QUERIES_PACK.version) const packString =
packString = `${packString}@${ML_POWERED_JS_QUERIES_PACK.version}`; pack.packName + (pack.version ? `@${pack.version}` : "");
if (augmentedConfig.packs === undefined) augmentedConfig.packs = []; if (augmentedConfig.packs === undefined) augmentedConfig.packs = [];
if (Array.isArray(augmentedConfig.packs)) { if (Array.isArray(augmentedConfig.packs)) {
augmentedConfig.packs.push(packString); augmentedConfig.packs.push(packString);

View File

@@ -1788,7 +1788,8 @@ const mlPoweredQueriesMacro = test.macro({
}`, }`,
}); });
// macro, isMlPoweredQueriesFlagEnabled, packsInput, queriesInput, versionString // macro, codeQLVersion, isMlPoweredQueriesFlagEnabled, packsInput, queriesInput, expectedVersionString
// Test that ML-powered queries aren't run on v2.7.4 of the CLI.
test( test(
mlPoweredQueriesMacro, mlPoweredQueriesMacro,
"2.7.4", "2.7.4",
@@ -1797,6 +1798,7 @@ test(
"security-extended", "security-extended",
undefined undefined
); );
// Test that ML-powered queries aren't run when the feature flag is off.
test( test(
mlPoweredQueriesMacro, mlPoweredQueriesMacro,
"2.7.5", "2.7.5",
@@ -1805,28 +1807,42 @@ test(
"security-extended", "security-extended",
undefined undefined
); );
// Test that ML-powered queries aren't run when the user hasn't specified that we should run the
// `security-extended` or `security-and-quality` query suite.
test(mlPoweredQueriesMacro, "2.7.5", true, undefined, undefined, undefined); test(mlPoweredQueriesMacro, "2.7.5", true, undefined, undefined, undefined);
// Test that ML-powered queries are run on non-Windows platforms running `security-extended`.
test( test(
mlPoweredQueriesMacro, mlPoweredQueriesMacro,
"2.7.5", "2.7.5",
true, true,
undefined, undefined,
"security-extended", "security-extended",
"~0.1.0" process.platform === "win32" ? undefined : "~0.1.0"
); );
// Test that ML-powered queries are run on non-Windows platforms running `security-and-quality`.
test( test(
mlPoweredQueriesMacro, mlPoweredQueriesMacro,
"2.7.5", "2.7.5",
true, true,
undefined, undefined,
"security-and-quality", "security-and-quality",
"~0.1.0" process.platform === "win32" ? undefined : "~0.1.0"
); );
// Test that we don't inject an ML-powered query pack if the user has already specified one.
test( test(
mlPoweredQueriesMacro, mlPoweredQueriesMacro,
"2.7.5", "2.7.5",
true, true,
"codeql/javascript-experimental-atm-queries@0.0.1", "codeql/javascript-experimental-atm-queries@0.0.1",
"security-and-quality", "security-and-quality",
"0.0.1" process.platform === "win32" ? undefined : "0.0.1"
);
// Test that the ~0.2.0 version of ML-powered queries is run on v2.8.4 of the CLI.
test(
mlPoweredQueriesMacro,
"2.8.4",
true,
undefined,
"security-extended",
process.platform === "win32" ? undefined : "~0.2.0"
); );

View File

@@ -17,8 +17,9 @@ import { Logger } from "./logging";
import { RepositoryNwo } from "./repository"; import { RepositoryNwo } from "./repository";
import { import {
codeQlVersionAbove, codeQlVersionAbove,
getMlPoweredJsQueriesPack,
GitHubVersion, GitHubVersion,
ML_POWERED_JS_QUERIES_PACK, ML_POWERED_JS_QUERIES_PACK_NAME,
} from "./util"; } from "./util";
// Property names from the user-supplied config file. // Property names from the user-supplied config file.
@@ -299,10 +300,12 @@ async function addBuiltinSuiteQueries(
// opted into the ML-powered queries beta, and a user hasn't already added the ML-powered query // opted into the ML-powered queries beta, and a user hasn't already added the ML-powered query
// pack, then add the ML-powered query pack so that we run ML-powered queries. // pack, then add the ML-powered query pack so that we run ML-powered queries.
if ( if (
// Disable ML-powered queries on Windows
process.platform !== "win32" &&
languages.includes("javascript") && languages.includes("javascript") &&
(found === "security-extended" || found === "security-and-quality") && (found === "security-extended" || found === "security-and-quality") &&
!packs.javascript?.some( !packs.javascript?.some(
(pack) => pack.packName === ML_POWERED_JS_QUERIES_PACK.packName (pack) => pack.packName === ML_POWERED_JS_QUERIES_PACK_NAME
) && ) &&
(await featureFlags.getValue(FeatureFlag.MlPoweredQueriesEnabled)) && (await featureFlags.getValue(FeatureFlag.MlPoweredQueriesEnabled)) &&
(await codeQlVersionAbove(codeQL, CODEQL_VERSION_ML_POWERED_QUERIES)) (await codeQlVersionAbove(codeQL, CODEQL_VERSION_ML_POWERED_QUERIES))
@@ -310,7 +313,7 @@ async function addBuiltinSuiteQueries(
if (!packs.javascript) { if (!packs.javascript) {
packs.javascript = []; packs.javascript = [];
} }
packs.javascript.push(ML_POWERED_JS_QUERIES_PACK); packs.javascript.push(await getMlPoweredJsQueriesPack(codeQL));
injectedMlQueries = true; injectedMlQueries = true;
} }

View File

@@ -294,32 +294,43 @@ async function mockStdInForAuthExpectError(
} }
const ML_POWERED_JS_STATUS_TESTS: Array<[PackWithVersion[], string]> = [ const ML_POWERED_JS_STATUS_TESTS: Array<[PackWithVersion[], string]> = [
// If no packs are loaded, status is false.
[[], "false"], [[], "false"],
// If another pack is loaded but not the ML-powered query pack, status is false.
[[{ packName: "someOtherPack" }], "false"], [[{ packName: "someOtherPack" }], "false"],
// If the ML-powered query pack is loaded with a specific version, status is that version.
[ [
[{ packName: "someOtherPack" }, util.ML_POWERED_JS_QUERIES_PACK], [{ packName: util.ML_POWERED_JS_QUERIES_PACK_NAME, version: "~0.1.0" }],
util.ML_POWERED_JS_QUERIES_PACK.version!, "~0.1.0",
],
[[util.ML_POWERED_JS_QUERIES_PACK], util.ML_POWERED_JS_QUERIES_PACK.version!],
[[{ packName: util.ML_POWERED_JS_QUERIES_PACK.packName }], "other"],
[
[{ packName: util.ML_POWERED_JS_QUERIES_PACK.packName, version: "~0.0.1" }],
"other",
],
[
[
{ packName: util.ML_POWERED_JS_QUERIES_PACK.packName, version: "0.0.1" },
{ packName: util.ML_POWERED_JS_QUERIES_PACK.packName, version: "0.0.2" },
],
"other",
], ],
// If the ML-powered query pack is loaded with a specific version and another pack is loaded, the
// status is the version of the ML-powered query pack.
[ [
[ [
{ packName: "someOtherPack" }, { packName: "someOtherPack" },
{ packName: util.ML_POWERED_JS_QUERIES_PACK.packName }, { packName: util.ML_POWERED_JS_QUERIES_PACK_NAME, version: "~0.1.0" },
],
"~0.1.0",
],
// If the ML-powered query pack is loaded without a version, the status is "latest".
[[{ packName: util.ML_POWERED_JS_QUERIES_PACK_NAME }], "latest"],
// If the ML-powered query pack is loaded with two different versions, the status is "other".
[
[
{ packName: util.ML_POWERED_JS_QUERIES_PACK_NAME, version: "0.0.1" },
{ packName: util.ML_POWERED_JS_QUERIES_PACK_NAME, version: "0.0.2" },
], ],
"other", "other",
], ],
// If the ML-powered query pack is loaded with no specific version, and another pack is loaded,
// the status is "latest".
[
[
{ packName: "someOtherPack" },
{ packName: util.ML_POWERED_JS_QUERIES_PACK_NAME },
],
"latest",
],
]; ];
for (const [packs, expectedStatus] of ML_POWERED_JS_STATUS_TESTS) { for (const [packs, expectedStatus] of ML_POWERED_JS_STATUS_TESTS) {

View File

@@ -653,25 +653,30 @@ export function isGoodVersion(versionSpec: string) {
return !BROKEN_VERSIONS.includes(versionSpec); return !BROKEN_VERSIONS.includes(versionSpec);
} }
export const ML_POWERED_JS_QUERIES_PACK_NAME =
"codeql/javascript-experimental-atm-queries";
/** /**
* The ML-powered JS query pack to add to the analysis if a repo is opted into the ML-powered * Gets the ML-powered JS query pack to add to the analysis if a repo is opted into the ML-powered
* queries beta. * queries beta.
*/ */
export const ML_POWERED_JS_QUERIES_PACK: PackWithVersion = { export async function getMlPoweredJsQueriesPack(
packName: "codeql/javascript-experimental-atm-queries", codeQL: CodeQL
version: "~0.1.0", ): Promise<PackWithVersion> {
}; if (await codeQlVersionAbove(codeQL, "2.8.4")) {
return { packName: ML_POWERED_JS_QUERIES_PACK_NAME, version: "~0.2.0" };
}
return { packName: ML_POWERED_JS_QUERIES_PACK_NAME, version: "~0.1.0" };
}
/** /**
* Get information about ML-powered JS queries to populate status reports with. * Get information about ML-powered JS queries to populate status reports with.
* *
* This will be: * This will be:
* *
* - The version string if the analysis is using the ML-powered query pack that will be added to the * - The version string if the analysis is using a single version of the ML-powered query pack.
* analysis if the repo is opted into the ML-powered queries beta, i.e. * - "latest" if the version string of the ML-powered query pack is undefined. This is unlikely to
* {@link ML_POWERED_JS_QUERIES_PACK.version}. If the version string * occur in practice (see comment below).
* {@link ML_POWERED_JS_QUERIES_PACK.version} is undefined, then the status report string will be
* "latest", however this shouldn't occur in practice (see comment below).
* - "false" if the analysis won't run any ML-powered JS queries. * - "false" if the analysis won't run any ML-powered JS queries.
* - "other" in all other cases. * - "other" in all other cases.
* *
@@ -681,32 +686,25 @@ export const ML_POWERED_JS_QUERIES_PACK: PackWithVersion = {
* version of the CodeQL Action. For instance, we might want to compare the `~0.1.0` and `~0.0.2` * version of the CodeQL Action. For instance, we might want to compare the `~0.1.0` and `~0.0.2`
* version strings. * version strings.
* *
* We restrict the set of strings we report here by excluding other version strings and combinations
* of version strings. We do this to limit the cardinality of the ML-powered JS queries status
* report field, since some platforms that ingest this status report bill based on the cardinality
* of its fields.
*
* This function lives here rather than in `init-action.ts` so it's easier to test, since tests for * This function lives here rather than in `init-action.ts` so it's easier to test, since tests for
* `init-action.ts` would each need to live in their own file. See `analyze-action-env.ts` for an * `init-action.ts` would each need to live in their own file. See `analyze-action-env.ts` for an
* explanation as to why this is. * explanation as to why this is.
*/ */
export function getMlPoweredJsQueriesStatus(config: Config): string { export function getMlPoweredJsQueriesStatus(config: Config): string {
const mlPoweredJsQueryPacks = (config.packs.javascript || []).filter( const mlPoweredJsQueryPacks = (config.packs.javascript || []).filter(
(pack) => pack.packName === ML_POWERED_JS_QUERIES_PACK.packName (pack) => pack.packName === ML_POWERED_JS_QUERIES_PACK_NAME
); );
if (mlPoweredJsQueryPacks.length === 0) { switch (mlPoweredJsQueryPacks.length) {
return "false"; case 1:
// We should always specify an explicit version string in `getMlPoweredJsQueriesPack`,
// otherwise we won't be able to make changes to the pack unless those changes are compatible
// with each version of the CodeQL Action. Therefore in practice we should only hit the
// `latest` case here when customers have explicitly added the ML-powered query pack to their
// CodeQL config.
return mlPoweredJsQueryPacks[0].version || "latest";
case 0:
return "false";
default:
return "other";
} }
const firstVersionString = mlPoweredJsQueryPacks[0].version;
if (
mlPoweredJsQueryPacks.length === 1 &&
ML_POWERED_JS_QUERIES_PACK.version === firstVersionString
) {
// We should always specify an explicit version string in `ML_POWERED_JS_QUERIES_PACK`,
// otherwise we won't be able to make changes to the pack unless those changes are compatible
// with each version of the CodeQL Action. Therefore in practice, we should never hit the
// `latest` case here.
return ML_POWERED_JS_QUERIES_PACK.version || "latest";
}
return "other";
} }

View File

@@ -0,0 +1,21 @@
const mongoose = require('mongoose');
Logger = require('./logger').Logger;
Note = require('./models/note').Note;
(async () => {
if (process.argv.length != 5) {
Logger.log("Creates a private note. Usage: node add-note.js <token> <title> <body>")
return;
}
// Open the default mongoose connection
await mongoose.connect('mongodb://localhost:27017/notes', { useFindAndModify: false });
const [userToken, title, body] = process.argv.slice(2);
await Note.create({ title, body, userToken });
Logger.log(`Created private note with title ${title} and body ${body} belonging to user with token ${userToken}.`);
await mongoose.connection.close();
})();

View File

@@ -0,0 +1,68 @@
const bodyParser = require('body-parser');
const express = require('express');
const mongoose = require('mongoose');
const notesApi = require('./notes-api');
const usersApi = require('./users-api');
const addSampleData = module.exports.addSampleData = async () => {
const [userA, userB] = await User.create([
{
name: "A",
token: "tokenA"
},
{
name: "B",
token: "tokenB"
}
]);
await Note.create([
{
title: "Public note belonging to A",
body: "This is a public note belonging to A",
isPublic: true,
ownerToken: userA.token
},
{
title: "Public note belonging to B",
body: "This is a public note belonging to B",
isPublic: true,
ownerToken: userB.token
},
{
title: "Private note belonging to A",
body: "This is a private note belonging to A",
ownerToken: userA.token
},
{
title: "Private note belonging to B",
body: "This is a private note belonging to B",
ownerToken: userB.token
}
]);
}
module.exports.startApp = async () => {
// Open the default mongoose connection
await mongoose.connect('mongodb://mongo:27017/notes', { useFindAndModify: false });
// Drop contents of DB
mongoose.connection.dropDatabase();
// Add some sample data
await addSampleData();
const app = express();
app.use(bodyParser.json());
app.use(bodyParser.urlencoded());
app.get('/', async (_req, res) => {
res.send('Hello World');
});
app.use('/api/notes', notesApi.router);
app.use('/api/users', usersApi.router);
app.listen(3000);
Logger.log('Express started on port 3000');
};

View File

@@ -0,0 +1,7 @@
const startApp = require('./app').startApp;
Logger = require('./logger').Logger;
Note = require('./models/note').Note;
User = require('./models/user').User;
startApp();

View File

@@ -0,0 +1,5 @@
module.exports.Logger = class {
log(message, ...objs) {
console.log(message, objs);
}
};

View File

@@ -0,0 +1,8 @@
const mongoose = require('mongoose');
module.exports.Note = mongoose.model('Note', new mongoose.Schema({
title: String,
body: String,
ownerToken: String,
isPublic: Boolean
}));

View File

@@ -0,0 +1,6 @@
const mongoose = require('mongoose');
module.exports.User = mongoose.model('User', new mongoose.Schema({
name: String,
token: String
}));

View File

@@ -0,0 +1,44 @@
const express = require('express')
const router = module.exports.router = express.Router();
function serializeNote(note) {
return {
title: note.title,
body: note.body
};
}
router.post('/find', async (req, res) => {
const notes = await Note.find({
ownerToken: req.body.token
}).exec();
res.json({
notes: notes.map(serializeNote)
});
});
router.get('/findPublic', async (_req, res) => {
const notes = await Note.find({
isPublic: true
}).exec();
res.json({
notes: notes.map(serializeNote)
});
});
router.post('/findVisible', async (req, res) => {
const notes = await Note.find({
$or: [
{
isPublic: true
},
{
ownerToken: req.body.token
}
]
}).exec();
res.json({
notes: notes.map(serializeNote)
});
});

View File

@@ -0,0 +1,37 @@
const mongoose = require('mongoose');
Logger = require('./logger').Logger;
Note = require('./models/note').Note;
User = require('./models/user').User;
(async () => {
if (process.argv.length != 3) {
Logger.log("Outputs all notes visible to a user. Usage: node read-notes.js <token>")
return;
}
// Open the default mongoose connection
await mongoose.connect('mongodb://localhost:27017/notes', { useFindAndModify: false });
const ownerToken = process.argv[2];
const user = await User.findOne({
token: ownerToken
}).exec();
const notes = await Note.find({
$or: [
{ isPublic: true },
{ ownerToken }
]
}).exec();
notes.map(note => {
Logger.log("Title:" + note.title);
Logger.log("By:" + user.name);
Logger.log("Body:" + note.body);
Logger.log();
});
await mongoose.connection.close();
})();

View File

@@ -0,0 +1,25 @@
const express = require('express')
Logger = require('./logger').Logger;
const router = module.exports.router = express.Router();
router.post('/updateName', async (req, res) => {
Logger.log("/updateName called with new name", req.body.name);
await User.findOneAndUpdate({
token: req.body.token
}, {
name: req.body.name
}).exec();
res.json({
name: req.body.name
});
});
router.post('/getName', async (req, res) => {
const user = await User.findOne({
token: req.body.token
}).exec();
res.json({
name: user.name
});
});