Merge pull request #1129 from github/update-v2.1.15-e8c48cc8

Merge main into releases/v2

.github/check-sarif/action.yml

@@ -0,0 +1,20 @@
+name: Check SARIF
+description: Checks a SARIF file to see if certain queries were run and others were not run.
+inputs:
+  sarif-file:
+    required: true
+    description: The SARIF file to check
+
+  queries-run:
+    required: true
+    description: |
+      Comma separated list of query ids that should be included in this SARIF file.
+
+  queries-not-run:
+    required: true
+    description: |
+      Comma separated list of query ids that should NOT be included in this SARIF file.
+
+runs:
+  using: node12
+  main: index.js

.github/check-sarif/index.js

@@ -0,0 +1,43 @@
+'use strict'
+
+const core = require('@actions/core')
+const fs = require('fs')
+
+const sarif = JSON.parse(fs.readFileSync(core.getInput('sarif-file'), 'utf8'))
+const rules = sarif.runs[0].tool.extensions.flatMap(ext => ext.rules || [])
+const ruleIds = rules.map(rule => rule.id)
+
+// Check that all the expected queries ran
+const expectedQueriesRun = getQueryIdsInput('queries-run')
+const queriesThatShouldHaveRunButDidNot = expectedQueriesRun.filter(queryId => !ruleIds.includes(queryId))
+
+if (queriesThatShouldHaveRunButDidNot.length > 0) {
+  core.setFailed(`The following queries were expected to run but did not: ${queriesThatShouldHaveRunButDidNot.join(', ')}`)
+}
+
+// Check that all the unexpected queries did not run
+const expectedQueriesNotRun = getQueryIdsInput('queries-not-run')
+
+const queriesThatShouldNotHaveRunButDid = expectedQueriesNotRun.filter(queryId => ruleIds.includes(queryId))
+
+if (queriesThatShouldNotHaveRunButDid.length > 0) {
+  core.setFailed(`The following queries were NOT expected to have run but did: ${queriesThatShouldNotHaveRunButDid.join(', ')}`)
+}
+
+
+core.startGroup('All queries run')
+rules.forEach(rule => {
+  core.info(`${rule.id}: ${(rule.properties && rule.properties.name) || rule.name}`)
+})
+core.endGroup()
+
+core.startGroup('Full SARIF')
+core.info(JSON.stringify(sarif, null, 2))
+core.endGroup()
+
+function getQueryIdsInput(name) {
+  return core.getInput(name)
+    .split(',')
+    .map(q => q.trim())
+    .filter(q => q.length > 0)
+}

.github/query-filter-test/action.yml

@@ -0,0 +1,52 @@
+name: Query Filter Test
+description: Runs a test of query filters using the check sarif action
+inputs:
+  sarif-file:
+    required: true
+    description: The SARIF file to check
+
+  queries-run:
+    required: true
+    description: |
+      Comma separated list of query ids that should be included in this SARIF file.
+
+  queries-not-run:
+    required: true
+    description: |
+      Comma separated list of query ids that should NOT be included in this SARIF file.
+
+  config-file:
+    required: true
+    description: |
+      The location of the codeql configuration file to use.
+
+  tools:
+    required: true
+    description: |
+      The url of codeql to use.
+
+runs:
+  using: composite
+  steps:
+    - uses: ./../action/init
+      with:
+        languages: javascript
+        config-file: ${{ inputs.config-file }}
+        tools: ${{ inputs.tools }}
+        db-location: ${{ runner.temp }}/query-filter-test
+    - uses: ./../action/analyze
+      with:
+        output: ${{ runner.temp }}/results
+        upload-database: false
+        upload: false
+      env:
+        TEST_MODE: "true"
+    - name: Check SARIF
+      uses: ./../action/.github/check-sarif
+      with:
+        sarif-file:  ${{ inputs.sarif-file }}
+        queries-run: ${{ inputs.queries-run}}
+        queries-not-run: ${{ inputs.queries-not-run}}
+    - name: Cleanup after test
+      shell: bash
+      run: rm -rf "$RUNNER_TEMP/results" "$RUNNER_TEMP//query-filter-test"

.github/workflows/check-for-conflicts.yml

@@ -4,7 +4,7 @@ name: Check for conflicts
 
 on:
   pull_request:
-    branches: [main, v1, v2]
+    branches: [main, releases/v1, releases/v2]
     # Run checks on reopened draft PRs to support triggering PR checks on draft PRs that were opened
     # by other workflows.
     types: [opened, synchronize, reopened, ready_for_review]

.github/workflows/expected-queries-runs.yml

@@ -0,0 +1,49 @@
+name: Expected queries runs
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+on:
+  push:
+    branches:
+    - main
+    - releases/v1
+    - releases/v2
+  pull_request:
+    types:
+    - opened
+    - synchronize
+    - reopened
+    - ready_for_review
+  workflow_dispatch: {}
+
+jobs:
+  expected-queries:
+    name: Expected Queries Tests
+    timeout-minutes: 45
+    runs-on: ubuntu-latest
+    steps:
+    - name: Check out repository
+      uses: actions/checkout@v3
+    - name: Prepare test
+      id: prepare-test
+      uses: ./.github/prepare-test
+      with:
+        version: latest
+    - uses: ./../action/init
+      with:
+        languages: javascript
+        tools: ${{ steps.prepare-test.outputs.tools-url }}
+    - uses: ./../action/analyze
+      with:
+        output: ${{ runner.temp }}/results
+        upload-database: false
+        upload: false
+      env:
+        TEST_MODE: true
+
+    - name: Check Sarif
+      uses: ./../action/.github/check-sarif
+      with:
+        sarif-file: ${{ runner.temp }}/results/javascript.sarif
+        queries-run: js/incomplete-hostname-regexp,js/path-injection
+        queries-not-run: foo,bar

.github/workflows/python-deps.yml

@@ -18,6 +18,11 @@ jobs:
           os: [ubuntu-latest, macos-latest]
           python_deps_type: [pipenv, poetry, requirements, setup_py]
           python_version: [2, 3]
+          exclude:
+            # Python2 and poetry are not supported. See https://github.com/actions/setup-python/issues/374
+            - python_version: 2
+              python_deps_type: poetry
+
 
     env:
       PYTHON_DEPS_TYPE: ${{ matrix.python_deps_type }}
@@ -115,6 +120,10 @@ jobs:
         matrix:
           python_deps_type: [pipenv, poetry, requirements, setup_py]
           python_version: [2, 3]
+          exclude:
+            # Python2 and poetry are not supported. See https://github.com/actions/setup-python/issues/374
+            - python_version: 2
+              python_deps_type: poetry
 
     env:
       PYTHON_DEPS_TYPE: ${{ matrix.python_deps_type }}

.github/workflows/script/update-required-checks.sh

@@ -21,7 +21,7 @@ fi
 echo "Getting checks for $GITHUB_SHA"
 
 # Ignore any checks with "https://", CodeQL, LGTM, and Update checks.
-CHECKS="$(gh api repos/github/codeql-action/commits/${GITHUB_SHA}/check-runs --paginate | jq --slurp --compact-output --raw-output '[.[].check_runs | .[].name | select(contains("https://") or . == "CodeQL" or . == "LGTM.com" or contains("Update") | not)] | sort')"
+CHECKS="$(gh api repos/github/codeql-action/commits/${GITHUB_SHA}/check-runs --paginate | jq --slurp --compact-output --raw-output '[.[].check_runs | .[].name | select(contains("https://") or . == "CodeQL" or . == "LGTM.com" or contains("Update") or contains("update") | not)] | unique | sort')"
 
 echo "$CHECKS" | jq
 

CHANGELOG.md

@@ -1,9 +1,23 @@
 # CodeQL Action Changelog
 
-## [UNRELEASED]
+## 2.1.15 - 28 Jun 2022
+
+- CodeQL query packs listed in the `packs` configuration field will be skipped if their target language is not being analyzed in the current Actions job. Previously, this would throw an error. [#1116](https://github.com/github/codeql-action/pull/1116)
+- The combination of python2 and poetry is no longer supported. See https://github.com/actions/setup-python/issues/374 for more details. [#1124](https://github.com/github/codeql-action/pull/1124)
+- Update default CodeQL bundle version to 2.10.0. [#1123](https://github.com/github/codeql-action/pull/1123)
+
+## 2.1.14 - 22 Jun 2022
 
 No user facing changes.
 
+## 2.1.13 - 21 Jun 2022
+
+- Update default CodeQL bundle version to 2.9.4. [#1100](https://github.com/github/codeql-action/pull/1100)
+
+## 2.1.12 - 01 Jun 2022
+
+- Update default CodeQL bundle version to 2.9.3. [#1084](https://github.com/github/codeql-action/pull/1084)
+
 ## 2.1.11 - 17 May 2022
 
 - Update default CodeQL bundle version to 2.9.2. [#1074](https://github.com/github/codeql-action/pull/1074)

lib/analyze-action.js

@@ -26,9 +26,11 @@ const artifact = __importStar(require("@actions/artifact"));
 const core = __importStar(require("@actions/core"));
 const actionsUtil = __importStar(require("./actions-util"));
 const analyze_1 = require("./analyze");
+const api_client_1 = require("./api-client");
 const codeql_1 = require("./codeql");
 const config_utils_1 = require("./config-utils");
 const database_upload_1 = require("./database-upload");
+const feature_flags_1 = require("./feature-flags");
 const logging_1 = require("./logging");
 const repository_1 = require("./repository");
 const upload_lib = __importStar(require("./upload-lib"));
@@ -76,7 +78,9 @@ async function run() {
         const threads = util.getThreadsFlag(actionsUtil.getOptionalInput("threads") || process.env["CODEQL_THREADS"], logger);
         const memory = util.getMemoryFlag(actionsUtil.getOptionalInput("ram") || process.env["CODEQL_RAM"]);
         const repositoryNwo = (0, repository_1.parseRepositoryNwo)(util.getRequiredEnvParam("GITHUB_REPOSITORY"));
-        await (0, analyze_1.runFinalize)(outputDir, threads, memory, config, logger);
+        const gitHubVersion = await (0, api_client_1.getGitHubVersionActionsOnly)();
+        const featureFlags = new feature_flags_1.GitHubFeatureFlags(gitHubVersion, apiDetails, repositoryNwo, logger);
+        await (0, analyze_1.runFinalize)(outputDir, threads, memory, config, logger, featureFlags);
         if (actionsUtil.getRequiredInput("skip-queries") !== "true") {
             runStats = await (0, analyze_1.runQueries)(outputDir, memory, util.getAddSnippetsFlag(actionsUtil.getRequiredInput("add-snippets")), threads, actionsUtil.getOptionalInput("category"), config, logger);
             if (config.debugMode) {

lib/analyze.js

@@ -18,11 +18,15 @@ var __importStar = (this && this.__importStar) || function (mod) {
     __setModuleDefault(result, mod);
     return result;
 };
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.runCleanup = exports.runFinalize = exports.runQueries = exports.CodeQLAnalysisError = void 0;
+exports.runCleanup = exports.runFinalize = exports.runQueries = exports.createdDBForScannedLanguages = exports.CodeQLAnalysisError = void 0;
 const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
 const toolrunner = __importStar(require("@actions/exec/lib/toolrunner"));
+const del_1 = __importDefault(require("del"));
 const yaml = __importStar(require("js-yaml"));
 const analysisPaths = __importStar(require("./analysis-paths"));
 const codeql_1 = require("./codeql");
@@ -64,11 +68,10 @@ async function setupPythonExtractor(logger) {
     logger.info(`Setting LGTM_PYTHON_SETUP_VERSION=${output}`);
     process.env["LGTM_PYTHON_SETUP_VERSION"] = output;
 }
-async function createdDBForScannedLanguages(config, logger) {
+async function createdDBForScannedLanguages(codeql, config, logger, featureFlags) {
     // Insert the LGTM_INDEX_X env vars at this point so they are set when
     // we extract any scanned languages.
     analysisPaths.includeAndExcludeAnalysisPaths(config);
-    const codeql = await (0, codeql_1.getCodeQL)(config.codeQLCmd);
     for (const language of config.languages) {
         if ((0, languages_1.isScannedLanguage)(language) &&
             !dbIsFinalized(config, language, logger)) {
@@ -76,11 +79,12 @@ async function createdDBForScannedLanguages(config, logger) {
             if (language === languages_1.Language.python) {
                 await setupPythonExtractor(logger);
             }
-            await codeql.extractScannedLanguage(util.getCodeQLDatabasePath(config, language), language);
+            await codeql.extractScannedLanguage(util.getCodeQLDatabasePath(config, language), language, featureFlags);
             logger.endGroup();
         }
     }
 }
+exports.createdDBForScannedLanguages = createdDBForScannedLanguages;
 function dbIsFinalized(config, language, logger) {
     const dbPath = util.getCodeQLDatabasePath(config, language);
     try {
@@ -92,9 +96,9 @@ function dbIsFinalized(config, language, logger) {
         return false;
     }
 }
-async function finalizeDatabaseCreation(config, threadsFlag, memoryFlag, logger) {
-    await createdDBForScannedLanguages(config, logger);
+async function finalizeDatabaseCreation(config, threadsFlag, memoryFlag, logger, featureFlags) {
     const codeql = await (0, codeql_1.getCodeQL)(config.codeQLCmd);
+    await createdDBForScannedLanguages(codeql, config, logger, featureFlags);
     for (const language of config.languages) {
         if (dbIsFinalized(config, language, logger)) {
             logger.info(`There is already a finalized database for ${language} at the location where the CodeQL Action places databases, so we did not create one.`);
@@ -234,7 +238,7 @@ exports.runQueries = runQueries;
 function createQuerySuiteContents(queries) {
     return queries.map((q) => `- query: ${q}`).join("\n");
 }
-async function runFinalize(outputDir, threadsFlag, memoryFlag, config, logger) {
+async function runFinalize(outputDir, threadsFlag, memoryFlag, config, logger, featureFlags) {
     const codeql = await (0, codeql_1.getCodeQL)(config.codeQLCmd);
     if (await util.codeQlVersionAbove(codeql, codeql_1.CODEQL_VERSION_NEW_TRACING)) {
         // Delete variables as specified by the end-tracing script
@@ -244,13 +248,8 @@ async function runFinalize(outputDir, threadsFlag, memoryFlag, config, logger) {
         // Delete the tracer config env var to avoid tracing ourselves
         delete process.env[sharedEnv.ODASA_TRACER_CONFIGURATION];
     }
-    // After switching to Node16, this entire block can be replaced with `await fs.promises.rm(outputDir, { recursive: true, force: true });`.
     try {
-        await fs.promises.rmdir(outputDir, {
-            recursive: true,
-            maxRetries: 5,
-            retryDelay: 2000,
-        });
+        await (0, del_1.default)(outputDir, { force: true });
     }
     catch (error) {
         if ((error === null || error === void 0 ? void 0 : error.code) !== "ENOENT") {
@@ -258,7 +257,7 @@ async function runFinalize(outputDir, threadsFlag, memoryFlag, config, logger) {
         }
     }
     await fs.promises.mkdir(outputDir, { recursive: true });
-    await finalizeDatabaseCreation(config, threadsFlag, memoryFlag, logger);
+    await finalizeDatabaseCreation(config, threadsFlag, memoryFlag, logger, featureFlags);
 }
 exports.runFinalize = runFinalize;
 async function runCleanup(config, cleanupLevel, logger) {

lib/analyze.test.js

@@ -29,7 +29,9 @@ const yaml = __importStar(require("js-yaml"));
 const sinon = __importStar(require("sinon"));
 const analyze_1 = require("./analyze");
 const codeql_1 = require("./codeql");
+const codeql_test_1 = require("./codeql.test");
 const count = __importStar(require("./count-loc"));
+const feature_flags_1 = require("./feature-flags");
 const languages_1 = require("./languages");
 const logging_1 = require("./logging");
 const testing_utils_1 = require("./testing-utils");
@@ -210,4 +212,72 @@ const util = __importStar(require("./util"));
         }
     }
 });
+const stubConfig = {
+    languages: [languages_1.Language.cpp, languages_1.Language.go],
+    queries: {},
+    pathsIgnore: [],
+    paths: [],
+    originalUserInput: {},
+    tempDir: "",
+    toolCacheDir: "",
+    codeQLCmd: "",
+    gitHubVersion: {
+        type: util.GitHubVariant.DOTCOM,
+    },
+    dbLocation: "",
+    packs: {},
+    debugMode: false,
+    debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
+    debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
+    injectedMlQueries: false,
+};
+for (const options of [
+    {
+        name: "Lua feature flag enabled, but old CLI",
+        version: "2.9.0",
+        featureFlags: [feature_flags_1.FeatureFlag.LuaTracerConfigEnabled],
+        yesFlagSet: false,
+        noFlagSet: false,
+    },
+    {
+        name: "Lua feature flag disabled, with old CLI",
+        version: "2.9.0",
+        featureFlags: [],
+        yesFlagSet: false,
+        noFlagSet: false,
+    },
+    {
+        name: "Lua feature flag enabled, with new CLI",
+        version: "2.10.0",
+        featureFlags: [feature_flags_1.FeatureFlag.LuaTracerConfigEnabled],
+        yesFlagSet: true,
+        noFlagSet: false,
+    },
+    {
+        name: "Lua feature flag disabled, with new CLI",
+        version: "2.10.0",
+        featureFlags: [],
+        yesFlagSet: false,
+        noFlagSet: true,
+    },
+]) {
+    (0, ava_1.default)(`createdDBForScannedLanguages() ${options.name}`, async (t) => {
+        const runnerConstructorStub = (0, codeql_test_1.stubToolRunnerConstructor)();
+        const codeqlObject = await (0, codeql_1.getCodeQLForTesting)("codeql/for-testing");
+        sinon.stub(codeqlObject, "getVersion").resolves(options.version);
+        const promise = (0, analyze_1.createdDBForScannedLanguages)(codeqlObject, stubConfig, (0, logging_1.getRunnerLogger)(true), (0, feature_flags_1.createFeatureFlags)(options.featureFlags));
+        // call listener on `codeql resolve extractor`
+        const mockToolRunner = runnerConstructorStub.getCall(0);
+        mockToolRunner.args[2].listeners.stdout('"/path/to/extractor"');
+        await promise;
+        if (options.yesFlagSet)
+            t.true(runnerConstructorStub.secondCall.args[1].includes("--internal-use-lua-tracing"), "--internal-use-lua-tracing should be present, but it is absent");
+        else
+            t.false(runnerConstructorStub.secondCall.args[1].includes("--internal-use-lua-tracing"), "--internal-use-lua-tracing should be absent, but it is present");
+        if (options.noFlagSet)
+            t.true(runnerConstructorStub.secondCall.args[1].includes("--no-internal-use-lua-tracing"), "--no-internal-use-lua-tracing should be present, but it is absent");
+        else
+            t.false(runnerConstructorStub.secondCall.args[1].includes("--no-internal-use-lua-tracing"), "--no-internal-use-lua-tracing should be absent, but it is present");
+    });
+}
 //# sourceMappingURL=analyze.test.js.map

lib/api-compatibility.json

@@ -1 +1 @@
-{ "maximumVersion": "3.5", "minimumVersion": "3.1" }
+{ "maximumVersion": "3.6", "minimumVersion": "3.2" }

lib/codeql.js

@@ -77,7 +77,7 @@ const CODEQL_VERSION_SARIF_GROUP = "2.5.3";
 exports.CODEQL_VERSION_COUNTS_LINES = "2.6.2";
 const CODEQL_VERSION_CUSTOM_QUERY_HELP = "2.7.1";
 exports.CODEQL_VERSION_ML_POWERED_QUERIES = "2.7.5";
-const CODEQL_VERSION_LUA_TRACER_CONFIG = "2.9.3";
+const CODEQL_VERSION_LUA_TRACER_CONFIG = "2.10.0";
 /**
  * This variable controls using the new style of tracing from the CodeQL
  * CLI. In particular, with versions above this we will use both indirect
@@ -384,8 +384,8 @@ exports.getCachedCodeQL = getCachedCodeQL;
  * a non-existent placeholder codeql command, so tests that use this function
  * should also stub the toolrunner.ToolRunner constructor.
  */
-async function getCodeQLForTesting() {
-    return getCodeQLForCmd("codeql-for-testing", false);
+async function getCodeQLForTesting(cmd = "codeql-for-testing") {
+    return getCodeQLForCmd(cmd, false);
 }
 exports.getCodeQLForTesting = getCodeQLForTesting;
 /**
@@ -510,7 +510,7 @@ async function getCodeQLForCmd(cmd, checkVersion) {
             ].join(" ");
             await runTool(autobuildCmd);
         },
-        async extractScannedLanguage(databasePath, language) {
+        async extractScannedLanguage(databasePath, language, featureFlags) {
             // Get extractor location
             let extractorPath = "";
             await new toolrunner.ToolRunner(cmd, [
@@ -533,10 +533,20 @@ async function getCodeQLForCmd(cmd, checkVersion) {
             // Set trace command
             const ext = process.platform === "win32" ? ".cmd" : ".sh";
             const traceCommand = path.resolve(JSON.parse(extractorPath), "tools", `autobuild${ext}`);
+            const extraArgs = [];
+            if (await util.codeQlVersionAbove(this, CODEQL_VERSION_LUA_TRACER_CONFIG)) {
+                if (await featureFlags.getValue(feature_flags_1.FeatureFlag.LuaTracerConfigEnabled)) {
+                    extraArgs.push("--internal-use-lua-tracing");
+                }
+                else {
+                    extraArgs.push("--no-internal-use-lua-tracing");
+                }
+            }
             // Run trace command
             await (0, toolrunner_error_catcher_1.toolrunnerErrorCatcher)(cmd, [
                 "database",
                 "trace-command",
+                ...extraArgs,
                 ...getExtraOptionsFromEnv(["database", "trace-command"]),
                 databasePath,
                 "--",

lib/codeql.test.js

@@ -22,6 +22,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.stubToolRunnerConstructor = void 0;
 const path = __importStar(require("path"));
 const toolrunner = __importStar(require("@actions/exec/lib/toolrunner"));
 const toolcache = __importStar(require("@actions/tool-cache"));
@@ -291,4 +292,5 @@ function stubToolRunnerConstructor() {
     runnerConstructorStub.returns(runnerObjectStub);
     return runnerConstructorStub;
 }
+exports.stubToolRunnerConstructor = stubToolRunnerConstructor;
 //# sourceMappingURL=codeql.test.js.map

lib/config-utils.js

@@ -19,7 +19,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
     return result;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getConfig = exports.getPathToParsedConfigFile = exports.initConfig = exports.parsePacks = exports.validatePacksSpecification = exports.parsePacksFromConfig = exports.getDefaultConfig = exports.getUnknownLanguagesError = exports.getNoLanguagesError = exports.getConfigFileDirectoryGivenMessage = exports.getConfigFileFormatInvalidMessage = exports.getConfigFileRepoFormatInvalidMessage = exports.getConfigFileDoesNotExistErrorMessage = exports.getConfigFileOutsideWorkspaceErrorMessage = exports.getLocalPathDoesNotExist = exports.getLocalPathOutsideOfRepository = exports.getPacksStrInvalid = exports.getPacksInvalid = exports.getPacksInvalidSplit = exports.getPacksRequireLanguage = exports.getPathsInvalid = exports.getPathsIgnoreInvalid = exports.getQueryUsesInvalid = exports.getQueriesInvalid = exports.getDisableDefaultQueriesInvalid = exports.getNameInvalid = exports.validateAndSanitisePath = void 0;
+exports.getConfig = exports.getPathToParsedConfigFile = exports.initConfig = exports.parsePacks = exports.validatePacksSpecification = exports.parsePacksFromConfig = exports.getDefaultConfig = exports.getUnknownLanguagesError = exports.getNoLanguagesError = exports.getConfigFileDirectoryGivenMessage = exports.getConfigFileFormatInvalidMessage = exports.getConfigFileRepoFormatInvalidMessage = exports.getConfigFileDoesNotExistErrorMessage = exports.getConfigFileOutsideWorkspaceErrorMessage = exports.getLocalPathDoesNotExist = exports.getLocalPathOutsideOfRepository = exports.getPacksStrInvalid = exports.getPacksInvalid = exports.getPacksInvalidSplit = exports.getPathsInvalid = exports.getPathsIgnoreInvalid = exports.getQueryUsesInvalid = exports.getQueriesInvalid = exports.getDisableDefaultQueriesInvalid = exports.getNameInvalid = exports.validateAndSanitisePath = void 0;
 const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
 const yaml = __importStar(require("js-yaml"));
@@ -306,9 +306,8 @@ function getPathsInvalid(configFile) {
 }
 exports.getPathsInvalid = getPathsInvalid;
 function getPacksRequireLanguage(lang, configFile) {
-    return getConfigFilePropertyError(configFile, PACKS_PROPERTY, `has "${lang}", but it is not one of the languages to analyze`);
+    return getConfigFilePropertyError(configFile, PACKS_PROPERTY, `has "${lang}", but it is not a valid language.`);
 }
-exports.getPacksRequireLanguage = getPacksRequireLanguage;
 function getPacksInvalidSplit(configFile) {
     return getConfigFilePropertyError(configFile, PACKS_PROPERTY, "must split packages by language");
 }
@@ -542,7 +541,7 @@ async function loadConfig(languagesInput, queriesInput, packsInput, configFile,
     if (!disableDefaultQueries) {
         await addDefaultQueries(codeQL, languages, queries);
     }
-    const packs = parsePacks((_a = parsedYAML[PACKS_PROPERTY]) !== null && _a !== void 0 ? _a : {}, packsInput, languages, configFile);
+    const packs = parsePacks((_a = parsedYAML[PACKS_PROPERTY]) !== null && _a !== void 0 ? _a : {}, packsInput, languages, configFile, logger);
     // If queries were provided using `with` in the action configuration,
     // they should take precedence over the queries in the config file
     // unless they're prefixed with "+", in which case they supplement those
@@ -616,7 +615,7 @@ const PACK_IDENTIFIER_PATTERN = (function () {
     return new RegExp(`^${component}/${component}$`);
 })();
 // Exported for testing
-function parsePacksFromConfig(packsByLanguage, languages, configFile) {
+function parsePacksFromConfig(packsByLanguage, languages, configFile, logger) {
     const packs = {};
     if (Array.isArray(packsByLanguage)) {
         if (languages.length === 1) {
@@ -636,7 +635,15 @@ function parsePacksFromConfig(packsByLanguage, languages, configFile) {
             throw new Error(getPacksInvalid(configFile));
         }
         if (!languages.includes(lang)) {
-            throw new Error(getPacksRequireLanguage(lang, configFile));
+            // This particular language is not being analyzed in this run.
+            if (languages_1.Language[lang]) {
+                logger.info(`Ignoring packs for ${lang} since this language is not being analyzed in this run.`);
+                continue;
+            }
+            else {
+                // This language is invalid, probably a misspelling
+                throw new Error(getPacksRequireLanguage(configFile, lang));
+            }
         }
         packs[lang] = [];
         for (const packStr of packsArr) {
@@ -734,9 +741,9 @@ function validatePacksSpecification(packStr, configFile) {
 }
 exports.validatePacksSpecification = validatePacksSpecification;
 // exported for testing
-function parsePacks(rawPacksFromConfig, rawPacksInput, languages, configFile) {
+function parsePacks(rawPacksFromConfig, rawPacksInput, languages, configFile, logger) {
     const packsFromInput = parsePacksFromInput(rawPacksInput, languages);
-    const packsFomConfig = parsePacksFromConfig(rawPacksFromConfig, languages, configFile);
+    const packsFomConfig = parsePacksFromConfig(rawPacksFromConfig, languages, configFile, logger);
     if (!packsFromInput) {
         return packsFomConfig;
     }

lib/config-utils.test.js

@@ -749,14 +749,14 @@ const invalidPaths = ["a/***/b", "a/**b", "a/b**", "**"];
  * Test macro for ensuring the packs block is valid
  */
 const parsePacksMacro = ava_1.default.macro({
-    exec: (t, packsByLanguage, languages, expected) => t.deepEqual(configUtils.parsePacksFromConfig(packsByLanguage, languages, "/a/b"), expected),
+    exec: (t, packsByLanguage, languages, expected) => t.deepEqual(configUtils.parsePacksFromConfig(packsByLanguage, languages, "/a/b", mockLogger), expected),
     title: (providedTitle = "") => `Parse Packs: ${providedTitle}`,
 });
 /**
  * Test macro for testing when the packs block is invalid
  */
 const parsePacksErrorMacro = ava_1.default.macro({
-    exec: (t, packsByLanguage, languages, expected) => t.throws(() => configUtils.parsePacksFromConfig(packsByLanguage, languages, "/a/b"), {
+    exec: (t, packsByLanguage, languages, expected) => t.throws(() => configUtils.parsePacksFromConfig(packsByLanguage, languages, "/a/b", {}), {
         message: expected,
     }),
     title: (providedTitle = "") => `Parse Packs Error: ${providedTitle}`,
@@ -782,6 +782,12 @@ const invalidPackNameMacro = ava_1.default.macro({
     [languages_1.Language.cpp]: ["a/b", "c/d@1.2.3"],
     [languages_1.Language.java]: ["d/e", "f/g@1.2.3"],
 });
+(0, ava_1.default)("two packs with unused language in config", parsePacksMacro, {
+    [languages_1.Language.cpp]: ["a/b", "c/d@1.2.3"],
+    [languages_1.Language.java]: ["d/e", "f/g@1.2.3"],
+}, [languages_1.Language.cpp, languages_1.Language.csharp], {
+    [languages_1.Language.cpp]: ["a/b", "c/d@1.2.3"],
+});
 (0, ava_1.default)("packs with other valid names", parsePacksMacro, [
     // ranges are ok
     "c/d@1.0",
@@ -814,7 +820,6 @@ const invalidPackNameMacro = ava_1.default.macro({
     ],
 });
 (0, ava_1.default)("no language", parsePacksErrorMacro, ["a/b@1.2.3"], [languages_1.Language.java, languages_1.Language.python], /The configuration file "\/a\/b" is invalid: property "packs" must split packages by language/);
-(0, ava_1.default)("invalid language", parsePacksErrorMacro, { [languages_1.Language.java]: ["c/d"] }, [languages_1.Language.cpp], /The configuration file "\/a\/b" is invalid: property "packs" has "java", but it is not one of the languages to analyze/);
 (0, ava_1.default)("not an array", parsePacksErrorMacro, { [languages_1.Language.cpp]: "c/d" }, [languages_1.Language.cpp], /The configuration file "\/a\/b" is invalid: property "packs" must be an array of non-empty strings/);
 (0, ava_1.default)(invalidPackNameMacro, "c"); // all packs require at least a scope and a name
 (0, ava_1.default)(invalidPackNameMacro, "c-/d");
@@ -832,12 +837,17 @@ const invalidPackNameMacro = ava_1.default.macro({
  * Test macro for testing the packs block and the packs input
  */
 function parseInputAndConfigMacro(t, packsFromConfig, packsFromInput, languages, expected) {
-    t.deepEqual(configUtils.parsePacks(packsFromConfig, packsFromInput, languages, "/a/b"), expected);
+    t.deepEqual(configUtils.parsePacks(packsFromConfig, packsFromInput, languages, "/a/b", mockLogger), expected);
 }
 parseInputAndConfigMacro.title = (providedTitle) => `Parse Packs input and config: ${providedTitle}`;
+const mockLogger = {
+    info: (message) => {
+        console.log(message);
+    },
+};
 function parseInputAndConfigErrorMacro(t, packsFromConfig, packsFromInput, languages, expected) {
     t.throws(() => {
-        configUtils.parsePacks(packsFromConfig, packsFromInput, languages, "/a/b");
+        configUtils.parsePacks(packsFromConfig, packsFromInput, languages, "/a/b", mockLogger);
     }, {
         message: expected,
     });
@@ -925,4 +935,10 @@ const mlPoweredQueriesMacro = ava_1.default.macro({
 (0, ava_1.default)(mlPoweredQueriesMacro, "2.9.0", true, undefined, "security-and-quality", "~0.2.0");
 // Test that we don't inject an ML-powered query pack if the user has already specified one.
 (0, ava_1.default)(mlPoweredQueriesMacro, "2.9.0", true, "codeql/javascript-experimental-atm-queries@0.0.1", "security-and-quality", "0.0.1");
+// Test that ML-powered queries are run on all platforms running `security-extended` on CodeQL
+// CLI 2.9.3+.
+(0, ava_1.default)(mlPoweredQueriesMacro, "2.9.3", true, undefined, "security-extended", "~0.3.0");
+// Test that ML-powered queries are run on all platforms running `security-and-quality` on CodeQL
+// CLI 2.9.3+.
+(0, ava_1.default)(mlPoweredQueriesMacro, "2.9.3", true, undefined, "security-and-quality", "~0.3.0");
 //# sourceMappingURL=config-utils.test.js.map

lib/defaults.json

@@ -1,3 +1,3 @@
 {
-    "bundleVersion": "codeql-bundle-20220512"
+    "bundleVersion": "codeql-bundle-20220623"
 }

lib/languages.js

@@ -11,6 +11,7 @@ var Language;
     Language["javascript"] = "javascript";
     Language["python"] = "python";
     Language["ruby"] = "ruby";
+    Language["swift"] = "swift";
 })(Language = exports.Language || (exports.Language = {}));
 // Additional names for languages
 const LANGUAGE_ALIASES = {
@@ -35,7 +36,7 @@ function parseLanguage(language) {
 }
 exports.parseLanguage = parseLanguage;
 function isTracedLanguage(language) {
-    return (["cpp", "java", "csharp"].includes(language) ||
+    return (["cpp", "java", "csharp", "swift"].includes(language) ||
         (process.env["CODEQL_EXTRACTOR_GO_BUILD_TRACING"] === "on" &&
             language === Language.go));
 }

lib/languages.js.map

@@ -1 +1 @@
-{"version":3,"file":"languages.js","sourceRoot":"","sources":["../src/languages.ts"],"names":[],"mappings":";;;AAAA,wCAAwC;AACxC,IAAY,QAQX;AARD,WAAY,QAAQ;IAClB,6BAAiB,CAAA;IACjB,uBAAW,CAAA;IACX,qBAAS,CAAA;IACT,yBAAa,CAAA;IACb,qCAAyB,CAAA;IACzB,6BAAiB,CAAA;IACjB,yBAAa,CAAA;AACf,CAAC,EARW,QAAQ,GAAR,gBAAQ,KAAR,gBAAQ,QAQnB;AAED,iCAAiC;AACjC,MAAM,gBAAgB,GAAiC;IACrD,CAAC,EAAE,QAAQ,CAAC,GAAG;IACf,KAAK,EAAE,QAAQ,CAAC,GAAG;IACnB,IAAI,EAAE,QAAQ,CAAC,MAAM;IACrB,UAAU,EAAE,QAAQ,CAAC,UAAU;CAChC,CAAC;AAEF,gGAAgG;AAChG,SAAgB,aAAa,CAAC,QAAgB;IAC5C,0BAA0B;IAC1B,QAAQ,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;IAElC,6BAA6B;IAC7B,IAAI,QAAQ,IAAI,QAAQ,EAAE;QACxB,OAAO,QAAoB,CAAC;KAC7B;IAED,yBAAyB;IACzB,IAAI,QAAQ,IAAI,gBAAgB,EAAE;QAChC,OAAO,gBAAgB,CAAC,QAAQ,CAAC,CAAC;KACnC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAfD,sCAeC;AAED,SAAgB,gBAAgB,CAAC,QAAkB;IACjD,OAAO,CACL,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC;QAC5C,CAAC,OAAO,CAAC,GAAG,CAAC,mCAAmC,CAAC,KAAK,IAAI;YACxD,QAAQ,KAAK,QAAQ,CAAC,EAAE,CAAC,CAC5B,CAAC;AACJ,CAAC;AAND,4CAMC;AAED,SAAgB,iBAAiB,CAAC,QAAkB;IAClD,OAAO,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;AACrC,CAAC;AAFD,8CAEC"}
+{"version":3,"file":"languages.js","sourceRoot":"","sources":["../src/languages.ts"],"names":[],"mappings":";;;AAAA,wCAAwC;AACxC,IAAY,QASX;AATD,WAAY,QAAQ;IAClB,6BAAiB,CAAA;IACjB,uBAAW,CAAA;IACX,qBAAS,CAAA;IACT,yBAAa,CAAA;IACb,qCAAyB,CAAA;IACzB,6BAAiB,CAAA;IACjB,yBAAa,CAAA;IACb,2BAAe,CAAA;AACjB,CAAC,EATW,QAAQ,GAAR,gBAAQ,KAAR,gBAAQ,QASnB;AAED,iCAAiC;AACjC,MAAM,gBAAgB,GAAiC;IACrD,CAAC,EAAE,QAAQ,CAAC,GAAG;IACf,KAAK,EAAE,QAAQ,CAAC,GAAG;IACnB,IAAI,EAAE,QAAQ,CAAC,MAAM;IACrB,UAAU,EAAE,QAAQ,CAAC,UAAU;CAChC,CAAC;AAEF,gGAAgG;AAChG,SAAgB,aAAa,CAAC,QAAgB;IAC5C,0BAA0B;IAC1B,QAAQ,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;IAElC,6BAA6B;IAC7B,IAAI,QAAQ,IAAI,QAAQ,EAAE;QACxB,OAAO,QAAoB,CAAC;KAC7B;IAED,yBAAyB;IACzB,IAAI,QAAQ,IAAI,gBAAgB,EAAE;QAChC,OAAO,gBAAgB,CAAC,QAAQ,CAAC,CAAC;KACnC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAfD,sCAeC;AAED,SAAgB,gBAAgB,CAAC,QAAkB;IACjD,OAAO,CACL,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC;QACrD,CAAC,OAAO,CAAC,GAAG,CAAC,mCAAmC,CAAC,KAAK,IAAI;YACxD,QAAQ,KAAK,QAAQ,CAAC,EAAE,CAAC,CAC5B,CAAC;AACJ,CAAC;AAND,4CAMC;AAED,SAAgB,iBAAiB,CAAC,QAAkB;IAClD,OAAO,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;AACrC,CAAC;AAFD,8CAEC"}

lib/runner.js

@@ -297,7 +297,7 @@ program
         }
         const threads = (0, util_1.getThreadsFlag)(cmd.threads || initEnv["CODEQL_THREADS"], logger);
         const memory = (0, util_1.getMemoryFlag)(cmd.ram || initEnv["CODEQL_RAM"]);
-        await (0, analyze_1.runFinalize)(outputDir, threads, memory, config, logger);
+        await (0, analyze_1.runFinalize)(outputDir, threads, memory, config, logger, (0, feature_flags_1.createFeatureFlags)([]));
         await (0, analyze_1.runQueries)(outputDir, memory, (0, util_1.getAddSnippetsFlag)(cmd.addSnippets), threads, cmd.category, config, logger);
         if (!cmd.upload) {
             logger.info("Not uploading results");

lib/util.js

@@ -552,10 +552,17 @@ exports.ML_POWERED_JS_QUERIES_PACK_NAME = "codeql/javascript-experimental-atm-qu
  * queries beta.
  */
 async function getMlPoweredJsQueriesPack(codeQL) {
-    if (await codeQlVersionAbove(codeQL, "2.8.4")) {
-        return `${exports.ML_POWERED_JS_QUERIES_PACK_NAME}@~0.2.0`;
+    let version;
+    if (await codeQlVersionAbove(codeQL, "2.9.3")) {
+        version = `~0.3.0`;
     }
-    return `${exports.ML_POWERED_JS_QUERIES_PACK_NAME}@~0.1.0`;
+    else if (await codeQlVersionAbove(codeQL, "2.8.4")) {
+        version = `~0.2.0`;
+    }
+    else {
+        version = `~0.1.0`;
+    }
+    return `${exports.ML_POWERED_JS_QUERIES_PACK_NAME}@${version}`;
 }
 exports.getMlPoweredJsQueriesPack = getMlPoweredJsQueriesPack;
 /**

node_modules/.package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "codeql",
-  "version": "2.1.12",
+  "version": "2.1.15",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
@@ -469,6 +469,12 @@
       "integrity": "sha512-rr+OQyAjxze7GgWrSaJwydHStIhHq2lvY3BOC2Mj7KnzI7XK0Uw1TOOdI9lDoajEbSWLiYgoo4f1R51erQfhPQ==",
       "dev": true
     },
+    "node_modules/@types/js-yaml": {
+      "version": "4.0.5",
+      "resolved": "https://registry.npmjs.org/@types/js-yaml/-/js-yaml-4.0.5.tgz",
+      "integrity": "sha512-FhpRzf927MNQdRZP0J5DLIdTXhjLYzeUTmLAu69mnVksLH9CJY3IuSeEgbKUki7GQZm0WqDkGzyxju2EZGD2wA==",
+      "dev": true
+    },
     "node_modules/@types/json-schema": {
       "version": "7.0.8",
       "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.8.tgz",

node_modules/@types/js-yaml/LICENSE

@@ -0,0 +1,21 @@
+    MIT License
+
+    Copyright (c) Microsoft Corporation.
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE

node_modules/@types/js-yaml/README.md

@@ -0,0 +1,16 @@
+# Installation
+> `npm install --save @types/js-yaml`
+
+# Summary
+This package contains type definitions for js-yaml (https://github.com/nodeca/js-yaml).
+
+# Details
+Files were exported from https://github.com/DefinitelyTyped/DefinitelyTyped/tree/master/types/js-yaml.
+
+### Additional Details
+ * Last updated: Fri, 19 Nov 2021 18:01:12 GMT
+ * Dependencies: none
+ * Global values: `jsyaml`
+
+# Credits
+These definitions were written by [Bart van der Schoor](https://github.com/Bartvds), [Sebastian Clausen](https://github.com/sclausen), [ExE Boss](https://github.com/ExE-Boss), [Armaan Tobaccowalla](https://github.com/ArmaanT), and [Linus Unnebäck](https://github.com/LinusU).

node_modules/@types/js-yaml/index.d.mts

@@ -0,0 +1,2 @@
+export * from "./index.js";
+export { default } from "./index.js";

node_modules/@types/js-yaml/index.d.ts

@@ -0,0 +1,154 @@
+// Type definitions for js-yaml 4.0
+// Project: https://github.com/nodeca/js-yaml
+// Definitions by: Bart van der Schoor <https://github.com/Bartvds>
+//                 Sebastian Clausen <https://github.com/sclausen>
+//                 ExE Boss <https://github.com/ExE-Boss>
+//                 Armaan Tobaccowalla <https://github.com/ArmaanT>
+//                 Linus Unnebäck <https://github.com/LinusU>
+// Definitions: https://github.com/DefinitelyTyped/DefinitelyTyped
+// TypeScript Version: 2.2
+
+export as namespace jsyaml;
+
+export function load(str: string, opts?: LoadOptions): unknown;
+
+export class Type {
+    constructor(tag: string, opts?: TypeConstructorOptions);
+    kind: 'sequence' | 'scalar' | 'mapping' | null;
+    resolve(data: any): boolean;
+    construct(data: any, type?: string): any;
+    instanceOf: object | null;
+    predicate: ((data: object) => boolean) | null;
+    represent: ((data: object) => any) | { [x: string]: (data: object) => any } | null;
+    representName: ((data: object) => any) | null;
+    defaultStyle: string | null;
+    multi: boolean;
+    styleAliases: { [x: string]: any };
+}
+
+export class Schema {
+    constructor(definition: SchemaDefinition | Type[] | Type);
+    extend(types: SchemaDefinition | Type[] | Type): Schema;
+}
+
+export function loadAll(str: string, iterator?: null, opts?: LoadOptions): unknown[];
+export function loadAll(str: string, iterator: (doc: unknown) => void, opts?: LoadOptions): void;
+
+export function dump(obj: any, opts?: DumpOptions): string;
+
+export interface LoadOptions {
+    /** string to be used as a file path in error/warning messages. */
+    filename?: string | undefined;
+    /** function to call on warning messages. */
+    onWarning?(this: null, e: YAMLException): void;
+    /** specifies a schema to use. */
+    schema?: Schema | undefined;
+    /** compatibility with JSON.parse behaviour. */
+    json?: boolean | undefined;
+    /** listener for parse events */
+    listener?(this: State, eventType: EventType, state: State): void;
+}
+
+export type EventType = 'open' | 'close';
+
+export interface State {
+    input: string;
+    filename: string | null;
+    schema: Schema;
+    onWarning: (this: null, e: YAMLException) => void;
+    json: boolean;
+    length: number;
+    position: number;
+    line: number;
+    lineStart: number;
+    lineIndent: number;
+    version: null | number;
+    checkLineBreaks: boolean;
+    kind: string;
+    result: any;
+    implicitTypes: Type[];
+}
+
+export interface DumpOptions {
+    /** indentation width to use (in spaces). */
+    indent?: number | undefined;
+    /** when true, will not add an indentation level to array elements */
+    noArrayIndent?: boolean | undefined;
+    /** do not throw on invalid types (like function in the safe schema) and skip pairs and single values with such types. */
+    skipInvalid?: boolean | undefined;
+    /** specifies level of nesting, when to switch from block to flow style for collections. -1 means block style everwhere */
+    flowLevel?: number | undefined;
+    /** Each tag may have own set of styles.    - "tag" => "style" map. */
+    styles?: { [x: string]: any } | undefined;
+    /** specifies a schema to use. */
+    schema?: Schema | undefined;
+    /** if true, sort keys when dumping YAML. If a function, use the function to sort the keys. (default: false) */
+    sortKeys?: boolean | ((a: any, b: any) => number) | undefined;
+    /** set max line width. (default: 80) */
+    lineWidth?: number | undefined;
+    /** if true, don't convert duplicate objects into references (default: false) */
+    noRefs?: boolean | undefined;
+    /** if true don't try to be compatible with older yaml versions. Currently: don't quote "yes", "no" and so on, as required for YAML 1.1 (default: false) */
+    noCompatMode?: boolean | undefined;
+    /**
+     * if true flow sequences will be condensed, omitting the space between `key: value` or `a, b`. Eg. `'[a,b]'` or `{a:{b:c}}`.
+     * Can be useful when using yaml for pretty URL query params as spaces are %-encoded. (default: false).
+     */
+    condenseFlow?: boolean | undefined;
+    /** strings will be quoted using this quoting style. If you specify single quotes, double quotes will still be used for non-printable characters. (default: `'`) */
+    quotingType?: "'" | '"' | undefined;
+    /** if true, all non-key strings will be quoted even if they normally don't need to. (default: false) */
+    forceQuotes?: boolean | undefined;
+    /** callback `function (key, value)` called recursively on each key/value in source object (see `replacer` docs for `JSON.stringify`). */
+    replacer?: ((key: string, value: any) => any) | undefined;
+}
+
+export interface TypeConstructorOptions {
+    kind?: 'sequence' | 'scalar' | 'mapping' | undefined;
+    resolve?: ((data: any) => boolean) | undefined;
+    construct?: ((data: any, type?: string) => any) | undefined;
+    instanceOf?: object | undefined;
+    predicate?: ((data: object) => boolean) | undefined;
+    represent?: ((data: object) => any) | { [x: string]: (data: object) => any } | undefined;
+    representName?: ((data: object) => any) | undefined;
+    defaultStyle?: string | undefined;
+    multi?: boolean | undefined;
+    styleAliases?: { [x: string]: any } | undefined;
+}
+
+export interface SchemaDefinition {
+    implicit?: Type[] | undefined;
+    explicit?: Type[] | undefined;
+}
+
+/** only strings, arrays and plain objects: http://www.yaml.org/spec/1.2/spec.html#id2802346 */
+export let FAILSAFE_SCHEMA: Schema;
+/** only strings, arrays and plain objects: http://www.yaml.org/spec/1.2/spec.html#id2802346 */
+export let JSON_SCHEMA: Schema;
+/** same as JSON_SCHEMA: http://www.yaml.org/spec/1.2/spec.html#id2804923 */
+export let CORE_SCHEMA: Schema;
+/** all supported YAML types */
+export let DEFAULT_SCHEMA: Schema;
+
+export interface Mark {
+    buffer: string;
+    column: number;
+    line: number;
+    name: string;
+    position: number;
+    snippet: string;
+}
+
+export class YAMLException extends Error {
+    constructor(reason?: string, mark?: Mark);
+
+    toString(compact?: boolean): string;
+
+    name: string;
+
+    reason: string;
+
+    message: string;
+
+    mark: Mark;
+}

node_modules/@types/js-yaml/package.json

@@ -0,0 +1,53 @@
+{
+  "name": "@types/js-yaml",
+  "version": "4.0.5",
+  "description": "TypeScript definitions for js-yaml",
+  "homepage": "https://github.com/DefinitelyTyped/DefinitelyTyped/tree/master/types/js-yaml",
+  "license": "MIT",
+  "contributors": [
+    {
+      "name": "Bart van der Schoor",
+      "url": "https://github.com/Bartvds",
+      "githubUsername": "Bartvds"
+    },
+    {
+      "name": "Sebastian Clausen",
+      "url": "https://github.com/sclausen",
+      "githubUsername": "sclausen"
+    },
+    {
+      "name": "ExE Boss",
+      "url": "https://github.com/ExE-Boss",
+      "githubUsername": "ExE-Boss"
+    },
+    {
+      "name": "Armaan Tobaccowalla",
+      "url": "https://github.com/ArmaanT",
+      "githubUsername": "ArmaanT"
+    },
+    {
+      "name": "Linus Unnebäck",
+      "url": "https://github.com/LinusU",
+      "githubUsername": "LinusU"
+    }
+  ],
+  "main": "",
+  "types": "index.d.ts",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/DefinitelyTyped/DefinitelyTyped.git",
+    "directory": "types/js-yaml"
+  },
+  "scripts": {},
+  "dependencies": {},
+  "typesPublisherContentHash": "6f40877154edac83ffa22d53a6aca74f151a0d094074c81ce7fb21df57ea5725",
+  "typeScriptVersion": "3.8",
+  "exports": {
+    ".": {
+      "types": {
+        "import": "./index.d.mts",
+        "default": "./index.d.ts"
+      }
+    }
+  }
+}

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "codeql",
-  "version": "2.1.12",
+  "version": "2.1.15",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "codeql",
-      "version": "2.1.12",
+      "version": "2.1.15",
       "license": "MIT",
       "dependencies": {
         "@actions/artifact": "^1.0.0",
@@ -38,6 +38,7 @@
       },
       "devDependencies": {
         "@ava/typescript": "3.0.1",
+        "@types/js-yaml": "^4.0.5",
         "@types/long": "4.0.1",
         "@types/node": "16.11.22",
         "@types/semver": "^7.3.8",
@@ -521,6 +522,12 @@
       "integrity": "sha512-rr+OQyAjxze7GgWrSaJwydHStIhHq2lvY3BOC2Mj7KnzI7XK0Uw1TOOdI9lDoajEbSWLiYgoo4f1R51erQfhPQ==",
       "dev": true
     },
+    "node_modules/@types/js-yaml": {
+      "version": "4.0.5",
+      "resolved": "https://registry.npmjs.org/@types/js-yaml/-/js-yaml-4.0.5.tgz",
+      "integrity": "sha512-FhpRzf927MNQdRZP0J5DLIdTXhjLYzeUTmLAu69mnVksLH9CJY3IuSeEgbKUki7GQZm0WqDkGzyxju2EZGD2wA==",
+      "dev": true
+    },
     "node_modules/@types/json-schema": {
       "version": "7.0.8",
       "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.8.tgz",
@@ -5885,6 +5892,12 @@
       "integrity": "sha512-rr+OQyAjxze7GgWrSaJwydHStIhHq2lvY3BOC2Mj7KnzI7XK0Uw1TOOdI9lDoajEbSWLiYgoo4f1R51erQfhPQ==",
       "dev": true
     },
+    "@types/js-yaml": {
+      "version": "4.0.5",
+      "resolved": "https://registry.npmjs.org/@types/js-yaml/-/js-yaml-4.0.5.tgz",
+      "integrity": "sha512-FhpRzf927MNQdRZP0J5DLIdTXhjLYzeUTmLAu69mnVksLH9CJY3IuSeEgbKUki7GQZm0WqDkGzyxju2EZGD2wA==",
+      "dev": true
+    },
     "@types/json-schema": {
       "version": "7.0.8",
       "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.8.tgz",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "codeql",
-  "version": "2.1.12",
+  "version": "2.1.15",
   "private": true,
   "description": "CodeQL action",
   "scripts": {
@@ -53,6 +53,7 @@
   ],
   "devDependencies": {
     "@ava/typescript": "3.0.1",
+    "@types/js-yaml": "^4.0.5",
     "@types/long": "4.0.1",
     "@types/node": "16.11.22",
     "@types/semver": "^7.3.8",

runner/package.json

@@ -7,11 +7,10 @@
     "build-runner": "webpack --mode production && pkg dist/codeql-runner.js  --out-path dist"
   },
   "license": "MIT",
-  "dependencies": {},
   "devDependencies": {
     "pkg": "^5.3.1",
-    "ts-loader": "9.2.5",
-    "webpack": "^5.50.0",
-    "webpack-cli": "^4.7.2"
+    "ts-loader": "^9.3.1",
+    "webpack": "^5.73.0",
+    "webpack-cli": "^4.10.0"
   }
 }

src/actions-util.test.ts

@@ -269,7 +269,7 @@ test("getWorkflowErrors() when on.push is correct with empty objects", (t) => {
 on:
   push:
   pull_request:
-`)
+`) as actionsutil.Workflow
   );
 
   t.deepEqual(...errorCodes(errors, []));
@@ -441,7 +441,7 @@ on:
   push:
     branches: ["main"]
   pull_request:
-`)
+`) as actionsutil.Workflow
   );
 
   t.deepEqual(
@@ -559,7 +559,7 @@ test("getWorkflowErrors() when branches contain dots", (t) => {
     pull_request:
       # The branches below must be a subset of the branches above
       branches: [4.1, master]
-`)
+`) as actionsutil.Workflow
   );
 
   t.deepEqual(...errorCodes(errors, []));
@@ -575,7 +575,7 @@ on:
   pull_request:
     # The branches below must be a subset of the branches above
     branches: [master]
-`)
+`) as actionsutil.Workflow
   );
 
   t.deepEqual(...errorCodes(errors, []));
@@ -604,7 +604,7 @@ jobs:
 
   test3:
     steps: []
-`)
+`) as actionsutil.Workflow
   );
 
   t.deepEqual(
@@ -635,7 +635,7 @@ jobs:
 
   test3:
     steps: []
-`)
+`) as actionsutil.Workflow
   );
 
   t.deepEqual(...errorCodes(errors, []));
@@ -645,7 +645,7 @@ test("getWorkflowErrors() when on is missing", (t) => {
   const errors = actionsutil.getWorkflowErrors(
     yaml.load(`
 name: "CodeQL"
-`)
+`) as actionsutil.Workflow
   );
 
   t.deepEqual(...errorCodes(errors, []));
@@ -658,7 +658,7 @@ test("getWorkflowErrors() with a different on setup", (t) => {
         yaml.load(`
 name: "CodeQL"
 on: "workflow_dispatch"
-`)
+`) as actionsutil.Workflow
       ),
       []
     )
@@ -670,7 +670,7 @@ on: "workflow_dispatch"
         yaml.load(`
 name: "CodeQL"
 on: [workflow_dispatch]
-`)
+`) as actionsutil.Workflow
       ),
       []
     )
@@ -683,7 +683,7 @@ on: [workflow_dispatch]
 name: "CodeQL"
 on:
   workflow_dispatch: {}
-`)
+`) as actionsutil.Workflow
       ),
       []
     )
@@ -699,7 +699,7 @@ name: "CodeQL"
 on:
   push:
     branches: [master]
-`)
+`) as actionsutil.Workflow
       ),
       []
     )
@@ -711,7 +711,7 @@ on:
         yaml.load(`
 name: "CodeQL"
 on: ["push"]
-`)
+`) as actionsutil.Workflow
       ),
       []
     )

src/actions-util.ts

@@ -191,7 +191,7 @@ interface WorkflowTriggers {
   pull_request?: WorkflowTrigger | null;
 }
 
-interface Workflow {
+export interface Workflow {
   jobs?: { [key: string]: WorkflowJob };
   on?: string | string[] | WorkflowTriggers;
 }
@@ -411,7 +411,7 @@ export async function getWorkflow(): Promise<Workflow> {
     relativePath
   );
 
-  return yaml.load(fs.readFileSync(absolutePath, "utf-8"));
+  return yaml.load(fs.readFileSync(absolutePath, "utf-8")) as Workflow;
 }
 
 /**

src/analyze-action.ts

@@ -12,9 +12,11 @@ import {
   runQueries,
   runFinalize,
 } from "./analyze";
+import { getGitHubVersionActionsOnly } from "./api-client";
 import { CODEQL_VERSION_NEW_TRACING, getCodeQL } from "./codeql";
 import { Config, getConfig } from "./config-utils";
 import { uploadDatabases } from "./database-upload";
+import { GitHubFeatureFlags } from "./feature-flags";
 import { getActionsLogger } from "./logging";
 import { parseRepositoryNwo } from "./repository";
 import * as upload_lib from "./upload-lib";
@@ -112,7 +114,16 @@ async function run() {
       util.getRequiredEnvParam("GITHUB_REPOSITORY")
     );
 
-    await runFinalize(outputDir, threads, memory, config, logger);
+    const gitHubVersion = await getGitHubVersionActionsOnly();
+
+    const featureFlags = new GitHubFeatureFlags(
+      gitHubVersion,
+      apiDetails,
+      repositoryNwo,
+      logger
+    );
+
+    await runFinalize(outputDir, threads, memory, config, logger, featureFlags);
     if (actionsUtil.getRequiredInput("skip-queries") !== "true") {
       runStats = await runQueries(
         outputDir,

src/analyze.test.ts

@@ -5,10 +5,12 @@ import test from "ava";
 import * as yaml from "js-yaml";
 import * as sinon from "sinon";
 
-import { runQueries } from "./analyze";
-import { setCodeQL } from "./codeql";
+import { runQueries, createdDBForScannedLanguages } from "./analyze";
+import { setCodeQL, getCodeQLForTesting } from "./codeql";
+import { stubToolRunnerConstructor } from "./codeql.test";
 import { Config } from "./config-utils";
 import * as count from "./count-loc";
+import { createFeatureFlags, FeatureFlag } from "./feature-flags";
 import { Language } from "./languages";
 import { getRunnerLogger } from "./logging";
 import { setupTests, setupActionsVars } from "./testing-utils";
@@ -249,3 +251,99 @@ test("status report fields and search path setting", async (t) => {
     }
   }
 });
+
+const stubConfig: Config = {
+  languages: [Language.cpp, Language.go],
+  queries: {},
+  pathsIgnore: [],
+  paths: [],
+  originalUserInput: {},
+  tempDir: "",
+  toolCacheDir: "",
+  codeQLCmd: "",
+  gitHubVersion: {
+    type: util.GitHubVariant.DOTCOM,
+  } as util.GitHubVersion,
+  dbLocation: "",
+  packs: {},
+  debugMode: false,
+  debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
+  debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
+  injectedMlQueries: false,
+};
+
+for (const options of [
+  {
+    name: "Lua feature flag enabled, but old CLI",
+    version: "2.9.0",
+    featureFlags: [FeatureFlag.LuaTracerConfigEnabled],
+    yesFlagSet: false,
+    noFlagSet: false,
+  },
+  {
+    name: "Lua feature flag disabled, with old CLI",
+    version: "2.9.0",
+    featureFlags: [],
+    yesFlagSet: false,
+    noFlagSet: false,
+  },
+  {
+    name: "Lua feature flag enabled, with new CLI",
+    version: "2.10.0",
+    featureFlags: [FeatureFlag.LuaTracerConfigEnabled],
+    yesFlagSet: true,
+    noFlagSet: false,
+  },
+  {
+    name: "Lua feature flag disabled, with new CLI",
+    version: "2.10.0",
+    featureFlags: [],
+    yesFlagSet: false,
+    noFlagSet: true,
+  },
+]) {
+  test(`createdDBForScannedLanguages() ${options.name}`, async (t) => {
+    const runnerConstructorStub = stubToolRunnerConstructor();
+    const codeqlObject = await getCodeQLForTesting("codeql/for-testing");
+    sinon.stub(codeqlObject, "getVersion").resolves(options.version);
+
+    const promise = createdDBForScannedLanguages(
+      codeqlObject,
+      stubConfig,
+      getRunnerLogger(true),
+      createFeatureFlags(options.featureFlags)
+    );
+    // call listener on `codeql resolve extractor`
+    const mockToolRunner = runnerConstructorStub.getCall(0);
+    mockToolRunner.args[2].listeners.stdout('"/path/to/extractor"');
+    await promise;
+    if (options.yesFlagSet)
+      t.true(
+        runnerConstructorStub.secondCall.args[1].includes(
+          "--internal-use-lua-tracing"
+        ),
+        "--internal-use-lua-tracing should be present, but it is absent"
+      );
+    else
+      t.false(
+        runnerConstructorStub.secondCall.args[1].includes(
+          "--internal-use-lua-tracing"
+        ),
+        "--internal-use-lua-tracing should be absent, but it is present"
+      );
+    if (options.noFlagSet)
+      t.true(
+        runnerConstructorStub.secondCall.args[1].includes(
+          "--no-internal-use-lua-tracing"
+        ),
+        "--no-internal-use-lua-tracing should be present, but it is absent"
+      );
+    else
+      t.false(
+        runnerConstructorStub.secondCall.args[1].includes(
+          "--no-internal-use-lua-tracing"
+        ),
+        "--no-internal-use-lua-tracing should be absent, but it is present"
+      );
+  });
+}

src/analyze.ts

@@ -2,16 +2,19 @@ import * as fs from "fs";
 import * as path from "path";
 
 import * as toolrunner from "@actions/exec/lib/toolrunner";
+import del from "del";
 import * as yaml from "js-yaml";
 
 import * as analysisPaths from "./analysis-paths";
 import {
+  CodeQL,
   CODEQL_VERSION_COUNTS_LINES,
   CODEQL_VERSION_NEW_TRACING,
   getCodeQL,
 } from "./codeql";
 import * as configUtils from "./config-utils";
 import { countLoc } from "./count-loc";
+import { FeatureFlags } from "./feature-flags";
 import { isScannedLanguage, Language } from "./languages";
 import { Logger } from "./logging";
 import * as sharedEnv from "./shared-environment";
@@ -113,15 +116,16 @@ async function setupPythonExtractor(logger: Logger) {
   process.env["LGTM_PYTHON_SETUP_VERSION"] = output;
 }
 
-async function createdDBForScannedLanguages(
+export async function createdDBForScannedLanguages(
+  codeql: CodeQL,
   config: configUtils.Config,
-  logger: Logger
+  logger: Logger,
+  featureFlags: FeatureFlags
 ) {
   // Insert the LGTM_INDEX_X env vars at this point so they are set when
   // we extract any scanned languages.
   analysisPaths.includeAndExcludeAnalysisPaths(config);
 
-  const codeql = await getCodeQL(config.codeQLCmd);
   for (const language of config.languages) {
     if (
       isScannedLanguage(language) &&
@@ -135,7 +139,8 @@ async function createdDBForScannedLanguages(
 
       await codeql.extractScannedLanguage(
         util.getCodeQLDatabasePath(config, language),
-        language
+        language,
+        featureFlags
       );
       logger.endGroup();
     }
@@ -151,7 +156,7 @@ function dbIsFinalized(
   try {
     const dbInfo = yaml.load(
       fs.readFileSync(path.resolve(dbPath, "codeql-database.yml"), "utf8")
-    );
+    ) as { inProgress?: boolean };
     return !("inProgress" in dbInfo);
   } catch (e) {
     logger.warning(
@@ -165,11 +170,12 @@ async function finalizeDatabaseCreation(
   config: configUtils.Config,
   threadsFlag: string,
   memoryFlag: string,
-  logger: Logger
+  logger: Logger,
+  featureFlags: FeatureFlags
 ) {
-  await createdDBForScannedLanguages(config, logger);
-
   const codeql = await getCodeQL(config.codeQLCmd);
+  await createdDBForScannedLanguages(codeql, config, logger, featureFlags);
+
   for (const language of config.languages) {
     if (dbIsFinalized(config, language, logger)) {
       logger.info(
@@ -424,7 +430,8 @@ export async function runFinalize(
   threadsFlag: string,
   memoryFlag: string,
   config: configUtils.Config,
-  logger: Logger
+  logger: Logger,
+  featureFlags: FeatureFlags
 ) {
   const codeql = await getCodeQL(config.codeQLCmd);
   if (await util.codeQlVersionAbove(codeql, CODEQL_VERSION_NEW_TRACING)) {
@@ -435,13 +442,8 @@ export async function runFinalize(
     delete process.env[sharedEnv.ODASA_TRACER_CONFIGURATION];
   }
 
-  // After switching to Node16, this entire block can be replaced with `await fs.promises.rm(outputDir, { recursive: true, force: true });`.
   try {
-    await fs.promises.rmdir(outputDir, {
-      recursive: true,
-      maxRetries: 5,
-      retryDelay: 2000,
-    } as any);
+    await del(outputDir, { force: true });
   } catch (error: any) {
     if (error?.code !== "ENOENT") {
       throw error;
@@ -449,7 +451,13 @@ export async function runFinalize(
   }
   await fs.promises.mkdir(outputDir, { recursive: true });
 
-  await finalizeDatabaseCreation(config, threadsFlag, memoryFlag, logger);
+  await finalizeDatabaseCreation(
+    config,
+    threadsFlag,
+    memoryFlag,
+    logger,
+    featureFlags
+  );
 }
 
 export async function runCleanup(

src/api-compatibility.json

@@ -1 +1 @@
-{"maximumVersion": "3.5", "minimumVersion": "3.1"}
+{"maximumVersion": "3.6", "minimumVersion": "3.2"}

src/codeql.test.ts

@@ -540,7 +540,7 @@ test("databaseInitCluster() Lua feature flag disabled, compatible CLI", async (t
   );
 });
 
-function stubToolRunnerConstructor(): sinon.SinonStub<
+export function stubToolRunnerConstructor(): sinon.SinonStub<
   any[],
   toolrunner.ToolRunner
 > {

src/codeql.ts

@@ -95,7 +95,11 @@ export interface CodeQL {
    * Extract code for a scanned language using 'codeql database trace-command'
    * and running the language extractor.
    */
-  extractScannedLanguage(database: string, language: Language): Promise<void>;
+  extractScannedLanguage(
+    database: string,
+    language: Language,
+    featureFlags: FeatureFlags
+  ): Promise<void>;
   /**
    * Finalize a database using 'codeql database finalize'.
    */
@@ -222,7 +226,7 @@ const CODEQL_VERSION_SARIF_GROUP = "2.5.3";
 export const CODEQL_VERSION_COUNTS_LINES = "2.6.2";
 const CODEQL_VERSION_CUSTOM_QUERY_HELP = "2.7.1";
 export const CODEQL_VERSION_ML_POWERED_QUERIES = "2.7.5";
-const CODEQL_VERSION_LUA_TRACER_CONFIG = "2.9.3";
+const CODEQL_VERSION_LUA_TRACER_CONFIG = "2.10.0";
 
 /**
  * This variable controls using the new style of tracing from the CodeQL
@@ -630,8 +634,10 @@ export function getCachedCodeQL(): CodeQL {
  * a non-existent placeholder codeql command, so tests that use this function
  * should also stub the toolrunner.ToolRunner constructor.
  */
-export async function getCodeQLForTesting(): Promise<CodeQL> {
-  return getCodeQLForCmd("codeql-for-testing", false);
+export async function getCodeQLForTesting(
+  cmd = "codeql-for-testing"
+): Promise<CodeQL> {
+  return getCodeQLForCmd(cmd, false);
 }
 
 /**
@@ -789,7 +795,11 @@ async function getCodeQLForCmd(
 
       await runTool(autobuildCmd);
     },
-    async extractScannedLanguage(databasePath: string, language: Language) {
+    async extractScannedLanguage(
+      databasePath: string,
+      language: Language,
+      featureFlags: FeatureFlags
+    ) {
       // Get extractor location
       let extractorPath = "";
       await new toolrunner.ToolRunner(
@@ -821,6 +831,16 @@ async function getCodeQLForCmd(
         "tools",
         `autobuild${ext}`
       );
+      const extraArgs: string[] = [];
+      if (
+        await util.codeQlVersionAbove(this, CODEQL_VERSION_LUA_TRACER_CONFIG)
+      ) {
+        if (await featureFlags.getValue(FeatureFlag.LuaTracerConfigEnabled)) {
+          extraArgs.push("--internal-use-lua-tracing");
+        } else {
+          extraArgs.push("--no-internal-use-lua-tracing");
+        }
+      }
 
       // Run trace command
       await toolrunnerErrorCatcher(
@@ -828,6 +848,7 @@ async function getCodeQLForCmd(
         [
           "database",
           "trace-command",
+          ...extraArgs,
           ...getExtraOptionsFromEnv(["database", "trace-command"]),
           databasePath,
           "--",

src/config-utils.test.ts

@@ -10,7 +10,7 @@ import { getCachedCodeQL, setCodeQL } from "./codeql";
 import * as configUtils from "./config-utils";
 import { createFeatureFlags, FeatureFlag } from "./feature-flags";
 import { Language } from "./languages";
-import { getRunnerLogger } from "./logging";
+import { getRunnerLogger, Logger } from "./logging";
 import { setupTests } from "./testing-utils";
 import * as util from "./util";
 
@@ -1424,7 +1424,12 @@ const parsePacksMacro = test.macro({
     expected: Partial<Record<Language, string[]>>
   ) =>
     t.deepEqual(
-      configUtils.parsePacksFromConfig(packsByLanguage, languages, "/a/b"),
+      configUtils.parsePacksFromConfig(
+        packsByLanguage,
+        languages,
+        "/a/b",
+        mockLogger
+      ),
       expected
     ),
 
@@ -1446,7 +1451,8 @@ const parsePacksErrorMacro = test.macro({
         configUtils.parsePacksFromConfig(
           packsByLanguage as string[] | Record<string, string[]>,
           languages,
-          "/a/b"
+          "/a/b",
+          {} as Logger
         ),
       {
         message: expected,
@@ -1499,6 +1505,19 @@ test(
   }
 );
 
+test(
+  "two packs with unused language in config",
+  parsePacksMacro,
+  {
+    [Language.cpp]: ["a/b", "c/d@1.2.3"],
+    [Language.java]: ["d/e", "f/g@1.2.3"],
+  },
+  [Language.cpp, Language.csharp],
+  {
+    [Language.cpp]: ["a/b", "c/d@1.2.3"],
+  }
+);
+
 test(
   "packs with other valid names",
   parsePacksMacro,
@@ -1544,13 +1563,6 @@ test(
   [Language.java, Language.python],
   /The configuration file "\/a\/b" is invalid: property "packs" must split packages by language/
 );
-test(
-  "invalid language",
-  parsePacksErrorMacro,
-  { [Language.java]: ["c/d"] },
-  [Language.cpp],
-  /The configuration file "\/a\/b" is invalid: property "packs" has "java", but it is not one of the languages to analyze/
-);
 test(
   "not an array",
   parsePacksErrorMacro,
@@ -1583,13 +1595,25 @@ function parseInputAndConfigMacro(
   expected
 ) {
   t.deepEqual(
-    configUtils.parsePacks(packsFromConfig, packsFromInput, languages, "/a/b"),
+    configUtils.parsePacks(
+      packsFromConfig,
+      packsFromInput,
+      languages,
+      "/a/b",
+      mockLogger
+    ),
     expected
   );
 }
 parseInputAndConfigMacro.title = (providedTitle: string) =>
   `Parse Packs input and config: ${providedTitle}`;
 
+const mockLogger = {
+  info: (message: string) => {
+    console.log(message);
+  },
+} as Logger;
+
 function parseInputAndConfigErrorMacro(
   t: ExecutionContext<unknown>,
   packsFromConfig: string[] | Record<string, string[]>,
@@ -1603,7 +1627,8 @@ function parseInputAndConfigErrorMacro(
         packsFromConfig,
         packsFromInput,
         languages,
-        "/a/b"
+        "/a/b",
+        mockLogger
       );
     },
     {
@@ -1865,3 +1890,23 @@ test(
   "security-and-quality",
   "0.0.1"
 );
+// Test that ML-powered queries are run on all platforms running `security-extended` on CodeQL
+// CLI 2.9.3+.
+test(
+  mlPoweredQueriesMacro,
+  "2.9.3",
+  true,
+  undefined,
+  "security-extended",
+  "~0.3.0"
+);
+// Test that ML-powered queries are run on all platforms running `security-and-quality` on CodeQL
+// CLI 2.9.3+.
+test(
+  mlPoweredQueriesMacro,
+  "2.9.3",
+  true,
+  undefined,
+  "security-and-quality",
+  "~0.3.0"
+);

src/config-utils.ts

@@ -629,14 +629,11 @@ export function getPathsInvalid(configFile: string): string {
   );
 }
 
-export function getPacksRequireLanguage(
-  lang: string,
-  configFile: string
-): string {
+function getPacksRequireLanguage(lang: string, configFile: string): string {
   return getConfigFilePropertyError(
     configFile,
     PACKS_PROPERTY,
-    `has "${lang}", but it is not one of the languages to analyze`
+    `has "${lang}", but it is not a valid language.`
   );
 }
 
@@ -1026,7 +1023,8 @@ async function loadConfig(
     parsedYAML[PACKS_PROPERTY] ?? {},
     packsInput,
     languages,
-    configFile
+    configFile,
+    logger
   );
 
   // If queries were provided using `with` in the action configuration,
@@ -1146,7 +1144,8 @@ const PACK_IDENTIFIER_PATTERN = (function () {
 export function parsePacksFromConfig(
   packsByLanguage: string[] | Record<string, string[]>,
   languages: Language[],
-  configFile: string
+  configFile: string,
+  logger: Logger
 ): Packs {
   const packs = {};
 
@@ -1168,7 +1167,16 @@ export function parsePacksFromConfig(
       throw new Error(getPacksInvalid(configFile));
     }
     if (!languages.includes(lang as Language)) {
-      throw new Error(getPacksRequireLanguage(lang, configFile));
+      // This particular language is not being analyzed in this run.
+      if (Language[lang as Language]) {
+        logger.info(
+          `Ignoring packs for ${lang} since this language is not being analyzed in this run.`
+        );
+        continue;
+      } else {
+        // This language is invalid, probably a misspelling
+        throw new Error(getPacksRequireLanguage(configFile, lang));
+      }
     }
     packs[lang] = [];
     for (const packStr of packsArr) {
@@ -1296,13 +1304,15 @@ export function parsePacks(
   rawPacksFromConfig: string[] | Record<string, string[]>,
   rawPacksInput: string | undefined,
   languages: Language[],
-  configFile: string
+  configFile: string,
+  logger: Logger
 ) {
   const packsFromInput = parsePacksFromInput(rawPacksInput, languages);
   const packsFomConfig = parsePacksFromConfig(
     rawPacksFromConfig,
     languages,
-    configFile
+    configFile,
+    logger
   );
 
   if (!packsFromInput) {
@@ -1448,7 +1458,7 @@ function getLocalConfig(configFile: string, workspacePath: string): UserConfig {
     throw new Error(getConfigFileDoesNotExistErrorMessage(configFile));
   }
 
-  return yaml.load(fs.readFileSync(configFile, "utf8"));
+  return yaml.load(fs.readFileSync(configFile, "utf8")) as UserConfig;
 }
 
 async function getRemoteConfig(
@@ -1483,7 +1493,9 @@ async function getRemoteConfig(
     throw new Error(getConfigFileFormatInvalidMessage(configFile));
   }
 
-  return yaml.load(Buffer.from(fileContents, "base64").toString("binary"));
+  return yaml.load(
+    Buffer.from(fileContents, "base64").toString("binary")
+  ) as UserConfig;
 }
 
 /**

src/defaults.json

@@ -1,3 +1,3 @@
 {
-  "bundleVersion": "codeql-bundle-20220512"
+  "bundleVersion": "codeql-bundle-20220623"
 }

src/languages.ts

@@ -7,6 +7,7 @@ export enum Language {
   javascript = "javascript",
   python = "python",
   ruby = "ruby",
+  swift = "swift",
 }
 
 // Additional names for languages
@@ -37,7 +38,7 @@ export function parseLanguage(language: string): Language | undefined {
 
 export function isTracedLanguage(language: Language): boolean {
   return (
-    ["cpp", "java", "csharp"].includes(language) ||
+    ["cpp", "java", "csharp", "swift"].includes(language) ||
     (process.env["CODEQL_EXTRACTOR_GO_BUILD_TRACING"] === "on" &&
       language === Language.go)
   );

src/runner.ts

@@ -501,7 +501,14 @@ program
         logger
       );
       const memory = getMemoryFlag(cmd.ram || initEnv["CODEQL_RAM"]);
-      await runFinalize(outputDir, threads, memory, config, logger);
+      await runFinalize(
+        outputDir,
+        threads,
+        memory,
+        config,
+        logger,
+        createFeatureFlags([])
+      );
       await runQueries(
         outputDir,
         memory,

src/util.ts

@@ -664,10 +664,15 @@ export const ML_POWERED_JS_QUERIES_PACK_NAME =
 export async function getMlPoweredJsQueriesPack(
   codeQL: CodeQL
 ): Promise<string> {
-  if (await codeQlVersionAbove(codeQL, "2.8.4")) {
-    return `${ML_POWERED_JS_QUERIES_PACK_NAME}@~0.2.0`;
+  let version;
+  if (await codeQlVersionAbove(codeQL, "2.9.3")) {
+    version = `~0.3.0`;
+  } else if (await codeQlVersionAbove(codeQL, "2.8.4")) {
+    version = `~0.2.0`;
+  } else {
+    version = `~0.1.0`;
   }
-  return `${ML_POWERED_JS_QUERIES_PACK_NAME}@~0.1.0`;
+  return `${ML_POWERED_JS_QUERIES_PACK_NAME}@${version}`;
 }
 
 /**

tests/multi-language-repo/.github/codeql/codeql-config-packaging.yml

@@ -6,6 +6,9 @@ packs:
     - dsp-testing/codeql-pack1@1.0.0
     - dsp-testing/codeql-pack2
     - dsp-testing/codeql-pack3:other-query.ql
+  ruby:
+    - dsp-testing/hucairz
+    - dsp-testing/i-dont-exist@1.0.0
 
 paths-ignore:
   - tests