Merge pull request #1547 from github/update-v2.2.5-237a258d2

Merge main into releases/v2

.github/workflows/script/check-node-modules.sh

@@ -10,6 +10,10 @@ fi
 # When updating this, make sure to update the npm version in
 # `.github/workflows/update-dependencies.yml` too.
 sudo npm install --force -g npm@9.2.0
+
+# clean the npm cache to ensure we don't have any files owned by root
+sudo npm cache clean --force
+
 # Reinstall modules and then clean to remove absolute paths
 # Use 'npm ci' instead of 'npm install' as this is intended to be reproducible
 npm ci

CHANGELOG.md

@@ -1,5 +1,9 @@
 # CodeQL Action Changelog
 
+## 2.2.5 - 24 Feb 2023
+
+- Update default CodeQL bundle version to 2.12.3. [#1543](https://github.com/github/codeql-action/pull/1543)
+
 ## 2.2.4 - 10 Feb 2023
 
 No user facing changes.

CONTRIBUTING.md

@@ -67,12 +67,8 @@ Here are a few things you can do that will increase the likelihood of your pull
     This mergeback incorporates the changelog updates into `main`, tags the release using the merge commit of the "Merge main into releases/v2" pull request, and bumps the patch version of the CodeQL Action.
 
     Approve the mergeback PR and automerge it.
-1. When the "Merge main into releases/v2" pull request is merged into the `releases/v2` branch, the "Update release branch" workflow will create a "Merge releases/v2 into releases/v1" pull request to merge the changes since the last release into the `releases/v1` release branch.
-    This ensures we keep both the `releases/v1` and `releases/v2` release branches up to date and fully supported.
 
-    Review the checklist items in the pull request description.
-    Once you've checked off all the items, approve the PR and automerge it.
-1. Once the mergeback has been merged to `main` and the "Merge releases/v2 into releases/v1" PR has been merged to `releases/v1`, the release is complete.
+Once the mergeback has been merged to `main`, the release is complete.
 
 ## Keeping the PR checks up to date (admin access required)
 

lib/analyze.js

@@ -126,6 +126,7 @@ async function finalizeDatabaseCreation(config, threadsFlag, memoryFlag, logger)
 async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag, automationDetailsId, config, logger, featureEnablement) {
     const statusReport = {};
     const codeql = await (0, codeql_1.getCodeQL)(config.codeQLCmd);
+    const queryFlags = [memoryFlag, threadsFlag];
     await util.logCodeScanningConfigInCli(codeql, featureEnablement, logger);
     for (const language of config.languages) {
         const queries = config.queries[language];
@@ -140,7 +141,7 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
                 // another to interpret the results.
                 logger.startGroup(`Running queries for ${language}`);
                 const startTimeBuiltIn = new Date().getTime();
-                await runQueryGroup(language, "all", undefined, undefined);
+                await runQueryGroup(language, "all", undefined, undefined, true);
                 // TODO should not be using `builtin` here. We should be using `all` instead.
                 // The status report does not support `all` yet.
                 statusReport[`analyze_builtin_queries_${language}_duration_ms`] =
@@ -164,24 +165,29 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
                     !hasPackWithCustomQueries) {
                     throw new Error(`Unable to analyze ${language} as no queries were selected for this language`);
                 }
+                const customQueryIndices = [];
+                for (let i = 0; i < queries.custom.length; ++i) {
+                    if (queries.custom[i].queries.length > 0) {
+                        customQueryIndices.push(i);
+                    }
+                }
                 logger.startGroup(`Running queries for ${language}`);
                 const querySuitePaths = [];
-                if (queries["builtin"].length > 0) {
+                if (queries.builtin.length > 0) {
                     const startTimeBuiltIn = new Date().getTime();
-                    querySuitePaths.push((await runQueryGroup(language, "builtin", createQuerySuiteContents(queries["builtin"], queryFilters), undefined)));
+                    querySuitePaths.push((await runQueryGroup(language, "builtin", createQuerySuiteContents(queries.builtin, queryFilters), undefined, customQueryIndices.length === 0 && packsWithVersion.length === 0)));
                     statusReport[`analyze_builtin_queries_${language}_duration_ms`] =
                         new Date().getTime() - startTimeBuiltIn;
                 }
                 const startTimeCustom = new Date().getTime();
                 let ranCustom = false;
-                for (let i = 0; i < queries["custom"].length; ++i) {
-                    if (queries["custom"][i].queries.length > 0) {
-                        querySuitePaths.push((await runQueryGroup(language, `custom-${i}`, createQuerySuiteContents(queries["custom"][i].queries, queryFilters), queries["custom"][i].searchPath)));
-                        ranCustom = true;
-                    }
+                for (const i of customQueryIndices) {
+                    querySuitePaths.push((await runQueryGroup(language, `custom-${i}`, createQuerySuiteContents(queries.custom[i].queries, queryFilters), queries.custom[i].searchPath, i === customQueryIndices[customQueryIndices.length - 1] &&
+                        packsWithVersion.length === 0)));
+                    ranCustom = true;
                 }
                 if (packsWithVersion.length > 0) {
-                    querySuitePaths.push(await runQueryPacks(language, "packs", packsWithVersion, queryFilters));
+                    querySuitePaths.push(await runQueryPacks(language, "packs", packsWithVersion, queryFilters, true));
                     ranCustom = true;
                 }
                 if (ranCustom) {
@@ -218,7 +224,7 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
         const databasePath = util.getCodeQLDatabasePath(config, language);
         return await codeql.databasePrintBaseline(databasePath);
     }
-    async function runQueryGroup(language, type, querySuiteContents, searchPath) {
+    async function runQueryGroup(language, type, querySuiteContents, searchPath, optimizeForLastQueryRun) {
         const databasePath = util.getCodeQLDatabasePath(config, language);
         // Pass the queries to codeql using a file instead of using the command
         // line to avoid command line length restrictions, particularly on windows.
@@ -229,11 +235,11 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
             fs.writeFileSync(querySuitePath, querySuiteContents);
             logger.debug(`Query suite file for ${language}-${type}...\n${querySuiteContents}`);
         }
-        await codeql.databaseRunQueries(databasePath, searchPath, querySuitePath, memoryFlag, threadsFlag);
+        await codeql.databaseRunQueries(databasePath, searchPath, querySuitePath, queryFlags, optimizeForLastQueryRun);
         logger.debug(`BQRS results produced for ${language} (queries: ${type})"`);
         return querySuitePath;
     }
-    async function runQueryPacks(language, type, packs, queryFilters) {
+    async function runQueryPacks(language, type, packs, queryFilters, optimizeForLastQueryRun) {
         const databasePath = util.getCodeQLDatabasePath(config, language);
         for (const pack of packs) {
             logger.debug(`Running query pack for ${language}-${type}: ${pack}`);
@@ -243,7 +249,7 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
         const querySuitePath = `${databasePath}-queries-${type}.qls`;
         fs.writeFileSync(querySuitePath, yaml.dump(querySuite));
         logger.debug(`BQRS results produced for ${language} (queries: ${type})"`);
-        await codeql.databaseRunQueries(databasePath, undefined, querySuitePath, memoryFlag, threadsFlag);
+        await codeql.databaseRunQueries(databasePath, undefined, querySuitePath, queryFlags, optimizeForLastQueryRun);
         return querySuitePath;
     }
 }

lib/analyze.test.js

@@ -30,8 +30,10 @@ const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
 const ava_1 = __importDefault(require("ava"));
 const yaml = __importStar(require("js-yaml"));
+const sinon = __importStar(require("sinon"));
 const analyze_1 = require("./analyze");
 const codeql_1 = require("./codeql");
+const feature_flags_1 = require("./feature-flags");
 const languages_1 = require("./languages");
 const logging_1 = require("./logging");
 const testing_utils_1 = require("./testing-utils");
@@ -188,6 +190,126 @@ const util = __importStar(require("./util"));
         }
     }
 });
+function mockCodeQL() {
+    return {
+        getVersion: async () => "2.12.2",
+        databaseRunQueries: sinon.spy(),
+        databaseInterpretResults: async () => "",
+        databasePrintBaseline: async () => "",
+    };
+}
+function createBaseConfig(tmpDir) {
+    return {
+        languages: [],
+        queries: {},
+        pathsIgnore: [],
+        paths: [],
+        originalUserInput: {},
+        tempDir: "tempDir",
+        codeQLCmd: "",
+        gitHubVersion: {
+            type: util.GitHubVariant.DOTCOM,
+        },
+        dbLocation: path.resolve(tmpDir, "codeql_databases"),
+        packs: {},
+        debugMode: false,
+        debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
+        debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
+        augmentationProperties: {
+            injectedMlQueries: false,
+            packsInputCombines: false,
+            queriesInputCombines: false,
+        },
+        trapCaches: {},
+        trapCacheDownloadTime: 0,
+    };
+}
+function createQueryConfig(builtin, custom) {
+    return {
+        builtin,
+        custom: custom.map((c) => ({ searchPath: "/search", queries: [c] })),
+    };
+}
+async function runQueriesWithConfig(config, features) {
+    for (const language of config.languages) {
+        fs.mkdirSync(util.getCodeQLDatabasePath(config, language), {
+            recursive: true,
+        });
+    }
+    return (0, analyze_1.runQueries)("sarif-folder", "--memFlag", "--addSnippetsFlag", "--threadsFlag", undefined, config, (0, logging_1.getRunnerLogger)(true), (0, testing_utils_1.createFeatures)(features));
+}
+function getDatabaseRunQueriesCalls(mock) {
+    return mock.databaseRunQueries.getCalls();
+}
+(0, ava_1.default)("optimizeForLastQueryRun for one language", async (t) => {
+    return await util.withTmpDir(async (tmpDir) => {
+        const codeql = mockCodeQL();
+        (0, codeql_1.setCodeQL)(codeql);
+        const config = createBaseConfig(tmpDir);
+        config.languages = [languages_1.Language.cpp];
+        config.queries.cpp = createQueryConfig(["foo.ql"], []);
+        await runQueriesWithConfig(config, []);
+        t.deepEqual(getDatabaseRunQueriesCalls(codeql).map((c) => c.args[4]), [true]);
+    });
+});
+(0, ava_1.default)("optimizeForLastQueryRun for two languages", async (t) => {
+    return await util.withTmpDir(async (tmpDir) => {
+        const codeql = mockCodeQL();
+        (0, codeql_1.setCodeQL)(codeql);
+        const config = createBaseConfig(tmpDir);
+        config.languages = [languages_1.Language.cpp, languages_1.Language.java];
+        config.queries.cpp = createQueryConfig(["foo.ql"], []);
+        config.queries.java = createQueryConfig(["bar.ql"], []);
+        await runQueriesWithConfig(config, []);
+        t.deepEqual(getDatabaseRunQueriesCalls(codeql).map((c) => c.args[4]), [true, true]);
+    });
+});
+(0, ava_1.default)("optimizeForLastQueryRun for two languages, with custom queries", async (t) => {
+    return await util.withTmpDir(async (tmpDir) => {
+        const codeql = mockCodeQL();
+        (0, codeql_1.setCodeQL)(codeql);
+        const config = createBaseConfig(tmpDir);
+        config.languages = [languages_1.Language.cpp, languages_1.Language.java];
+        config.queries.cpp = createQueryConfig(["foo.ql"], ["c1.ql", "c2.ql"]);
+        config.queries.java = createQueryConfig(["bar.ql"], ["c3.ql"]);
+        await runQueriesWithConfig(config, []);
+        t.deepEqual(getDatabaseRunQueriesCalls(codeql).map((c) => c.args[4]), [false, false, true, false, true]);
+    });
+});
+(0, ava_1.default)("optimizeForLastQueryRun for two languages, with custom queries and packs", async (t) => {
+    return await util.withTmpDir(async (tmpDir) => {
+        const codeql = mockCodeQL();
+        (0, codeql_1.setCodeQL)(codeql);
+        const config = createBaseConfig(tmpDir);
+        config.languages = [languages_1.Language.cpp, languages_1.Language.java];
+        config.queries.cpp = createQueryConfig(["foo.ql"], ["c1.ql", "c2.ql"]);
+        config.queries.java = createQueryConfig(["bar.ql"], ["c3.ql"]);
+        config.packs.cpp = ["a/cpp-pack1@0.1.0"];
+        config.packs.java = ["b/java-pack1@0.2.0", "b/java-pack2@0.3.3"];
+        await runQueriesWithConfig(config, []);
+        t.deepEqual(getDatabaseRunQueriesCalls(codeql).map((c) => c.args[4]), [false, false, false, true, false, false, true]);
+    });
+});
+(0, ava_1.default)("optimizeForLastQueryRun for one language, CliConfigFileEnabled", async (t) => {
+    return await util.withTmpDir(async (tmpDir) => {
+        const codeql = mockCodeQL();
+        (0, codeql_1.setCodeQL)(codeql);
+        const config = createBaseConfig(tmpDir);
+        config.languages = [languages_1.Language.cpp];
+        await runQueriesWithConfig(config, [feature_flags_1.Feature.CliConfigFileEnabled]);
+        t.deepEqual(getDatabaseRunQueriesCalls(codeql).map((c) => c.args[4]), [true]);
+    });
+});
+(0, ava_1.default)("optimizeForLastQueryRun for two languages, CliConfigFileEnabled", async (t) => {
+    return await util.withTmpDir(async (tmpDir) => {
+        const codeql = mockCodeQL();
+        (0, codeql_1.setCodeQL)(codeql);
+        const config = createBaseConfig(tmpDir);
+        config.languages = [languages_1.Language.cpp, languages_1.Language.java];
+        await runQueriesWithConfig(config, [feature_flags_1.Feature.CliConfigFileEnabled]);
+        t.deepEqual(getDatabaseRunQueriesCalls(codeql).map((c) => c.args[4]), [true, true]);
+    });
+});
 (0, ava_1.default)("validateQueryFilters", (t) => {
     t.notThrows(() => (0, analyze_1.validateQueryFilters)([]));
     t.notThrows(() => (0, analyze_1.validateQueryFilters)(undefined));

lib/codeql.js

@@ -472,17 +472,20 @@ async function getCodeQLForCmd(cmd, checkVersion) {
                 throw new Error(`Unexpected output from codeql resolve queries: ${e}`);
             }
         },
-        async databaseRunQueries(databasePath, extraSearchPath, querySuitePath, memoryFlag, threadsFlag) {
+        async databaseRunQueries(databasePath, extraSearchPath, querySuitePath, flags, optimizeForLastQueryRun) {
             const codeqlArgs = [
                 "database",
                 "run-queries",
-                memoryFlag,
-                threadsFlag,
+                ...flags,
                 databasePath,
                 "--min-disk-free=1024",
                 "-v",
                 ...getExtraOptionsFromEnv(["database", "run-queries"]),
             ];
+            if (optimizeForLastQueryRun &&
+                (await util.supportExpectDiscardedCache(this))) {
+                codeqlArgs.push("--expect-discarded-cache");
+            }
             if (extraSearchPath !== undefined) {
                 codeqlArgs.push("--additional-packs", extraSearchPath);
             }

lib/defaults.json

@@ -1,6 +1,6 @@
 {
-    "bundleVersion": "codeql-bundle-20230207",
-    "cliVersion": "2.12.2",
-    "priorBundleVersion": "codeql-bundle-20230120",
-    "priorCliVersion": "2.12.1"
+    "bundleVersion": "codeql-bundle-20230217",
+    "cliVersion": "2.12.3",
+    "priorBundleVersion": "codeql-bundle-20230207",
+    "priorCliVersion": "2.12.2"
 }

lib/feature-flags.js

@@ -37,7 +37,6 @@ var Feature;
     Feature["CliConfigFileEnabled"] = "cli_config_file_enabled";
     Feature["DisableKotlinAnalysisEnabled"] = "disable_kotlin_analysis_enabled";
     Feature["MlPoweredQueriesEnabled"] = "ml_powered_queries_enabled";
-    Feature["TrapCachingEnabled"] = "trap_caching_enabled";
     Feature["UploadFailedSarifEnabled"] = "upload_failed_sarif_enabled";
 })(Feature = exports.Feature || (exports.Feature = {}));
 exports.featureConfig = {
@@ -53,10 +52,6 @@ exports.featureConfig = {
         envVar: "CODEQL_ML_POWERED_QUERIES",
         minimumVersion: "2.7.5",
     },
-    [Feature.TrapCachingEnabled]: {
-        envVar: "CODEQL_TRAP_CACHING",
-        minimumVersion: undefined,
-    },
     [Feature.UploadFailedSarifEnabled]: {
         envVar: "CODEQL_ACTION_UPLOAD_FAILED_SARIF",
         minimumVersion: "2.11.3",

lib/init-action.js

@@ -46,12 +46,13 @@ async function sendInitStatusReport(actionStatus, startedAt, config, toolsDownlo
         tools_source: toolsSource || init_1.ToolsSource.Unknown,
         workflow_languages: workflowLanguages || "",
     };
-    let initToolsDownloadFields = {};
-    if (toolsSource === init_1.ToolsSource.Download) {
-        initToolsDownloadFields = {
-            tools_download_duration_ms: toolsDownloadDurationMs,
-            tools_feature_flags_valid: toolsFeatureFlagsValid,
-        };
+    const initToolsDownloadFields = {};
+    if (toolsDownloadDurationMs !== undefined) {
+        initToolsDownloadFields.tools_download_duration_ms =
+            toolsDownloadDurationMs;
+    }
+    if (toolsFeatureFlagsValid !== undefined) {
+        initToolsDownloadFields.tools_feature_flags_valid = toolsFeatureFlagsValid;
     }
     if (config !== undefined) {
         const languages = config.languages.join(",");
@@ -128,7 +129,7 @@ async function run() {
         toolsVersion = initCodeQLResult.toolsVersion;
         toolsSource = initCodeQLResult.toolsSource;
         await (0, util_1.enrichEnvironment)(codeql);
-        config = await (0, init_1.initConfig)((0, actions_util_1.getOptionalInput)("languages"), (0, actions_util_1.getOptionalInput)("queries"), (0, actions_util_1.getOptionalInput)("packs"), (0, actions_util_1.getOptionalInput)("registries"), (0, actions_util_1.getOptionalInput)("config-file"), (0, actions_util_1.getOptionalInput)("db-location"), await getTrapCachingEnabled(features), 
+        config = await (0, init_1.initConfig)((0, actions_util_1.getOptionalInput)("languages"), (0, actions_util_1.getOptionalInput)("queries"), (0, actions_util_1.getOptionalInput)("packs"), (0, actions_util_1.getOptionalInput)("registries"), (0, actions_util_1.getOptionalInput)("config-file"), (0, actions_util_1.getOptionalInput)("db-location"), getTrapCachingEnabled(), 
         // Debug mode is enabled if:
         // - The `init` Action is passed `debug: true`.
         // - Actions step debugging is enabled (e.g. by [enabling debug logging for a rerun](https://docs.github.com/en/actions/managing-workflow-runs/re-running-workflows-and-jobs#re-running-all-the-jobs-in-a-workflow),
@@ -192,7 +193,7 @@ async function run() {
     }
     await sendInitStatusReport("success", startedAt, config, toolsDownloadDurationMs, toolsFeatureFlagsValid, toolsSource, toolsVersion, logger);
 }
-async function getTrapCachingEnabled(featureEnablement) {
+function getTrapCachingEnabled() {
     // If the workflow specified something always respect that
     const trapCaching = (0, actions_util_1.getOptionalInput)("trap-caching");
     if (trapCaching !== undefined)
@@ -200,8 +201,8 @@ async function getTrapCachingEnabled(featureEnablement) {
     // On self-hosted runners which may have slow network access, disable TRAP caching by default
     if (!(0, util_1.isHostedRunner)())
         return false;
-    // On hosted runners, respect the feature flag
-    return await featureEnablement.getValue(feature_flags_1.Feature.TrapCachingEnabled);
+    // On hosted runners, enable TRAP caching by default
+    return true;
 }
 async function runWrapper() {
     try {

lib/upload-lib.js

@@ -330,7 +330,9 @@ async function waitForProcessing(repositoryNwo, sarifID, logger, options = {
             else {
                 util.assertNever(status);
             }
-            await util.delay(STATUS_CHECK_FREQUENCY_MILLISECONDS);
+            await util.delay(STATUS_CHECK_FREQUENCY_MILLISECONDS, {
+                allowProcessExit: false,
+            });
         }
     }
     finally {

lib/util.js

@@ -26,7 +26,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.parseMatrixInput = exports.isHostedRunner = exports.checkForTimeout = exports.withTimeout = exports.tryGetFolderBytes = exports.listFolder = exports.doesDirectoryExist = exports.logCodeScanningConfigInCli = exports.useCodeScanningConfigInCli = exports.isInTestMode = exports.getMlPoweredJsQueriesStatus = exports.getMlPoweredJsQueriesPack = exports.ML_POWERED_JS_QUERIES_PACK_NAME = exports.isGoodVersion = exports.delay = exports.bundleDb = exports.codeQlVersionAbove = exports.getCachedCodeQlVersion = exports.cacheCodeQlVersion = exports.isHTTPError = exports.UserError = exports.HTTPError = exports.getRequiredEnvParam = exports.enrichEnvironment = exports.initializeEnvironment = exports.EnvVar = exports.assertNever = exports.apiVersionInRange = exports.DisallowedAPIVersionReason = exports.checkGitHubVersionInRange = exports.getGitHubVersion = exports.GitHubVariant = exports.parseGitHubUrl = exports.getCodeQLDatabasePath = exports.getThreadsFlag = exports.getThreadsFlagValue = exports.getAddSnippetsFlag = exports.getMemoryFlag = exports.getMemoryFlagValue = exports.withTmpDir = exports.getToolNames = exports.getExtraOptionsEnvParam = exports.DID_AUTOBUILD_GO_ENV_VAR_NAME = exports.DEFAULT_DEBUG_DATABASE_NAME = exports.DEFAULT_DEBUG_ARTIFACT_NAME = exports.GITHUB_DOTCOM_URL = void 0;
+exports.parseMatrixInput = exports.isHostedRunner = exports.checkForTimeout = exports.withTimeout = exports.tryGetFolderBytes = exports.listFolder = exports.doesDirectoryExist = exports.logCodeScanningConfigInCli = exports.useCodeScanningConfigInCli = exports.isInTestMode = exports.getMlPoweredJsQueriesStatus = exports.getMlPoweredJsQueriesPack = exports.ML_POWERED_JS_QUERIES_PACK_NAME = exports.supportExpectDiscardedCache = exports.isGoodVersion = exports.delay = exports.bundleDb = exports.codeQlVersionAbove = exports.getCachedCodeQlVersion = exports.cacheCodeQlVersion = exports.isHTTPError = exports.UserError = exports.HTTPError = exports.getRequiredEnvParam = exports.enrichEnvironment = exports.initializeEnvironment = exports.EnvVar = exports.assertNever = exports.apiVersionInRange = exports.DisallowedAPIVersionReason = exports.checkGitHubVersionInRange = exports.getGitHubVersion = exports.GitHubVariant = exports.parseGitHubUrl = exports.getCodeQLDatabasePath = exports.getThreadsFlag = exports.getThreadsFlagValue = exports.getAddSnippetsFlag = exports.getMemoryFlag = exports.getMemoryFlagValue = exports.withTmpDir = exports.getToolNames = exports.getExtraOptionsEnvParam = exports.DID_AUTOBUILD_GO_ENV_VAR_NAME = exports.DEFAULT_DEBUG_DATABASE_NAME = exports.DEFAULT_DEBUG_ARTIFACT_NAME = exports.GITHUB_DOTCOM_URL = void 0;
 const fs = __importStar(require("fs"));
 const os = __importStar(require("os"));
 const path = __importStar(require("path"));
@@ -455,16 +455,33 @@ async function bundleDb(config, language, codeql, dbName) {
     return databaseBundlePath;
 }
 exports.bundleDb = bundleDb;
-async function delay(milliseconds) {
-    // Immediately `unref` the timer such that it only prevents the process from exiting if the
-    // surrounding promise is being awaited.
-    return new Promise((resolve) => setTimeout(resolve, milliseconds).unref());
+/**
+ * @param milliseconds time to delay
+ * @param opts options
+ * @param opts.allowProcessExit if true, the timer will not prevent the process from exiting
+ */
+async function delay(milliseconds, { allowProcessExit }) {
+    return new Promise((resolve) => {
+        const timer = setTimeout(resolve, milliseconds);
+        if (allowProcessExit) {
+            // Immediately `unref` the timer such that it only prevents the process from exiting if the
+            // surrounding promise is being awaited.
+            timer.unref();
+        }
+    });
 }
 exports.delay = delay;
 function isGoodVersion(versionSpec) {
     return !BROKEN_VERSIONS.includes(versionSpec);
 }
 exports.isGoodVersion = isGoodVersion;
+/**
+ * Checks whether the CodeQL CLI supports the `--expect-discarded-cache` command-line flag.
+ */
+async function supportExpectDiscardedCache(codeQL) {
+    return codeQlVersionAbove(codeQL, "2.12.1");
+}
+exports.supportExpectDiscardedCache = supportExpectDiscardedCache;
 exports.ML_POWERED_JS_QUERIES_PACK_NAME = "codeql/javascript-experimental-atm-queries";
 /**
  * Gets the ML-powered JS query pack to add to the analysis if a repo is opted into the ML-powered
@@ -636,7 +653,7 @@ async function withTimeout(timeoutMs, promise, onTimeout) {
         return result;
     };
     const timeoutTask = async () => {
-        await delay(timeoutMs);
+        await delay(timeoutMs, { allowProcessExit: true });
         if (!finished) {
             // Workaround: While the promise racing below will allow the main code
             // to continue, the process won't normally exit until the asynchronous
@@ -659,7 +676,7 @@ exports.withTimeout = withTimeout;
 async function checkForTimeout() {
     if (hadTimeout === true) {
         core.info("A timeout occurred, force exiting the process after 30 seconds to prevent hanging.");
-        await delay(30000);
+        await delay(30000, { allowProcessExit: true });
         process.exit();
     }
 }

node_modules/.package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "codeql",
-  "version": "2.2.4",
+  "version": "2.2.5",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "codeql",
-  "version": "2.2.4",
+  "version": "2.2.5",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "codeql",
-      "version": "2.2.4",
+      "version": "2.2.5",
       "license": "MIT",
       "dependencies": {
         "@actions/artifact": "^1.1.0",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "codeql",
-  "version": "2.2.4",
+  "version": "2.2.5",
   "private": true,
   "description": "CodeQL action",
   "scripts": {

src/analyze.test.ts

@@ -3,15 +3,18 @@ import * as path from "path";
 
 import test, { ExecutionContext } from "ava";
 import * as yaml from "js-yaml";
+import * as sinon from "sinon";
 
 import {
   convertPackToQuerySuiteEntry,
   createQuerySuiteContents,
   runQueries,
   validateQueryFilters,
+  QueriesStatusReport,
 } from "./analyze";
-import { setCodeQL } from "./codeql";
-import { Config } from "./config-utils";
+import { CodeQL, setCodeQL } from "./codeql";
+import { Config, QueriesWithSearchPath } from "./config-utils";
+import { Feature } from "./feature-flags";
 import { Language } from "./languages";
 import { getRunnerLogger } from "./logging";
 import { setupTests, setupActionsVars, createFeatures } from "./testing-utils";
@@ -229,6 +232,175 @@ test("status report fields and search path setting", async (t) => {
   }
 });
 
+function mockCodeQL(): Partial<CodeQL> {
+  return {
+    getVersion: async () => "2.12.2",
+    databaseRunQueries: sinon.spy(),
+    databaseInterpretResults: async () => "",
+    databasePrintBaseline: async () => "",
+  };
+}
+
+function createBaseConfig(tmpDir: string): Config {
+  return {
+    languages: [],
+    queries: {},
+    pathsIgnore: [],
+    paths: [],
+    originalUserInput: {},
+    tempDir: "tempDir",
+    codeQLCmd: "",
+    gitHubVersion: {
+      type: util.GitHubVariant.DOTCOM,
+    } as util.GitHubVersion,
+    dbLocation: path.resolve(tmpDir, "codeql_databases"),
+    packs: {},
+    debugMode: false,
+    debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
+    debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
+    augmentationProperties: {
+      injectedMlQueries: false,
+      packsInputCombines: false,
+      queriesInputCombines: false,
+    },
+    trapCaches: {},
+    trapCacheDownloadTime: 0,
+  };
+}
+
+function createQueryConfig(
+  builtin: string[],
+  custom: string[]
+): { builtin: string[]; custom: QueriesWithSearchPath[] } {
+  return {
+    builtin,
+    custom: custom.map((c) => ({ searchPath: "/search", queries: [c] })),
+  };
+}
+
+async function runQueriesWithConfig(
+  config: Config,
+  features: Feature[]
+): Promise<QueriesStatusReport> {
+  for (const language of config.languages) {
+    fs.mkdirSync(util.getCodeQLDatabasePath(config, language), {
+      recursive: true,
+    });
+  }
+  return runQueries(
+    "sarif-folder",
+    "--memFlag",
+    "--addSnippetsFlag",
+    "--threadsFlag",
+    undefined,
+    config,
+    getRunnerLogger(true),
+    createFeatures(features)
+  );
+}
+
+function getDatabaseRunQueriesCalls(mock: Partial<CodeQL>) {
+  return (mock.databaseRunQueries as sinon.SinonSpy).getCalls();
+}
+
+test("optimizeForLastQueryRun for one language", async (t) => {
+  return await util.withTmpDir(async (tmpDir) => {
+    const codeql = mockCodeQL();
+    setCodeQL(codeql);
+    const config: Config = createBaseConfig(tmpDir);
+    config.languages = [Language.cpp];
+    config.queries.cpp = createQueryConfig(["foo.ql"], []);
+
+    await runQueriesWithConfig(config, []);
+    t.deepEqual(
+      getDatabaseRunQueriesCalls(codeql).map((c) => c.args[4]),
+      [true]
+    );
+  });
+});
+
+test("optimizeForLastQueryRun for two languages", async (t) => {
+  return await util.withTmpDir(async (tmpDir) => {
+    const codeql = mockCodeQL();
+    setCodeQL(codeql);
+    const config: Config = createBaseConfig(tmpDir);
+    config.languages = [Language.cpp, Language.java];
+    config.queries.cpp = createQueryConfig(["foo.ql"], []);
+    config.queries.java = createQueryConfig(["bar.ql"], []);
+
+    await runQueriesWithConfig(config, []);
+    t.deepEqual(
+      getDatabaseRunQueriesCalls(codeql).map((c) => c.args[4]),
+      [true, true]
+    );
+  });
+});
+
+test("optimizeForLastQueryRun for two languages, with custom queries", async (t) => {
+  return await util.withTmpDir(async (tmpDir) => {
+    const codeql = mockCodeQL();
+    setCodeQL(codeql);
+    const config: Config = createBaseConfig(tmpDir);
+    config.languages = [Language.cpp, Language.java];
+    config.queries.cpp = createQueryConfig(["foo.ql"], ["c1.ql", "c2.ql"]);
+    config.queries.java = createQueryConfig(["bar.ql"], ["c3.ql"]);
+
+    await runQueriesWithConfig(config, []);
+    t.deepEqual(
+      getDatabaseRunQueriesCalls(codeql).map((c) => c.args[4]),
+      [false, false, true, false, true]
+    );
+  });
+});
+
+test("optimizeForLastQueryRun for two languages, with custom queries and packs", async (t) => {
+  return await util.withTmpDir(async (tmpDir) => {
+    const codeql = mockCodeQL();
+    setCodeQL(codeql);
+    const config: Config = createBaseConfig(tmpDir);
+    config.languages = [Language.cpp, Language.java];
+    config.queries.cpp = createQueryConfig(["foo.ql"], ["c1.ql", "c2.ql"]);
+    config.queries.java = createQueryConfig(["bar.ql"], ["c3.ql"]);
+    config.packs.cpp = ["a/cpp-pack1@0.1.0"];
+    config.packs.java = ["b/java-pack1@0.2.0", "b/java-pack2@0.3.3"];
+    await runQueriesWithConfig(config, []);
+    t.deepEqual(
+      getDatabaseRunQueriesCalls(codeql).map((c) => c.args[4]),
+      [false, false, false, true, false, false, true]
+    );
+  });
+});
+
+test("optimizeForLastQueryRun for one language, CliConfigFileEnabled", async (t) => {
+  return await util.withTmpDir(async (tmpDir) => {
+    const codeql = mockCodeQL();
+    setCodeQL(codeql);
+    const config: Config = createBaseConfig(tmpDir);
+    config.languages = [Language.cpp];
+
+    await runQueriesWithConfig(config, [Feature.CliConfigFileEnabled]);
+    t.deepEqual(
+      getDatabaseRunQueriesCalls(codeql).map((c) => c.args[4]),
+      [true]
+    );
+  });
+});
+
+test("optimizeForLastQueryRun for two languages, CliConfigFileEnabled", async (t) => {
+  return await util.withTmpDir(async (tmpDir) => {
+    const codeql = mockCodeQL();
+    setCodeQL(codeql);
+    const config: Config = createBaseConfig(tmpDir);
+    config.languages = [Language.cpp, Language.java];
+
+    await runQueriesWithConfig(config, [Feature.CliConfigFileEnabled]);
+    t.deepEqual(
+      getDatabaseRunQueriesCalls(codeql).map((c) => c.args[4]),
+      [true, true]
+    );
+  });
+});
+
 test("validateQueryFilters", (t) => {
   t.notThrows(() => validateQueryFilters([]));
   t.notThrows(() => validateQueryFilters(undefined));

src/analyze.ts

@@ -212,6 +212,7 @@ export async function runQueries(
   const statusReport: QueriesStatusReport = {};
 
   const codeql = await getCodeQL(config.codeQLCmd);
+  const queryFlags = [memoryFlag, threadsFlag];
 
   await util.logCodeScanningConfigInCli(codeql, featureEnablement, logger);
 
@@ -231,7 +232,7 @@ export async function runQueries(
         // another to interpret the results.
         logger.startGroup(`Running queries for ${language}`);
         const startTimeBuiltIn = new Date().getTime();
-        await runQueryGroup(language, "all", undefined, undefined);
+        await runQueryGroup(language, "all", undefined, undefined, true);
         // TODO should not be using `builtin` here. We should be using `all` instead.
         // The status report does not support `all` yet.
         statusReport[`analyze_builtin_queries_${language}_duration_ms`] =
@@ -267,16 +268,24 @@ export async function runQueries(
           );
         }
 
+        const customQueryIndices: number[] = [];
+        for (let i = 0; i < queries.custom.length; ++i) {
+          if (queries.custom[i].queries.length > 0) {
+            customQueryIndices.push(i);
+          }
+        }
+
         logger.startGroup(`Running queries for ${language}`);
         const querySuitePaths: string[] = [];
-        if (queries["builtin"].length > 0) {
+        if (queries.builtin.length > 0) {
           const startTimeBuiltIn = new Date().getTime();
           querySuitePaths.push(
             (await runQueryGroup(
               language,
               "builtin",
-              createQuerySuiteContents(queries["builtin"], queryFilters),
-              undefined
+              createQuerySuiteContents(queries.builtin, queryFilters),
+              undefined,
+              customQueryIndices.length === 0 && packsWithVersion.length === 0
             )) as string
           );
           statusReport[`analyze_builtin_queries_${language}_duration_ms`] =
@@ -284,21 +293,18 @@ export async function runQueries(
         }
         const startTimeCustom = new Date().getTime();
         let ranCustom = false;
-        for (let i = 0; i < queries["custom"].length; ++i) {
-          if (queries["custom"][i].queries.length > 0) {
-            querySuitePaths.push(
-              (await runQueryGroup(
-                language,
-                `custom-${i}`,
-                createQuerySuiteContents(
-                  queries["custom"][i].queries,
-                  queryFilters
-                ),
-                queries["custom"][i].searchPath
-              )) as string
-            );
-            ranCustom = true;
-          }
+        for (const i of customQueryIndices) {
+          querySuitePaths.push(
+            (await runQueryGroup(
+              language,
+              `custom-${i}`,
+              createQuerySuiteContents(queries.custom[i].queries, queryFilters),
+              queries.custom[i].searchPath,
+              i === customQueryIndices[customQueryIndices.length - 1] &&
+                packsWithVersion.length === 0
+            )) as string
+          );
+          ranCustom = true;
         }
         if (packsWithVersion.length > 0) {
           querySuitePaths.push(
@@ -306,7 +312,8 @@ export async function runQueries(
               language,
               "packs",
               packsWithVersion,
-              queryFilters
+              queryFilters,
+              true
             )
           );
           ranCustom = true;
@@ -373,7 +380,8 @@ export async function runQueries(
     language: Language,
     type: string,
     querySuiteContents: string | undefined,
-    searchPath: string | undefined
+    searchPath: string | undefined,
+    optimizeForLastQueryRun: boolean
   ): Promise<string | undefined> {
     const databasePath = util.getCodeQLDatabasePath(config, language);
     // Pass the queries to codeql using a file instead of using the command
@@ -391,8 +399,8 @@ export async function runQueries(
       databasePath,
       searchPath,
       querySuitePath,
-      memoryFlag,
-      threadsFlag
+      queryFlags,
+      optimizeForLastQueryRun
     );
 
     logger.debug(`BQRS results produced for ${language} (queries: ${type})"`);
@@ -402,7 +410,8 @@ export async function runQueries(
     language: Language,
     type: string,
     packs: string[],
-    queryFilters: configUtils.QueryFilter[]
+    queryFilters: configUtils.QueryFilter[],
+    optimizeForLastQueryRun: boolean
   ): Promise<string> {
     const databasePath = util.getCodeQLDatabasePath(config, language);
 
@@ -424,8 +433,8 @@ export async function runQueries(
       databasePath,
       undefined,
       querySuitePath,
-      memoryFlag,
-      threadsFlag
+      queryFlags,
+      optimizeForLastQueryRun
     );
 
     return querySuitePath;

src/codeql.ts

@@ -148,13 +148,19 @@ export interface CodeQL {
   ): Promise<void>;
   /**
    * Run 'codeql database run-queries'.
+   *
+   * @param optimizeForLastQueryRun Whether to apply additional optimization for
+   *                                the last database query run in the action.
+   *                                It is always safe to set it to false.
+   *                                It should be set to true only for the very
+   *                                last databaseRunQueries() call.
    */
   databaseRunQueries(
     databasePath: string,
     extraSearchPath: string | undefined,
     querySuitePath: string | undefined,
-    memoryFlag: string,
-    threadsFlag: string
+    flags: string[],
+    optimizeForLastQueryRun: boolean
   ): Promise<void>;
   /**
    * Run 'codeql database interpret-results'.
@@ -789,19 +795,24 @@ export async function getCodeQLForCmd(
       databasePath: string,
       extraSearchPath: string | undefined,
       querySuitePath: string | undefined,
-      memoryFlag: string,
-      threadsFlag: string
+      flags: string[],
+      optimizeForLastQueryRun: boolean
     ): Promise<void> {
       const codeqlArgs = [
         "database",
         "run-queries",
-        memoryFlag,
-        threadsFlag,
+        ...flags,
         databasePath,
         "--min-disk-free=1024", // Try to leave at least 1GB free
         "-v",
         ...getExtraOptionsFromEnv(["database", "run-queries"]),
       ];
+      if (
+        optimizeForLastQueryRun &&
+        (await util.supportExpectDiscardedCache(this))
+      ) {
+        codeqlArgs.push("--expect-discarded-cache");
+      }
       if (extraSearchPath !== undefined) {
         codeqlArgs.push("--additional-packs", extraSearchPath);
       }

src/defaults.json

@@ -1,6 +1,6 @@
 {
-  "bundleVersion": "codeql-bundle-20230207",
-  "cliVersion": "2.12.2",
-  "priorBundleVersion": "codeql-bundle-20230120",
-  "priorCliVersion": "2.12.1"
+  "bundleVersion": "codeql-bundle-20230217",
+  "cliVersion": "2.12.3",
+  "priorBundleVersion": "codeql-bundle-20230207",
+  "priorCliVersion": "2.12.2"
 }

src/feature-flags.ts

@@ -37,7 +37,6 @@ export enum Feature {
   CliConfigFileEnabled = "cli_config_file_enabled",
   DisableKotlinAnalysisEnabled = "disable_kotlin_analysis_enabled",
   MlPoweredQueriesEnabled = "ml_powered_queries_enabled",
-  TrapCachingEnabled = "trap_caching_enabled",
   UploadFailedSarifEnabled = "upload_failed_sarif_enabled",
 }
 
@@ -57,10 +56,6 @@ export const featureConfig: Record<
     envVar: "CODEQL_ML_POWERED_QUERIES",
     minimumVersion: "2.7.5",
   },
-  [Feature.TrapCachingEnabled]: {
-    envVar: "CODEQL_TRAP_CACHING",
-    minimumVersion: undefined,
-  },
   [Feature.UploadFailedSarifEnabled]: {
     envVar: "CODEQL_ACTION_UPLOAD_FAILED_SARIF",
     minimumVersion: "2.11.3",

src/init-action.ts

@@ -16,7 +16,7 @@ import {
 import { getGitHubVersion } from "./api-client";
 import { CodeQL, CODEQL_VERSION_NEW_TRACING } from "./codeql";
 import * as configUtils from "./config-utils";
-import { Feature, FeatureEnablement, Features } from "./feature-flags";
+import { Feature, Features } from "./feature-flags";
 import {
   initCodeQL,
   initConfig,
@@ -118,13 +118,14 @@ async function sendInitStatusReport(
     workflow_languages: workflowLanguages || "",
   };
 
-  let initToolsDownloadFields: InitToolsDownloadFields = {};
+  const initToolsDownloadFields: InitToolsDownloadFields = {};
 
-  if (toolsSource === ToolsSource.Download) {
-    initToolsDownloadFields = {
-      tools_download_duration_ms: toolsDownloadDurationMs,
-      tools_feature_flags_valid: toolsFeatureFlagsValid,
-    };
+  if (toolsDownloadDurationMs !== undefined) {
+    initToolsDownloadFields.tools_download_duration_ms =
+      toolsDownloadDurationMs;
+  }
+  if (toolsFeatureFlagsValid !== undefined) {
+    initToolsDownloadFields.tools_feature_flags_valid = toolsFeatureFlagsValid;
   }
 
   if (config !== undefined) {
@@ -253,7 +254,7 @@ async function run() {
       getOptionalInput("registries"),
       getOptionalInput("config-file"),
       getOptionalInput("db-location"),
-      await getTrapCachingEnabled(features),
+      getTrapCachingEnabled(),
       // Debug mode is enabled if:
       // - The `init` Action is passed `debug: true`.
       // - Actions step debugging is enabled (e.g. by [enabling debug logging for a rerun](https://docs.github.com/en/actions/managing-workflow-runs/re-running-workflows-and-jobs#re-running-all-the-jobs-in-a-workflow),
@@ -385,9 +386,7 @@ async function run() {
   );
 }
 
-async function getTrapCachingEnabled(
-  featureEnablement: FeatureEnablement
-): Promise<boolean> {
+function getTrapCachingEnabled(): boolean {
   // If the workflow specified something always respect that
   const trapCaching = getOptionalInput("trap-caching");
   if (trapCaching !== undefined) return trapCaching === "true";
@@ -395,8 +394,8 @@ async function getTrapCachingEnabled(
   // On self-hosted runners which may have slow network access, disable TRAP caching by default
   if (!isHostedRunner()) return false;
 
-  // On hosted runners, respect the feature flag
-  return await featureEnablement.getValue(Feature.TrapCachingEnabled);
+  // On hosted runners, enable TRAP caching by default
+  return true;
 }
 
 async function runWrapper() {

src/upload-lib.ts

@@ -463,7 +463,9 @@ export async function waitForProcessing(
         util.assertNever(status);
       }
 
-      await util.delay(STATUS_CHECK_FREQUENCY_MILLISECONDS);
+      await util.delay(STATUS_CHECK_FREQUENCY_MILLISECONDS, {
+        allowProcessExit: false,
+      });
     }
   } finally {
     logger.endGroup();

src/util.ts

@@ -548,16 +548,38 @@ export async function bundleDb(
   return databaseBundlePath;
 }
 
-export async function delay(milliseconds: number) {
-  // Immediately `unref` the timer such that it only prevents the process from exiting if the
-  // surrounding promise is being awaited.
-  return new Promise((resolve) => setTimeout(resolve, milliseconds).unref());
+/**
+ * @param milliseconds time to delay
+ * @param opts options
+ * @param opts.allowProcessExit if true, the timer will not prevent the process from exiting
+ */
+export async function delay(
+  milliseconds: number,
+  { allowProcessExit }: { allowProcessExit: boolean }
+) {
+  return new Promise((resolve) => {
+    const timer = setTimeout(resolve, milliseconds);
+    if (allowProcessExit) {
+      // Immediately `unref` the timer such that it only prevents the process from exiting if the
+      // surrounding promise is being awaited.
+      timer.unref();
+    }
+  });
 }
 
 export function isGoodVersion(versionSpec: string) {
   return !BROKEN_VERSIONS.includes(versionSpec);
 }
 
+/**
+ * Checks whether the CodeQL CLI supports the `--expect-discarded-cache` command-line flag.
+ */
+export async function supportExpectDiscardedCache(
+  codeQL: CodeQL
+): Promise<boolean> {
+  return codeQlVersionAbove(codeQL, "2.12.1");
+}
+
 export const ML_POWERED_JS_QUERIES_PACK_NAME =
   "codeql/javascript-experimental-atm-queries";
 
@@ -748,7 +770,7 @@ export async function withTimeout<T>(
     return result;
   };
   const timeoutTask = async () => {
-    await delay(timeoutMs);
+    await delay(timeoutMs, { allowProcessExit: true });
     if (!finished) {
       // Workaround: While the promise racing below will allow the main code
       // to continue, the process won't normally exit until the asynchronous
@@ -773,7 +795,7 @@ export async function checkForTimeout() {
     core.info(
       "A timeout occurred, force exiting the process after 30 seconds to prevent hanging."
     );
-    await delay(30_000);
+    await delay(30_000, { allowProcessExit: true });
     process.exit();
   }
 }