Moves calls to pack download to the init action

This ensures all steps to gather queries happens in the init action.
This is where checking out queries in other repos happens as well.

lib/analyze.js

@@ -170,15 +170,6 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
                 logger.info(analysisSummary);
             }
             else {
-                if (hasPackWithCustomQueries) {
-                    logger.info("Performing analysis with custom CodeQL Packs.");
-                    logger.startGroup(`Downloading custom packs for ${language}`);
-                    const results = await codeql.packDownload(packsWithVersion);
-                    logger.info(`Downloaded packs: ${results.packs
-                        .map((r) => `${r.name}@${r.version || "latest"}`)
-                        .join(", ")}`);
-                    logger.endGroup();
-                }
                 logger.startGroup(`Running queries for ${language}`);
                 const querySuitePaths = [];
                 if (queries["builtin"].length > 0) {

lib/config-utils.js

@@ -19,7 +19,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
     return result;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getConfig = exports.getPathToParsedConfigFile = exports.initConfig = exports.parsePacks = exports.validatePackSpecification = exports.prettyPrintPack = exports.parsePacksSpecification = exports.parsePacksFromConfig = exports.calculateAugmentation = exports.getDefaultConfig = exports.getUnknownLanguagesError = exports.getNoLanguagesError = exports.getConfigFileDirectoryGivenMessage = exports.getConfigFileFormatInvalidMessage = exports.getConfigFileRepoFormatInvalidMessage = exports.getConfigFileDoesNotExistErrorMessage = exports.getConfigFileOutsideWorkspaceErrorMessage = exports.getLocalPathDoesNotExist = exports.getLocalPathOutsideOfRepository = exports.getPacksStrInvalid = exports.getPacksInvalid = exports.getPacksInvalidSplit = exports.getPathsInvalid = exports.getPathsIgnoreInvalid = exports.getQueryUsesInvalid = exports.getQueriesMissingUses = exports.getQueriesInvalid = exports.getDisableDefaultQueriesInvalid = exports.getNameInvalid = exports.validateAndSanitisePath = exports.defaultAugmentationProperties = void 0;
+exports.downloadPacks = exports.getConfig = exports.getPathToParsedConfigFile = exports.initConfig = exports.parsePacks = exports.validatePackSpecification = exports.prettyPrintPack = exports.parsePacksSpecification = exports.parsePacksFromConfig = exports.calculateAugmentation = exports.getDefaultConfig = exports.getUnknownLanguagesError = exports.getNoLanguagesError = exports.getConfigFileDirectoryGivenMessage = exports.getConfigFileFormatInvalidMessage = exports.getConfigFileRepoFormatInvalidMessage = exports.getConfigFileDoesNotExistErrorMessage = exports.getConfigFileOutsideWorkspaceErrorMessage = exports.getLocalPathDoesNotExist = exports.getLocalPathOutsideOfRepository = exports.getPacksStrInvalid = exports.getPacksInvalid = exports.getPacksInvalidSplit = exports.getPathsInvalid = exports.getPathsIgnoreInvalid = exports.getQueryUsesInvalid = exports.getQueriesMissingUses = exports.getQueriesInvalid = exports.getDisableDefaultQueriesInvalid = exports.getNameInvalid = exports.validateAndSanitisePath = exports.defaultAugmentationProperties = void 0;
 const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
 // We need to import `performance` on Node 12
@@ -905,6 +905,11 @@ async function initConfig(languagesInput, queriesInput, packsInput, configFile,
                 "Please make sure that the default queries are enabled, or you are specifying queries to run.");
         }
     }
+    // if using the codescanning config in the CLI, pack downloads
+    // happen in the CLI, so no need to do them here.
+    if (!(await (0, util_1.useCodeScanningConfigInCli)(codeQL))) {
+        await downloadPacks(codeQL, config.languages, config.packs, logger);
+    }
     // Save the config so we can easily access it again in the future
     await saveConfig(config, logger);
     return config;
@@ -989,4 +994,27 @@ async function getConfig(tempDir, logger) {
     return JSON.parse(configString);
 }
 exports.getConfig = getConfig;
+async function downloadPacks(codeQL, languages, packs, logger) {
+    let packsDownloaded = 0;
+    logger.startGroup("Downloading packs");
+    for (const language of languages) {
+        const packsWithVersion = packs[language];
+        if (packsWithVersion === null || packsWithVersion === void 0 ? void 0 : packsWithVersion.length) {
+            logger.info(`Downloading custom packs for ${language}`);
+            const results = await codeQL.packDownload(packsWithVersion);
+            packsDownloaded += results.packs.length;
+            logger.info(`Downloaded packs: ${results.packs
+                .map((r) => `${r.name}@${r.version || "latest"}`)
+                .join(", ")}`);
+        }
+        if (packsDownloaded) {
+            logger.info(`Downloaded ${packsDownloaded} packs`);
+        }
+        else {
+            logger.info("No packs to download");
+        }
+        logger.endGroup();
+    }
+}
+exports.downloadPacks = downloadPacks;
 //# sourceMappingURL=config-utils.js.map

lib/config-utils.test.js

@@ -88,6 +88,9 @@ function mockListLanguages(languages) {
                     multipleDeclaredLanguages: {},
                 };
             },
+            async packDownload() {
+                return { packs: [] };
+            },
         });
         const config = await configUtils.initConfig(languages, undefined, undefined, undefined, undefined, false, false, "", "", { owner: "github", repo: "example " }, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), logger);
         t.deepEqual(config, await configUtils.getDefaultConfig(languages, undefined, undefined, undefined, false, false, "", "", { owner: "github", repo: "example " }, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), logger));
@@ -107,6 +110,9 @@ function mockListLanguages(languages) {
                     multipleDeclaredLanguages: {},
                 };
             },
+            async packDownload() {
+                return { packs: [] };
+            },
         });
         // Sanity check the saved config file does not already exist
         t.false(fs.existsSync(configUtils.getPathToParsedConfigFile(tmpDir)));
@@ -178,6 +184,9 @@ function mockListLanguages(languages) {
                     multipleDeclaredLanguages: {},
                 };
             },
+            async packDownload() {
+                return { packs: [] };
+            },
         });
         // Just create a generic config object with non-default values for all fields
         const inputFileContents = `
@@ -254,6 +263,9 @@ function mockListLanguages(languages) {
                     multipleDeclaredLanguages: {},
                 };
             },
+            async packDownload() {
+                return { packs: [] };
+            },
         });
         // The important point of this config is that it doesn't specify
         // the disable-default-queries field.
@@ -305,6 +317,9 @@ function queriesToResolvedQueryForm(queries) {
                 resolveQueriesArgs.push({ queries, extraSearchPath });
                 return queriesToResolvedQueryForm(queries);
             },
+            async packDownload() {
+                return { packs: [] };
+            },
         });
         const languages = "javascript";
         const config = await configUtils.initConfig(languages, undefined, undefined, configFilePath, undefined, false, false, "", "", { owner: "github", repo: "example " }, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), (0, logging_1.getRunnerLogger)(true));
@@ -338,6 +353,9 @@ function queriesToResolvedQueryForm(queries) {
                 resolveQueriesArgs.push({ queries, extraSearchPath });
                 return queriesToResolvedQueryForm(queries);
             },
+            async packDownload() {
+                return { packs: [] };
+            },
         });
         const languages = "javascript";
         const config = await configUtils.initConfig(languages, testQueries, undefined, configFilePath, undefined, false, false, "", "", { owner: "github", repo: "example " }, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), (0, logging_1.getRunnerLogger)(true));
@@ -370,6 +388,9 @@ function queriesToResolvedQueryForm(queries) {
                 resolveQueriesArgs.push({ queries, extraSearchPath });
                 return queriesToResolvedQueryForm(queries);
             },
+            async packDownload() {
+                return { packs: [] };
+            },
         });
         const languages = "javascript";
         const config = await configUtils.initConfig(languages, testQueries, undefined, configFilePath, undefined, false, false, "", "", { owner: "github", repo: "example " }, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), (0, logging_1.getRunnerLogger)(true));
@@ -396,6 +417,9 @@ function queriesToResolvedQueryForm(queries) {
                 resolveQueriesArgs.push({ queries, extraSearchPath });
                 return queriesToResolvedQueryForm(queries);
             },
+            async packDownload() {
+                return { packs: [] };
+            },
         });
         const languages = "javascript";
         const config = await configUtils.initConfig(languages, testQueries, undefined, undefined, undefined, false, false, "", "", { owner: "github", repo: "example " }, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), (0, logging_1.getRunnerLogger)(true));
@@ -435,6 +459,9 @@ function queriesToResolvedQueryForm(queries) {
                 resolveQueriesArgs.push({ queries, extraSearchPath });
                 return queriesToResolvedQueryForm(queries);
             },
+            async packDownload() {
+                return { packs: [] };
+            },
         });
         const languages = "javascript";
         const config = await configUtils.initConfig(languages, testQueries, undefined, configFilePath, undefined, false, false, "", "", { owner: "github", repo: "example " }, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), (0, logging_1.getRunnerLogger)(true));
@@ -474,6 +501,9 @@ function queriesToResolvedQueryForm(queries) {
                     multipleDeclaredLanguages: {},
                 };
             },
+            async packDownload() {
+                return { packs: [] };
+            },
         });
         try {
             await configUtils.initConfig(languages, queries, undefined, undefined, undefined, false, false, "", "", { owner: "github", repo: "example " }, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), (0, logging_1.getRunnerLogger)(true));
@@ -498,6 +528,9 @@ function queriesToResolvedQueryForm(queries) {
                     multipleDeclaredLanguages: {},
                 };
             },
+            async packDownload() {
+                return { packs: [] };
+            },
         });
         const inputFileContents = `
       name: my config
@@ -560,6 +593,9 @@ function queriesToResolvedQueryForm(queries) {
             async resolveLanguages() {
                 return {};
             },
+            async packDownload() {
+                return { packs: [] };
+            },
         });
         try {
             await configUtils.initConfig(undefined, undefined, undefined, undefined, undefined, false, false, "", "", { owner: "github", repo: "example " }, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), (0, logging_1.getRunnerLogger)(true));
@@ -592,6 +628,9 @@ function queriesToResolvedQueryForm(queries) {
                     multipleDeclaredLanguages: {},
                 };
             },
+            async packDownload() {
+                return { packs: [] };
+            },
         });
         const inputFileContents = `
       name: my config
@@ -620,6 +659,9 @@ function queriesToResolvedQueryForm(queries) {
                     multipleDeclaredLanguages: {},
                 };
             },
+            async packDownload() {
+                return { packs: [] };
+            },
         });
         const inputFileContents = `
       name: my config
@@ -673,6 +715,9 @@ function doInvalidInputTest(testName, inputFileContents, expectedErrorMessageGen
                         multipleDeclaredLanguages: {},
                     };
                 },
+                async packDownload() {
+                    return { packs: [] };
+                },
             });
             const languages = "javascript";
             const configFile = "input";
@@ -936,6 +981,9 @@ const mlPoweredQueriesMacro = ava_1.default.macro({
                         multipleDeclaredLanguages: {},
                     };
                 },
+                async packDownload() {
+                    return { packs: [] };
+                },
             });
             const { packs } = await configUtils.initConfig("javascript", queriesInput, packsInput, undefined, undefined, false, false, "", "", { owner: "github", repo: "example " }, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)(isMlPoweredQueriesFlagEnabled
                 ? [feature_flags_1.FeatureFlag.MlPoweredQueriesEnabled]
@@ -1039,4 +1087,24 @@ const calculateAugmentationErrorMacro = ava_1.default.macro({
 (0, ava_1.default)(calculateAugmentationErrorMacro, "Packs input with multiple languages", "   +  a/b, c/d ", undefined, [languages_1.Language.javascript, languages_1.Language.java], /Cannot specify a 'packs' input in a multi-language analysis/);
 (0, ava_1.default)(calculateAugmentationErrorMacro, "Packs input with no languages", "   +  a/b, c/d ", undefined, [], /No languages specified/);
 (0, ava_1.default)(calculateAugmentationErrorMacro, "Invalid packs", " a-pack-without-a-scope ", undefined, [languages_1.Language.javascript], /"a-pack-without-a-scope" is not a valid pack/);
+(0, ava_1.default)("downloadPacks", async (t) => {
+    const packDownloadStub = sinon.stub();
+    packDownloadStub.callsFake((packs) => ({
+        packs,
+    }));
+    const codeQL = (0, codeql_1.setCodeQL)({
+        packDownload: packDownloadStub,
+    });
+    const logger = (0, logging_1.getRunnerLogger)(true);
+    // packs are supplied for go, java, and python
+    // analyzed languages are java, javascript, and python
+    await configUtils.downloadPacks(codeQL, [languages_1.Language.javascript, languages_1.Language.java, languages_1.Language.python], {
+        java: ["a", "b"],
+        go: ["c", "d"],
+        python: ["e", "f"],
+    }, logger);
+    t.deepEqual(packDownloadStub.callCount, 2);
+    t.deepEqual(packDownloadStub.firstCall.args, [["a", "b"]]);
+    t.deepEqual(packDownloadStub.secondCall.args, [["e", "f"]]);
+});
 //# sourceMappingURL=config-utils.test.js.map

src/analyze.ts

@@ -280,21 +280,6 @@ export async function runQueries(
         logger.endGroup();
         logger.info(analysisSummary);
       } else {
-        if (hasPackWithCustomQueries) {
-          logger.info("Performing analysis with custom CodeQL Packs.");
-          logger.startGroup(`Downloading custom packs for ${language}`);
-
-          const results = await codeql.packDownload(packsWithVersion);
-
-          logger.info(
-            `Downloaded packs: ${results.packs
-              .map((r) => `${r.name}@${r.version || "latest"}`)
-              .join(", ")}`
-          );
-
-          logger.endGroup();
-        }
-
         logger.startGroup(`Running queries for ${language}`);
         const querySuitePaths: string[] = [];
         if (queries["builtin"].length > 0) {

src/config-utils.test.ts

@@ -6,7 +6,7 @@ import test, { ExecutionContext } from "ava";
 import * as sinon from "sinon";
 
 import * as api from "./api-client";
-import { getCachedCodeQL, setCodeQL } from "./codeql";
+import { getCachedCodeQL, PackDownloadOutput, setCodeQL } from "./codeql";
 import * as configUtils from "./config-utils";
 import { createFeatureFlags, FeatureFlag } from "./feature-flags";
 import { Language } from "./languages";
@@ -78,6 +78,9 @@ test("load empty config", async (t) => {
           multipleDeclaredLanguages: {},
         };
       },
+      async packDownload(): Promise<PackDownloadOutput> {
+        return { packs: [] };
+      },
     });
 
     const config = await configUtils.initConfig(
@@ -139,6 +142,9 @@ test("loading config saves config", async (t) => {
           multipleDeclaredLanguages: {},
         };
       },
+      async packDownload(): Promise<PackDownloadOutput> {
+        return { packs: [] };
+      },
     });
 
     // Sanity check the saved config file does not already exist
@@ -311,6 +317,9 @@ test("load non-empty input", async (t) => {
           multipleDeclaredLanguages: {},
         };
       },
+      async packDownload(): Promise<PackDownloadOutput> {
+        return { packs: [] };
+      },
     });
 
     // Just create a generic config object with non-default values for all fields
@@ -419,6 +428,9 @@ test("Default queries are used", async (t) => {
           multipleDeclaredLanguages: {},
         };
       },
+      async packDownload(): Promise<PackDownloadOutput> {
+        return { packs: [] };
+      },
     });
 
     // The important point of this config is that it doesn't specify
@@ -504,6 +516,9 @@ test("Queries can be specified in config file", async (t) => {
         resolveQueriesArgs.push({ queries, extraSearchPath });
         return queriesToResolvedQueryForm(queries);
       },
+      async packDownload(): Promise<PackDownloadOutput> {
+        return { packs: [] };
+      },
     });
 
     const languages = "javascript";
@@ -578,6 +593,9 @@ test("Queries from config file can be overridden in workflow file", async (t) =>
         resolveQueriesArgs.push({ queries, extraSearchPath });
         return queriesToResolvedQueryForm(queries);
       },
+      async packDownload(): Promise<PackDownloadOutput> {
+        return { packs: [] };
+      },
     });
 
     const languages = "javascript";
@@ -650,6 +668,9 @@ test("Queries in workflow file can be used in tandem with the 'disable default q
         resolveQueriesArgs.push({ queries, extraSearchPath });
         return queriesToResolvedQueryForm(queries);
       },
+      async packDownload(): Promise<PackDownloadOutput> {
+        return { packs: [] };
+      },
     });
 
     const languages = "javascript";
@@ -713,6 +734,9 @@ test("Multiple queries can be specified in workflow file, no config file require
         resolveQueriesArgs.push({ queries, extraSearchPath });
         return queriesToResolvedQueryForm(queries);
       },
+      async packDownload(): Promise<PackDownloadOutput> {
+        return { packs: [] };
+      },
     });
 
     const languages = "javascript";
@@ -797,6 +821,9 @@ test("Queries in workflow file can be added to the set of queries without overri
         resolveQueriesArgs.push({ queries, extraSearchPath });
         return queriesToResolvedQueryForm(queries);
       },
+      async packDownload(): Promise<PackDownloadOutput> {
+        return { packs: [] };
+      },
     });
 
     const languages = "javascript";
@@ -876,6 +903,9 @@ test("Invalid queries in workflow file handled correctly", async (t) => {
           multipleDeclaredLanguages: {},
         };
       },
+      async packDownload(): Promise<PackDownloadOutput> {
+        return { packs: [] };
+      },
     });
 
     try {
@@ -922,6 +952,9 @@ test("API client used when reading remote config", async (t) => {
           multipleDeclaredLanguages: {},
         };
       },
+      async packDownload(): Promise<PackDownloadOutput> {
+        return { packs: [] };
+      },
     });
 
     const inputFileContents = `
@@ -1051,6 +1084,9 @@ test("No detected languages", async (t) => {
       async resolveLanguages() {
         return {};
       },
+      async packDownload(): Promise<PackDownloadOutput> {
+        return { packs: [] };
+      },
     });
 
     try {
@@ -1124,6 +1160,9 @@ test("Config specifies packages", async (t) => {
           multipleDeclaredLanguages: {},
         };
       },
+      async packDownload(): Promise<PackDownloadOutput> {
+        return { packs: [] };
+      },
     });
 
     const inputFileContents = `
@@ -1175,6 +1214,9 @@ test("Config specifies packages for multiple languages", async (t) => {
           multipleDeclaredLanguages: {},
         };
       },
+      async packDownload(): Promise<PackDownloadOutput> {
+        return { packs: [] };
+      },
     });
 
     const inputFileContents = `
@@ -1255,6 +1297,9 @@ function doInvalidInputTest(
             multipleDeclaredLanguages: {},
           };
         },
+        async packDownload(): Promise<PackDownloadOutput> {
+          return { packs: [] };
+        },
       });
 
       const languages = "javascript";
@@ -1845,6 +1890,9 @@ const mlPoweredQueriesMacro = test.macro({
             multipleDeclaredLanguages: {},
           };
         },
+        async packDownload(): Promise<PackDownloadOutput> {
+          return { packs: [] };
+        },
       });
 
       const { packs } = await configUtils.initConfig(
@@ -2159,3 +2207,31 @@ test(
   [Language.javascript],
   /"a-pack-without-a-scope" is not a valid pack/
 );
+
+test("downloadPacks", async (t) => {
+  const packDownloadStub = sinon.stub();
+  packDownloadStub.callsFake((packs) => ({
+    packs,
+  }));
+  const codeQL = setCodeQL({
+    packDownload: packDownloadStub,
+  });
+  const logger = getRunnerLogger(true);
+
+  // packs are supplied for go, java, and python
+  // analyzed languages are java, javascript, and python
+  await configUtils.downloadPacks(
+    codeQL,
+    [Language.javascript, Language.java, Language.python],
+    {
+      java: ["a", "b"],
+      go: ["c", "d"],
+      python: ["e", "f"],
+    },
+    logger
+  );
+
+  t.deepEqual(packDownloadStub.callCount, 2);
+  t.deepEqual(packDownloadStub.firstCall.args, [["a", "b"]]);
+  t.deepEqual(packDownloadStub.secondCall.args, [["e", "f"]]);
+});

src/config-utils.ts

@@ -24,6 +24,7 @@ import {
   getMlPoweredJsQueriesPack,
   GitHubVersion,
   ML_POWERED_JS_QUERIES_PACK_NAME,
+  useCodeScanningConfigInCli,
 } from "./util";
 
 // Property names from the user-supplied config file.
@@ -953,6 +954,7 @@ async function addQueriesAndPacksFromWorkflow(
     );
     injectedMlQueries = injectedMlQueries || didInject;
   }
+
   return injectedMlQueries;
 }
 
@@ -1680,6 +1682,12 @@ export async function initConfig(
     }
   }
 
+  // if using the codescanning config in the CLI, pack downloads
+  // happen in the CLI, so no need to do them here.
+  if (!(await useCodeScanningConfigInCli(codeQL))) {
+    await downloadPacks(codeQL, config.languages, config.packs, logger);
+  }
+
   // Save the config so we can easily access it again in the future
   await saveConfig(config, logger);
   return config;
@@ -1781,3 +1789,33 @@ export async function getConfig(
   logger.debug(configString);
   return JSON.parse(configString);
 }
+
+export async function downloadPacks(
+  codeQL: CodeQL,
+  languages: Language[],
+  packs: Packs,
+  logger: Logger
+) {
+  let packsDownloaded = 0;
+  logger.startGroup("Downloading packs");
+  for (const language of languages) {
+    const packsWithVersion = packs[language];
+    if (packsWithVersion?.length) {
+      logger.info(`Downloading custom packs for ${language}`);
+      const results = await codeQL.packDownload(packsWithVersion);
+      packsDownloaded += results.packs.length;
+      logger.info(
+        `Downloaded packs: ${results.packs
+          .map((r) => `${r.name}@${r.version || "latest"}`)
+          .join(", ")}`
+      );
+    }
+
+    if (packsDownloaded) {
+      logger.info(`Downloaded ${packsDownloaded} packs`);
+    } else {
+      logger.info("No packs to download");
+    }
+    logger.endGroup();
+  }
+}