Add new packs input to init action

This input allows users to specify which packs to run. It works in unison with the packs block of the config file and it is similar to how `queries` works. They both use `+` in the same way. Note that the `#TODO` in the pr check is still around, but the CLI is available. I will remove the TODO in the next commit.
2026-01-06 14:40:10 +08:00 · 2021-06-23 15:41:52 -07:00
parent 7729b51956
commit 6e577cfca3
18 changed files with 535 additions and 59 deletions
--- a/src/config-utils.test.ts
+++ b/src/config-utils.test.ts
@@ -84,6 +84,7 @@ test("load empty config", async (t) => {
      undefined,
      undefined,
      undefined,
+      undefined,
      { owner: "github", repo: "example " },
      tmpDir,
      tmpDir,
@@ -100,6 +101,7 @@ test("load empty config", async (t) => {
        languages,
        undefined,
        undefined,
+        undefined,
        { owner: "github", repo: "example " },
        tmpDir,
        tmpDir,
@@ -141,6 +143,7 @@ test("loading config saves config", async (t) => {
      undefined,
      undefined,
      undefined,
+      undefined,
      { owner: "github", repo: "example " },
      tmpDir,
      tmpDir,
@@ -164,6 +167,7 @@ test("load input outside of workspace", async (t) => {
  return await util.withTmpDir(async (tmpDir) => {
    try {
      await configUtils.initConfig(
+        undefined,
        undefined,
        undefined,
        "../input",
@@ -198,6 +202,7 @@ test("load non-local input with invalid repo syntax", async (t) => {

    try {
      await configUtils.initConfig(
+        undefined,
        undefined,
        undefined,
        configFile,
@@ -235,6 +240,7 @@ test("load non-existent input", async (t) => {
      await configUtils.initConfig(
        languages,
        undefined,
+        undefined,
        configFile,
        undefined,
        { owner: "github", repo: "example " },
@@ -328,6 +334,7 @@ test("load non-empty input", async (t) => {
    const actualConfig = await configUtils.initConfig(
      languages,
      undefined,
+      undefined,
      configFilePath,
      undefined,
      { owner: "github", repo: "example " },
@@ -390,6 +397,7 @@ test("Default queries are used", async (t) => {
    await configUtils.initConfig(
      languages,
      undefined,
+      undefined,
      configFilePath,
      undefined,
      { owner: "github", repo: "example " },
@@ -460,6 +468,7 @@ test("Queries can be specified in config file", async (t) => {
    const config = await configUtils.initConfig(
      languages,
      undefined,
+      undefined,
      configFilePath,
      undefined,
      { owner: "github", repo: "example " },
@@ -524,6 +533,7 @@ test("Queries from config file can be overridden in workflow file", async (t) =>
    const config = await configUtils.initConfig(
      languages,
      testQueries,
+      undefined,
      configFilePath,
      undefined,
      { owner: "github", repo: "example " },
@@ -586,6 +596,7 @@ test("Queries in workflow file can be used in tandem with the 'disable default q
    const config = await configUtils.initConfig(
      languages,
      testQueries,
+      undefined,
      configFilePath,
      undefined,
      { owner: "github", repo: "example " },
@@ -643,6 +654,7 @@ test("Multiple queries can be specified in workflow file, no config file require
      testQueries,
      undefined,
      undefined,
+      undefined,
      { owner: "github", repo: "example " },
      tmpDir,
      tmpDir,
@@ -717,6 +729,7 @@ test("Queries in workflow file can be added to the set of queries without overri
    const config = await configUtils.initConfig(
      languages,
      testQueries,
+      undefined,
      configFilePath,
      undefined,
      { owner: "github", repo: "example " },
@@ -785,6 +798,7 @@ test("Invalid queries in workflow file handled correctly", async (t) => {
        queries,
        undefined,
        undefined,
+        undefined,
        { owner: "github", repo: "example " },
        tmpDir,
        tmpDir,
@@ -846,6 +860,7 @@ test("API client used when reading remote config", async (t) => {
    await configUtils.initConfig(
      languages,
      undefined,
+      undefined,
      configFile,
      undefined,
      { owner: "github", repo: "example " },
@@ -869,6 +884,7 @@ test("Remote config handles the case where a directory is provided", async (t) =
    const repoReference = "octo-org/codeql-config/config.yaml@main";
    try {
      await configUtils.initConfig(
+        undefined,
        undefined,
        undefined,
        repoReference,
@@ -902,6 +918,7 @@ test("Invalid format of remote config handled correctly", async (t) => {
    const repoReference = "octo-org/codeql-config/config.yaml@main";
    try {
      await configUtils.initConfig(
+        undefined,
        undefined,
        undefined,
        repoReference,
@@ -940,6 +957,7 @@ test("No detected languages", async (t) => {
        undefined,
        undefined,
        undefined,
+        undefined,
        { owner: "github", repo: "example " },
        tmpDir,
        tmpDir,
@@ -966,6 +984,7 @@ test("Unknown languages", async (t) => {
        undefined,
        undefined,
        undefined,
+        undefined,
        { owner: "github", repo: "example " },
        tmpDir,
        tmpDir,
@@ -1012,6 +1031,7 @@ test("Config specifies packages", async (t) => {
    const { packs } = await configUtils.initConfig(
      languages,
      undefined,
+      undefined,
      configFile,
      undefined,
      { owner: "github", repo: "example " },
@@ -1069,6 +1089,7 @@ test("Config specifies packages for multiple languages", async (t) => {
    const { packs, queries } = await configUtils.initConfig(
      languages,
      undefined,
+      undefined,
      configFile,
      undefined,
      { owner: "github", repo: "example" },
@@ -1142,6 +1163,7 @@ function doInvalidInputTest(
        await configUtils.initConfig(
          languages,
          undefined,
+          undefined,
          configFile,
          undefined,
          { owner: "github", repo: "example " },
@@ -1321,7 +1343,7 @@ function parsePacksMacro(
  expected
 ) {
  t.deepEqual(
-    configUtils.parsePacks(packsByLanguage, languages, "/a/b"),
+    configUtils.parsePacksFromConfig(packsByLanguage, languages, "/a/b"),
    expected
  );
 }
@@ -1339,7 +1361,7 @@ function parsePacksErrorMacro(
 ) {
  t.throws(
    () => {
-      configUtils.parsePacks(packsByLanguage, languages, "/a/b");
+      configUtils.parsePacksFromConfig(packsByLanguage, languages, "/a/b");
    },
    {
      message: expected,
@@ -1349,6 +1371,9 @@ function parsePacksErrorMacro(
 parsePacksErrorMacro.title = (providedTitle: string) =>
  `Parse Packs Error: ${providedTitle}`;

+/**
+ * Test macro for testing when the packs block is invalid
+ */
 function invalidPackNameMacro(t: ExecutionContext<unknown>, name: string) {
  parsePacksErrorMacro(
    t,
@@ -1369,6 +1394,18 @@ test("two packs", parsePacksMacro, ["a/b", "c/d@1.2.3"], [Language.cpp], {
    { packName: "c/d", version: clean("1.2.3") },
  ],
 });
+test(
+  "two packs with spaces",
+  parsePacksMacro,
+  [" a/b ", " c/d@1.2.3 "],
+  [Language.cpp],
+  {
+    [Language.cpp]: [
+      { packName: "a/b", version: undefined },
+      { packName: "c/d", version: clean("1.2.3") },
+    ],
+  }
+);
 test(
  "two packs with language",
  parsePacksMacro,
@@ -1416,3 +1453,160 @@ test(invalidPackNameMacro, "c-/d");
 test(invalidPackNameMacro, "-c/d");
 test(invalidPackNameMacro, "c/d_d");
 test(invalidPackNameMacro, "c/d@x");
+
+/**
+ * Test macro for testing the packs block and the packs input
+ */
+function parseInputAndConfigMacro(
+  t: ExecutionContext<unknown>,
+  packsFromConfig: string[] | Record<string, string[]>,
+  packsFromInput: string | undefined,
+  languages: Language[],
+  expected
+) {
+  t.deepEqual(
+    configUtils.parsePacks(packsFromConfig, packsFromInput, languages, "/a/b"),
+    expected
+  );
+}
+parseInputAndConfigMacro.title = (providedTitle: string) =>
+  `Parse Packs input and config: ${providedTitle}`;
+
+function parseInputAndConfigErrorMacro(
+  t: ExecutionContext<unknown>,
+  packsFromConfig: string[] | Record<string, string[]>,
+  packsFromInput: string | undefined,
+  languages: Language[],
+  expected: RegExp
+) {
+  t.throws(
+    () => {
+      configUtils.parsePacks(
+        packsFromConfig,
+        packsFromInput,
+        languages,
+        "/a/b"
+      );
+    },
+    {
+      message: expected,
+    }
+  );
+}
+parseInputAndConfigErrorMacro.title = (providedTitle: string) =>
+  `Parse Packs input and config Error: ${providedTitle}`;
+
+test("input only", parseInputAndConfigMacro, {}, " c/d ", [Language.cpp], {
+  [Language.cpp]: [{ packName: "c/d", version: undefined }],
+});
+
+test(
+  "input only with multiple",
+  parseInputAndConfigMacro,
+  {},
+  "a/b , c/d@1.2.3",
+  [Language.cpp],
+  {
+    [Language.cpp]: [
+      { packName: "a/b", version: undefined },
+      { packName: "c/d", version: "1.2.3" },
+    ],
+  }
+);
+
+test(
+  "input only with +",
+  parseInputAndConfigMacro,
+  {},
+  "  +  a/b , c/d@1.2.3 ",
+  [Language.cpp],
+  {
+    [Language.cpp]: [
+      { packName: "a/b", version: undefined },
+      { packName: "c/d", version: "1.2.3" },
+    ],
+  }
+);
+
+test(
+  "config only",
+  parseInputAndConfigMacro,
+  ["a/b", "c/d"],
+  "  ",
+  [Language.cpp],
+  {
+    [Language.cpp]: [
+      { packName: "a/b", version: undefined },
+      { packName: "c/d", version: undefined },
+    ],
+  }
+);
+
+test(
+  "input overrides",
+  parseInputAndConfigMacro,
+  ["a/b", "c/d"],
+  " e/f, g/h@1.2.3 ",
+  [Language.cpp],
+  {
+    [Language.cpp]: [
+      { packName: "e/f", version: undefined },
+      { packName: "g/h", version: "1.2.3" },
+    ],
+  }
+);
+
+test(
+  "input and config",
+  parseInputAndConfigMacro,
+  ["a/b", "c/d"],
+  " +e/f, g/h@1.2.3 ",
+  [Language.cpp],
+  {
+    [Language.cpp]: [
+      { packName: "e/f", version: undefined },
+      { packName: "g/h", version: "1.2.3" },
+      { packName: "a/b", version: undefined },
+      { packName: "c/d", version: undefined },
+    ],
+  }
+);
+
+test(
+  "input with no language",
+  parseInputAndConfigErrorMacro,
+  {},
+  "c/d",
+  [],
+  /No languages specified/
+);
+
+test(
+  "input with two languages",
+  parseInputAndConfigErrorMacro,
+  {},
+  "c/d",
+  [Language.cpp, Language.csharp],
+  /multi-language analysis/
+);
+
+test(
+  "input with + only",
+  parseInputAndConfigErrorMacro,
+  {},
+  " + ",
+  [Language.cpp],
+  /Remove the '\+'/
+);
+
+test(
+  "input with invalid pack name",
+  parseInputAndConfigErrorMacro,
+  {},
+  " xxx",
+  [Language.cpp],
+  /"xxx" is not a valid pack/
+);
+
+// errors
+// input w invalid pack name
--- a/src/config-utils.ts
+++ b/src/config-utils.ts
@@ -585,13 +585,15 @@ export function getPacksInvalid(configFile: string): string {

 export function getPacksStrInvalid(
  packStr: string,
-  configFile: string
+  configFile?: string
 ): string {
-  return getConfigFilePropertyError(
-    configFile,
-    PACKS_PROPERTY,
-    `"${packStr}" is not a valid pack`
-  );
+  return configFile
+    ? getConfigFilePropertyError(
+        configFile,
+        PACKS_PROPERTY,
+        `"${packStr}" is not a valid pack`
+      )
+    : `"${packStr}" is not a valid pack`;
 }

 export function getLocalPathOutsideOfRepository(
@@ -802,6 +804,7 @@ function shouldAddConfigFileQueries(queriesInput: string | undefined): boolean {
 export async function getDefaultConfig(
  languagesInput: string | undefined,
  queriesInput: string | undefined,
+  packsInput: string | undefined,
  dbLocation: string | undefined,
  repository: RepositoryNwo,
  tempDir: string,
@@ -840,12 +843,14 @@ export async function getDefaultConfig(
    );
  }

+  const packs = parsePacksInput(packsInput, languages) ?? {};
+
  return {
    languages,
    queries,
    pathsIgnore: [],
    paths: [],
-    packs: {},
+    packs,
    originalUserInput: {},
    tempDir,
    toolCacheDir,
@@ -861,6 +866,7 @@ export async function getDefaultConfig(
 async function loadConfig(
  languagesInput: string | undefined,
  queriesInput: string | undefined,
+  packsInput: string | undefined,
  configFile: string,
  dbLocation: string | undefined,
  repository: RepositoryNwo,
@@ -1002,6 +1008,7 @@ async function loadConfig(

  const packs = parsePacks(
    parsedYAML[PACKS_PROPERTY] ?? {},
+    packsInput,
    languages,
    configFile
  );
@@ -1033,7 +1040,7 @@ const PACK_IDENTIFIER_PATTERN = (function () {
 })();

 // Exported for testing
-export function parsePacks(
+export function parsePacksFromConfig(
  packsByLanguage: string[] | Record<string, string[]>,
  languages: Language[],
  configFile: string
@@ -1068,12 +1075,44 @@ export function parsePacks(
  return packs;
 }

-function toPackWithVersion(packStr, configFile: string): PackWithVersion {
+function parsePacksInput(
+  packsInput: string | undefined,
+  languages: Language[]
+): Packs | undefined {
+  if (!packsInput?.trim()) {
+    return undefined;
+  }
+
+  if (languages.length > 1) {
+    throw new Error(
+      "Cannot specify a 'packs' input in a multi-language analysis. Use a codeql-config.yml file instead and specify packs by library."
+    );
+  } else if (languages.length === 0) {
+    throw new Error("No languages specified. Cannot process the packs input.");
+  }
+
+  packsInput = packsInput.trim();
+  if (packsInput.startsWith("+")) {
+    packsInput = packsInput.substring(1).trim();
+    if (!packsInput) {
+      throw new Error("Remove the '+' from the packs input.");
+    }
+  }
+
+  return {
+    [languages[0]]: packsInput.split(",").reduce((packs, pack) => {
+      packs.push(toPackWithVersion(pack, ""));
+      return packs;
+    }, [] as PackWithVersion[]),
+  };
+}
+
+function toPackWithVersion(packStr, configFile?: string): PackWithVersion {
  if (typeof packStr !== "string") {
    throw new Error(getPacksStrInvalid(packStr, configFile));
  }

-  const nameWithVersion = packStr.split("@");
+  const nameWithVersion = packStr.trim().split("@");
  let version: string | undefined;
  if (
    nameWithVersion.length > 2 ||
@@ -1088,11 +1127,52 @@ function toPackWithVersion(packStr, configFile: string): PackWithVersion {
  }

  return {
-    packName: nameWithVersion[0],
+    packName: nameWithVersion[0].trim(),
    version,
  };
 }

+// exported for testing
+export function parsePacks(
+  rawPacksFromConfig: string[] | Record<string, string[]>,
+  rawPacksInput: string | undefined,
+  languages: Language[],
+  configFile: string
+) {
+  const packsFromInput = parsePacksInput(rawPacksInput, languages);
+  const packsFomConfig = parsePacksFromConfig(
+    rawPacksFromConfig,
+    languages,
+    configFile
+  );
+
+  if (!packsFromInput) {
+    return packsFomConfig;
+  }
+  if (!shouldCombinePacks(rawPacksInput)) {
+    return packsFromInput;
+  }
+
+  return combinePacks(packsFromInput, packsFomConfig);
+}
+
+function shouldCombinePacks(packsInput?: string): boolean {
+  return !!packsInput?.trim().startsWith("+");
+}
+
+function combinePacks(packs1: Packs, packs2: Packs): Packs {
+  const packs = {};
+  for (const lang of Object.keys(packs1)) {
+    packs[lang] = packs1[lang].concat(packs2[lang] || []);
+  }
+  for (const lang of Object.keys(packs2)) {
+    if (!packs[lang]) {
+      packs[lang] = packs2[lang];
+    }
+  }
+  return packs;
+}
+
 function dbLocationOrDefault(
  dbLocation: string | undefined,
  tempDir: string
@@ -1109,6 +1189,7 @@ function dbLocationOrDefault(
 export async function initConfig(
  languagesInput: string | undefined,
  queriesInput: string | undefined,
+  packsInput: string | undefined,
  configFile: string | undefined,
  dbLocation: string | undefined,
  repository: RepositoryNwo,
@@ -1128,6 +1209,7 @@ export async function initConfig(
    config = await getDefaultConfig(
      languagesInput,
      queriesInput,
+      packsInput,
      dbLocation,
      repository,
      tempDir,
@@ -1142,6 +1224,7 @@ export async function initConfig(
    config = await loadConfig(
      languagesInput,
      queriesInput,
+      packsInput,
      configFile,
      dbLocation,
      repository,
--- a/src/init-action.ts
+++ b/src/init-action.ts
@@ -153,6 +153,7 @@ async function run() {
    config = await initConfig(
      getOptionalInput("languages"),
      getOptionalInput("queries"),
+      getOptionalInput("packs"),
      getOptionalInput("config-file"),
      getOptionalInput("db-location"),
      parseRepositoryNwo(getRequiredEnvParam("GITHUB_REPOSITORY")),
--- a/src/init.ts
+++ b/src/init.ts
@@ -38,6 +38,7 @@ export async function initCodeQL(
 export async function initConfig(
  languagesInput: string | undefined,
  queriesInput: string | undefined,
+  packsInput: string | undefined,
  configFile: string | undefined,
  dbLocation: string | undefined,
  repository: RepositoryNwo,
@@ -53,6 +54,7 @@ export async function initConfig(
  const config = await configUtils.initConfig(
    languagesInput,
    queriesInput,
+    packsInput,
    configFile,
    dbLocation,
    repository,
--- a/src/runner.ts
+++ b/src/runner.ts
@@ -96,6 +96,7 @@ function parseTraceProcessLevel(): number | undefined {
 interface InitArgs {
  languages: string | undefined;
  queries: string | undefined;
+  packs: string | undefined;
  configFile: string | undefined;
  codeqlPath: string | undefined;
  tempDir: string | undefined;
@@ -129,6 +130,14 @@ program
    "--queries <queries>",
    "Comma-separated list of additional queries to run. This overrides the same setting in a configuration file."
  )
+  .option(
+    "--packs <packs>",
+    `Comma-separated list of packs to run. Reference a pack in the format scope/name[@version]. If version is not
+    specified, then the latest version of the pack is used. By default, this overrides the same setting in a
+    configuration file; prefix with "+" to use both sets of packs.
+
+    This option is only available in single-language analyses.`
+  )
  .option("--config-file <file>", "Path to config file.")
  .option(
    "--codeql-path <path>",
@@ -201,6 +210,7 @@ program
      const config = await initConfig(
        cmd.languages,
        cmd.queries,
+        cmd.packs,
        cmd.configFile,
        undefined,
        parseRepositoryNwo(cmd.repository),