Add support for downloading packs from GHES

This change adds:

- new `registries` block allowed in code scanning config file
- new `registries-auth-tokens` input in init action
- Change the downloadPacks function so that it accepts new parameters:
    - registries block
    - api auth
- Generate a qlconfig.yml file with the registries block if one is
  supplied. Use this file when downloading packs.
- temporarily set the `GITHUB_TOKEN` and `CODEQL_REGISTRIES_AUTH` based
  on api auth

TODO:

1. integration test
2. handle pack downloads when the config is generated by the CLI

src/analyze-action.ts

@@ -143,6 +143,9 @@ async function run() {
       auth: actionsUtil.getRequiredInput("token"),
       url: util.getRequiredEnvParam("GITHUB_SERVER_URL"),
       apiURL: util.getRequiredEnvParam("GITHUB_API_URL"),
+
+      // not needed or available in the analize action
+      registriesAuthTokens: undefined,
     };
     const outputDir = actionsUtil.getRequiredInput("output");
     const threads = util.getThreadsFlag(

src/api-client.ts

@@ -4,7 +4,7 @@ import * as githubUtils from "@actions/github/lib/utils";
 import * as retry from "@octokit/plugin-retry";
 import consoleLogLevel from "console-log-level";
 
-import { getRequiredInput } from "./actions-util";
+import { getOptionalInput, getRequiredInput } from "./actions-util";
 import * as util from "./util";
 import { getMode, getRequiredEnvParam, GitHubVersion } from "./util";
 
@@ -23,6 +23,7 @@ export interface GitHubApiDetails {
   auth: string;
   url: string;
   apiURL: string | undefined;
+  registriesAuthTokens: string | undefined;
 }
 
 export interface GitHubApiExternalRepoDetails {
@@ -68,6 +69,9 @@ function getApiDetails() {
     auth: getRequiredInput("token"),
     url: getRequiredEnvParam("GITHUB_SERVER_URL"),
     apiURL: getRequiredEnvParam("GITHUB_API_URL"),
+
+    // only available in the init action
+    registriesAuthTokens: getOptionalInput("registries-auth-tokens"),
   };
 }
 

src/codeql.test.ts

@@ -26,12 +26,14 @@ const sampleApiDetails = {
   auth: "token",
   url: "https://github.com",
   apiURL: undefined,
+  registriesAuthTokens: undefined,
 };
 
 const sampleGHAEApiDetails = {
   auth: "token",
   url: "https://example.githubenterprise.com",
   apiURL: undefined,
+  registriesAuthTokens: undefined,
 };
 
 let stubConfig: Config;

src/codeql.ts

@@ -134,7 +134,10 @@ export interface CodeQL {
   /**
    * Run 'codeql pack download'.
    */
-  packDownload(packs: string[]): Promise<PackDownloadOutput>;
+  packDownload(
+    packs: string[],
+    qlconfigFile: string | undefined
+  ): Promise<PackDownloadOutput>;
 
   /**
    * Run 'codeql database cleanup'.
@@ -1086,11 +1089,22 @@ async function getCodeQLForCmd(
      * If no version is specified, then the latest version is
      * downloaded. The check to determine what the latest version is is done
      * each time this package is requested.
+     *
+     * Optionally, a `qlconfigFile` is included. If used, then this file
+     * is used to determine which registry each pack is downloaded from.
      */
-    async packDownload(packs: string[]): Promise<PackDownloadOutput> {
+    async packDownload(
+      packs: string[],
+      qlconfigFile: string | undefined
+    ): Promise<PackDownloadOutput> {
+      const qlconfigArg = qlconfigFile
+        ? [`--qlconfig-file=${qlconfigFile}`]
+        : ([] as string[]);
+
       const codeqlArgs = [
         "pack",
         "download",
+        ...qlconfigArg,
         "--format=json",
         "--resolve-query-specs",
         ...getExtraOptionsFromEnv(["pack", "download"]),

src/config-utils.test.ts

@@ -3,6 +3,7 @@ import * as path from "path";
 
 import * as github from "@actions/github";
 import test, { ExecutionContext } from "ava";
+import * as yaml from "js-yaml";
 import * as sinon from "sinon";
 
 import * as api from "./api-client";
@@ -21,6 +22,7 @@ const sampleApiDetails = {
   externalRepoAuth: "token",
   url: "https://github.example.com",
   apiURL: undefined,
+  registriesAuthTokens: undefined,
 };
 
 const gitHubVersion = { type: util.GitHubVariant.DOTCOM } as util.GitHubVersion;
@@ -2208,30 +2210,114 @@ test(
   /"a-pack-without-a-scope" is not a valid pack/
 );
 
-test("downloadPacks", async (t) => {
-  const packDownloadStub = sinon.stub();
-  packDownloadStub.callsFake((packs) => ({
-    packs,
-  }));
-  const codeQL = setCodeQL({
-    packDownload: packDownloadStub,
+test("downloadPacks-no-registries", async (t) => {
+  return await util.withTmpDir(async (tmpDir) => {
+    const packDownloadStub = sinon.stub();
+    packDownloadStub.callsFake((packs) => ({
+      packs,
+    }));
+    const codeQL = setCodeQL({
+      packDownload: packDownloadStub,
+    });
+    const logger = getRunnerLogger(true);
+
+    // packs are supplied for go, java, and python
+    // analyzed languages are java, javascript, and python
+    await configUtils.downloadPacks(
+      codeQL,
+      [Language.javascript, Language.java, Language.python],
+      {
+        java: ["a", "b"],
+        go: ["c", "d"],
+        python: ["e", "f"],
+      },
+      undefined,
+      sampleApiDetails,
+      tmpDir,
+      logger
+    );
+
+    t.deepEqual(packDownloadStub.callCount, 2);
+    // no config file was created, so pass `undefined` as the config file path
+    t.deepEqual(packDownloadStub.firstCall.args, [["a", "b"], undefined]);
+    t.deepEqual(packDownloadStub.secondCall.args, [["e", "f"], undefined]);
+  });
+});
+
+test("downloadPacks-with-registries", async (t) => {
+  // same thing, but this time include a registries block and
+  // associated env vars
+  return await util.withTmpDir(async (tmpDir) => {
+    process.env.GITHUB_TOKEN = "not-a-token";
+    process.env.CODEQL_REGISTRIES_AUTH = "not-a-registries-auth";
+    const logger = getRunnerLogger(true);
+
+    const apiDetails = {
+      ...sampleApiDetails,
+      registriesAuthTokens: "registries-auth",
+    };
+    const registries = [
+      {
+        url: "http://ghcr.io",
+        packages: ["codeql/*", "dsp-testing/*"],
+      },
+      {
+        url: "https://containers.GHEHOSTNAME1/v2/",
+        packages: "semmle/*",
+      },
+    ];
+
+    const expectedConfigFile = path.join(tmpDir, "qlconfig.yml");
+    const packDownloadStub = sinon.stub();
+    packDownloadStub.callsFake((packs, configFile) => {
+      t.deepEqual(configFile, expectedConfigFile);
+      // verify the env vars were set correctly
+      t.deepEqual(process.env.GITHUB_TOKEN, "token");
+      t.deepEqual(process.env.CODEQL_REGISTRIES_AUTH, "registries-auth");
+
+      // verify the config file contents were set correctly
+      const config = yaml.load(
+        fs.readFileSync(configFile, "utf8")
+      ) as configUtils.UserConfig;
+      t.deepEqual(config.registries, registries);
+      return {
+        packs,
+      };
+    });
+
+    const codeQL = setCodeQL({
+      packDownload: packDownloadStub,
+    });
+
+    // packs are supplied for go, java, and python
+    // analyzed languages are java, javascript, and python
+    await configUtils.downloadPacks(
+      codeQL,
+      [Language.javascript, Language.java, Language.python],
+      {
+        java: ["a", "b"],
+        go: ["c", "d"],
+        python: ["e", "f"],
+      },
+      registries,
+      apiDetails,
+      tmpDir,
+      logger
+    );
+
+    // Same packs are downloaded as in previous test
+    t.deepEqual(packDownloadStub.callCount, 2);
+    t.deepEqual(packDownloadStub.firstCall.args, [
+      ["a", "b"],
+      expectedConfigFile,
+    ]);
+    t.deepEqual(packDownloadStub.secondCall.args, [
+      ["e", "f"],
+      expectedConfigFile,
+    ]);
+
+    // Verify that the env vars were unset.
+    t.deepEqual(process.env.GITHUB_TOKEN, "not-a-token");
+    t.deepEqual(process.env.CODEQL_REGISTRIES_AUTH, "not-a-registries-auth");
   });
-  const logger = getRunnerLogger(true);
-
-  // packs are supplied for go, java, and python
-  // analyzed languages are java, javascript, and python
-  await configUtils.downloadPacks(
-    codeQL,
-    [Language.javascript, Language.java, Language.python],
-    {
-      java: ["a", "b"],
-      go: ["c", "d"],
-      python: ["e", "f"],
-    },
-    logger
-  );
-
-  t.deepEqual(packDownloadStub.callCount, 2);
-  t.deepEqual(packDownloadStub.firstCall.args, [["a", "b"]]);
-  t.deepEqual(packDownloadStub.secondCall.args, [["e", "f"]]);
 });

src/config-utils.ts

@@ -57,10 +57,24 @@ export interface UserConfig {
   // Set of query filters to include and exclude extra queries based on
   // codeql query suite `include` and `exclude` properties
   "query-filters"?: QueryFilter[];
+
+  // Set of external registries and which packs to retrieve from each
+  registries?: RegistryConfig[];
 }
 
 export type QueryFilter = ExcludeQueryFilter | IncludeQueryFilter;
 
+/**
+ * The list of registries and the associated pack globs that determine where each pack can be downloaded from.
+ */
+export interface RegistryConfig {
+  // URL of a package registry, eg- https://ghcr.io/v2/
+  url: string;
+
+  // List of globs that determine which packs are associated with this registry.
+  packages: string[] | string;
+}
+
 interface ExcludeQueryFilter {
   exclude: Record<string, string[] | string>;
 }
@@ -1686,7 +1700,15 @@ export async function initConfig(
   // happen in the CLI during the `database init` command, so no need
   // to download them here.
   if (!(await useCodeScanningConfigInCli(codeQL))) {
-    await downloadPacks(codeQL, config.languages, config.packs, logger);
+    await downloadPacks(
+      codeQL,
+      config.languages,
+      config.packs,
+      config.originalUserInput.registries,
+      apiDetails,
+      config.tempDir,
+      logger
+    );
   }
 
   // Save the config so we can easily access it again in the future
@@ -1795,30 +1817,79 @@ export async function downloadPacks(
   codeQL: CodeQL,
   languages: Language[],
   packs: Packs,
+  registries: RegistryConfig[] | undefined,
+  apiDetails: api.GitHubApiDetails,
+  tmpDir: string,
   logger: Logger
 ) {
-  let numPacksDownloaded = 0;
-  logger.startGroup("Downloading packs");
-  for (const language of languages) {
-    const packsWithVersion = packs[language];
-    if (packsWithVersion?.length) {
-      logger.info(`Downloading custom packs for ${language}`);
-      const results = await codeQL.packDownload(packsWithVersion);
-      numPacksDownloaded += results.packs.length;
-      logger.info(
-        `Downloaded packs: ${results.packs
-          .map((r) => `${r.name}@${r.version || "latest"}`)
-          .join(", ")}`
-      );
+  let qlconfigFile: string | undefined;
+  if (registries) {
+    // generate a qlconfig.yml file to hold the registry configs.
+    const qlconfig = {
+      registries,
+    };
+    qlconfigFile = path.join(tmpDir, "qlconfig.yml");
+    fs.writeFileSync(qlconfigFile, yaml.dump(qlconfig), "utf8");
+  }
+
+  await wrapEnvironment(
+    {
+      GITHUB_TOKEN: apiDetails.auth,
+      CODEQL_REGISTRIES_AUTH: apiDetails.registriesAuthTokens,
+    },
+    async () => {
+      let numPacksDownloaded = 0;
+      logger.startGroup("Downloading packs");
+      for (const language of languages) {
+        const packsWithVersion = packs[language];
+        if (packsWithVersion?.length) {
+          logger.info(`Downloading custom packs for ${language}`);
+          const results = await codeQL.packDownload(
+            packsWithVersion,
+            qlconfigFile
+          );
+          numPacksDownloaded += results.packs.length;
+          logger.info(
+            `Downloaded: ${results.packs
+              .map((r) => `${r.name}@${r.version || "latest"}`)
+              .join(", ")}`
+          );
+        }
+      }
+      if (numPacksDownloaded > 0) {
+        logger.info(
+          `Downloaded ${numPacksDownloaded} ${packs === 1 ? "pack" : "packs"}`
+        );
+      } else {
+        logger.info("No packs to download");
+      }
+      logger.endGroup();
+    }
+  );
+}
+
+async function wrapEnvironment(
+  env: Record<string, string | undefined>,
+  operation: Function
+) {
+  // Remember the original env
+  const oldEnv = { ...process.env };
+
+  // Set the new env
+  for (const [key, value] of Object.entries(env)) {
+    // Ignore undefined keys
+    if (value !== undefined) {
+      process.env[key] = value;
     }
   }
 
-  if (numPacksDownloaded > 0) {
-    logger.info(
-      `Downloaded ${numPacksDownloaded} ${packs === 1 ? "pack" : "packs"}`
-    );
-  } else {
-    logger.info("No packs to download");
+  try {
+    // Run the operation
+    await operation();
+  } finally {
+    // Restore the old env
+    for (const [key, value] of Object.entries(oldEnv)) {
+      process.env[key] = value;
+    }
   }
-  logger.endGroup();
 }

src/database-upload.test.ts

@@ -39,6 +39,7 @@ const testApiDetails: GitHubApiDetails = {
   auth: "1234",
   url: "https://github.com",
   apiURL: undefined,
+  registriesAuthTokens: undefined,
 };
 
 function getTestConfig(tmpDir: string): Config {

src/feature-flags.test.ts

@@ -24,6 +24,7 @@ const testApiDetails: GitHubApiDetails = {
   auth: "1234",
   url: "https://github.com",
   apiURL: undefined,
+  registriesAuthTokens: undefined,
 };
 
 const testRepositoryNwo = parseRepositoryNwo("github/example");

src/init-action.ts

@@ -148,6 +148,7 @@ async function run() {
     externalRepoAuth: getOptionalInput("external-repository-token"),
     url: getRequiredEnvParam("GITHUB_SERVER_URL"),
     apiURL: getRequiredEnvParam("GITHUB_API_URL"),
+    registriesAuthTokens: getOptionalInput("registries-auth-tokens"),
   };
 
   const gitHubVersion = await getGitHubVersionActionsOnly();

src/runner.ts

@@ -203,6 +203,7 @@ program
         externalRepoAuth: auth,
         url: parseGitHubUrl(cmd.githubUrl),
         apiURL: undefined,
+        registriesAuthTokens: undefined,
       };
 
       const gitHubVersion = await getGitHubVersion(apiDetails);
@@ -479,6 +480,7 @@ program
         auth,
         url: parseGitHubUrl(cmd.githubUrl),
         apiURL: undefined,
+        registriesAuthTokens: undefined,
       };
 
       const outputDir =
@@ -592,6 +594,7 @@ program
       auth,
       url: parseGitHubUrl(cmd.githubUrl),
       apiURL: undefined,
+      registriesAuthTokens: undefined,
     };
     try {
       const gitHubVersion = await getGitHubVersion(apiDetails);

src/upload-sarif-action.ts

@@ -57,6 +57,7 @@ async function run() {
       auth: actionsUtil.getRequiredInput("token"),
       url: getRequiredEnvParam("GITHUB_SERVER_URL"),
       apiURL: getRequiredEnvParam("GITHUB_API_URL"),
+      registriesAuthTokens: undefined,
     };
 
     const gitHubVersion = await getGitHubVersionActionsOnly();

src/util.test.ts

@@ -209,6 +209,7 @@ test("getGitHubVersion", async (t) => {
     auth: "",
     url: "https://github.com",
     apiURL: undefined,
+    registriesAuthTokens: undefined,
   });
   t.deepEqual(util.GitHubVariant.DOTCOM, v.type);
 
@@ -217,6 +218,7 @@ test("getGitHubVersion", async (t) => {
     auth: "",
     url: "https://ghe.example.com",
     apiURL: undefined,
+    registriesAuthTokens: undefined,
   });
   t.deepEqual(
     { type: util.GitHubVariant.GHES, version: "2.0" } as util.GitHubVersion,
@@ -228,6 +230,7 @@ test("getGitHubVersion", async (t) => {
     auth: "",
     url: "https://example.githubenterprise.com",
     apiURL: undefined,
+    registriesAuthTokens: undefined,
   });
   t.deepEqual({ type: util.GitHubVariant.GHAE }, ghae);
 
@@ -236,6 +239,7 @@ test("getGitHubVersion", async (t) => {
     auth: "",
     url: "https://ghe.example.com",
     apiURL: undefined,
+    registriesAuthTokens: undefined,
   });
   t.deepEqual({ type: util.GitHubVariant.DOTCOM }, v3);
 });