Merge branch 'main' into aeisenberg/checkout-path-commitoid

src/actions-util.ts

@@ -1,4 +1,5 @@
 import * as fs from "fs";
+import * as os from "os";
 import * as path from "path";
 
 import * as core from "@actions/core";
@@ -9,12 +10,17 @@ import * as yaml from "js-yaml";
 import * as api from "./api-client";
 import * as sharedEnv from "./shared-environment";
 import {
+  getCachedCodeQlVersion,
   getRequiredEnvParam,
   GITHUB_DOTCOM_URL,
+  isGitHubGhesVersionBelow,
   isHTTPError,
   UserError,
 } from "./util";
 
+// eslint-disable-next-line import/no-commonjs
+const pkg = require("../package.json");
+
 /**
  * The utils in this module are meant to be run inside of the action only.
  * Code paths from the runner should not enter this module.
@@ -608,6 +614,16 @@ export interface StatusReportBase {
   cause?: string;
   /** Stack trace of the failure (or undefined if status is not failure). */
   exception?: string;
+  /** Action runner operating system (context runner.os). */
+  runner_os: string;
+  /** Action runner hardware architecture (context runner.arch). */
+  runner_arch?: string;
+  /** Action runner operating system release (x.y.z from os.release()). */
+  runner_os_release?: string;
+  /** Action version (x.y.z from package.json). */
+  action_version: string;
+  /** CodeQL CLI version (x.y.z from the CLI). */
+  codeql_version?: string;
 }
 
 export function getActionsStatus(
@@ -655,6 +671,9 @@ export async function createStatusReportBase(
       workflowStartedAt
     );
   }
+  const runnerOs = getRequiredEnvParam("RUNNER_OS");
+  const codeQlCliVersion = getCachedCodeQlVersion();
+
   // If running locally then the GITHUB_ACTION_REF cannot be trusted as it may be for the previous action
   // See https://github.com/actions/runner/issues/803
   const actionRef = isRunningLocalAction()
@@ -674,6 +693,8 @@ export async function createStatusReportBase(
     started_at: workflowStartedAt,
     action_started_at: actionStartedAt.toISOString(),
     status,
+    runner_os: runnerOs,
+    action_version: pkg.version,
   };
 
   // Add optional parameters
@@ -695,6 +716,17 @@ export async function createStatusReportBase(
   if (matrix) {
     statusReport.matrix_vars = matrix;
   }
+  if ("RUNNER_ARCH" in process.env) {
+    // RUNNER_ARCH is available only in GHES 3.4 and later
+    // Values other than X86, X64, ARM, or ARM64 are discarded server side
+    statusReport.runner_arch = process.env["RUNNER_ARCH"];
+  }
+  if (runnerOs === "Windows" || runnerOs === "macOS") {
+    statusReport.runner_os_release = os.release();
+  }
+  if (codeQlCliVersion !== undefined) {
+    statusReport.codeql_version = codeQlCliVersion;
+  }
 
   return statusReport;
 }
@@ -720,6 +752,14 @@ const INCOMPATIBLE_MSG =
 export async function sendStatusReport<S extends StatusReportBase>(
   statusReport: S
 ): Promise<boolean> {
+  const gitHubVersion = await api.getGitHubVersionActionsOnly();
+  if (isGitHubGhesVersionBelow(gitHubVersion, "3.2.0")) {
+    // GHES 3.1 and earlier versions reject unexpected properties, which means
+    // that they will reject status reports with newly added properties.
+    // Inhibiting status reporting for GHES < 3.2 avoids such failures.
+    return true;
+  }
+
   const statusReportJSON = JSON.stringify(statusReport);
   core.debug(`Sending status report: ${statusReportJSON}`);
   // If in test mode we don't want to upload the results
@@ -830,7 +870,7 @@ export async function isAnalyzingDefaultBranch(): Promise<boolean> {
   // Get the current ref and trim and refs/heads/ prefix
   let currentRef = await getRef();
   currentRef = currentRef.startsWith("refs/heads/")
-    ? currentRef.substr("refs/heads/".length)
+    ? currentRef.slice("refs/heads/".length)
     : currentRef;
 
   const event = getWorkflowEvent();

src/analyze.ts

@@ -6,6 +6,7 @@ import * as yaml from "js-yaml";
 
 import * as analysisPaths from "./analysis-paths";
 import {
+  CODEQL_VERSION_CONFIG_FILES,
   CODEQL_VERSION_COUNTS_LINES,
   CODEQL_VERSION_NEW_TRACING,
   getCodeQL,
@@ -235,12 +236,15 @@ export async function runQueries(
       );
     }
 
+    const codeql = await getCodeQL(config.codeQLCmd);
     try {
-      if (hasPackWithCustomQueries) {
+      if (
+        hasPackWithCustomQueries &&
+        !(await util.codeQlVersionAbove(codeql, CODEQL_VERSION_CONFIG_FILES))
+      ) {
         logger.info("Performing analysis with custom CodeQL Packs.");
         logger.startGroup(`Downloading custom packs for ${language}`);
 
-        const codeql = await getCodeQL(config.codeQLCmd);
         const results = await codeql.packDownload(packsWithVersion);
         logger.info(
           `Downloaded packs: ${results.packs

src/api-client.ts

@@ -5,7 +5,8 @@ import * as retry from "@octokit/plugin-retry";
 import consoleLogLevel from "console-log-level";
 
 import { getRequiredInput } from "./actions-util";
-import { getMode, getRequiredEnvParam } from "./util";
+import * as util from "./util";
+import { getMode, getRequiredEnvParam, GitHubVersion } from "./util";
 
 // eslint-disable-next-line import/no-commonjs
 const pkg = require("../package.json");
@@ -58,14 +59,36 @@ function getApiUrl(githubUrl: string): string {
   return url.toString();
 }
 
+function getApiDetails() {
+  return {
+    auth: getRequiredInput("token"),
+    url: getRequiredEnvParam("GITHUB_SERVER_URL"),
+  };
+}
+
 // Temporary function to aid in the transition to running on and off of github actions.
 // Once all code has been converted this function should be removed or made canonical
 // and called only from the action entrypoints.
 export function getActionsApiClient() {
-  const apiDetails = {
-    auth: getRequiredInput("token"),
-    url: getRequiredEnvParam("GITHUB_SERVER_URL"),
-  };
-
-  return getApiClient(apiDetails);
+  return getApiClient(getApiDetails());
+}
+
+let cachedGitHubVersion: GitHubVersion | undefined = undefined;
+
+/**
+ * Report the GitHub server version. This is a wrapper around
+ * util.getGitHubVersion() that automatically supplies GitHub API details using
+ * GitHub Action inputs. If you need to get the GitHub server version from the
+ * Runner, please call util.getGitHubVersion() instead.
+ *
+ * @returns GitHub version
+ */
+export async function getGitHubVersionActionsOnly(): Promise<GitHubVersion> {
+  if (!util.isActions()) {
+    throw new Error("getGitHubVersionActionsOnly() works only in an action");
+  }
+  if (cachedGitHubVersion === undefined) {
+    cachedGitHubVersion = await util.getGitHubVersion(getApiDetails());
+  }
+  return cachedGitHubVersion;
 }

src/api-compatibility.json

@@ -1 +1 @@
-{"maximumVersion": "3.4", "minimumVersion": "3.1"}
+{"maximumVersion": "3.5", "minimumVersion": "3.1"}

src/codeql.ts

@@ -4,12 +4,13 @@ import * as path from "path";
 import * as toolrunner from "@actions/exec/lib/toolrunner";
 import { IHeaders } from "@actions/http-client/interfaces";
 import { default as deepEqual } from "fast-deep-equal";
+import * as yaml from "js-yaml";
 import { default as queryString } from "query-string";
 import * as semver from "semver";
 
 import { isRunningLocalAction, getRelativeScriptPath } from "./actions-util";
 import * as api from "./api-client";
-import { PackWithVersion } from "./config-utils";
+import { Config, PackWithVersion } from "./config-utils";
 import * as defaults from "./defaults.json"; // Referenced from codeql-action-sync-tool!
 import { errorMatchers } from "./error-matcher";
 import { isTracedLanguage, Language } from "./languages";
@@ -80,8 +81,7 @@ export interface CodeQL {
    * Run 'codeql database init --db-cluster'.
    */
   databaseInitCluster(
-    databasePath: string,
-    languages: Language[],
+    config: Config,
     sourceRoot: string,
     processName: string | undefined,
     processLevel: number | undefined
@@ -220,6 +220,7 @@ const CODEQL_VERSION_GROUP_RULES = "2.5.5";
 const CODEQL_VERSION_SARIF_GROUP = "2.5.3";
 export const CODEQL_VERSION_COUNTS_LINES = "2.6.2";
 const CODEQL_VERSION_CUSTOM_QUERY_HELP = "2.7.1";
+export const CODEQL_VERSION_CONFIG_FILES = "2.8.2"; // Versions before 2.8.2 weren't tolerant to unknown properties
 export const CODEQL_VERSION_ML_POWERED_QUERIES = "2.7.5";
 
 /**
@@ -366,6 +367,19 @@ async function getCodeQLBundleDownloadURL(
   return `https://github.com/${CODEQL_DEFAULT_ACTION_REPOSITORY}/releases/download/${CODEQL_BUNDLE_VERSION}/${codeQLBundleName}`;
 }
 
+/**
+ * Set up CodeQL CLI access.
+ *
+ * @param codeqlURL
+ * @param apiDetails
+ * @param tempDir
+ * @param toolCacheDir
+ * @param variant
+ * @param logger
+ * @param checkVersion Whether to check that CodeQL CLI meets the minimum
+ *        version requirement. Must be set to true outside tests.
+ * @returns
+ */
 export async function setupCodeQL(
   codeqlURL: string | undefined,
   apiDetails: api.GitHubApiDetails,
@@ -610,19 +624,29 @@ export async function getCodeQLForTesting(): Promise<CodeQL> {
   return getCodeQLForCmd("codeql-for-testing", false);
 }
 
+/**
+ * Return a CodeQL object for CodeQL CLI access.
+ *
+ * @param cmd Path to CodeQL CLI
+ * @param checkVersion Whether to check that CodeQL CLI meets the minimum
+ *        version requirement. Must be set to true outside tests.
+ * @returns A new CodeQL object
+ */
 async function getCodeQLForCmd(
   cmd: string,
   checkVersion: boolean
 ): Promise<CodeQL> {
-  let cachedVersion: undefined | Promise<string> = undefined;
   const codeql = {
     getPath() {
       return cmd;
     },
     async getVersion() {
-      if (cachedVersion === undefined)
-        cachedVersion = runTool(cmd, ["version", "--format=terse"]);
-      return await cachedVersion;
+      let result = util.getCachedCodeQlVersion();
+      if (result === undefined) {
+        result = await runTool(cmd, ["version", "--format=terse"]);
+        util.cacheCodeQlVersion(result);
+      }
+      return result;
     },
     async printVersion() {
       await runTool(cmd, ["version", "--format=json"]);
@@ -692,26 +716,35 @@ async function getCodeQLForCmd(
       ]);
     },
     async databaseInitCluster(
-      databasePath: string,
-      languages: Language[],
+      config: Config,
       sourceRoot: string,
       processName: string | undefined,
       processLevel: number | undefined
     ) {
-      const extraArgs = languages.map((language) => `--language=${language}`);
-      if (languages.filter(isTracedLanguage).length > 0) {
+      const extraArgs = config.languages.map(
+        (language) => `--language=${language}`
+      );
+      if (config.languages.filter(isTracedLanguage).length > 0) {
         extraArgs.push("--begin-tracing");
         if (processName !== undefined) {
           extraArgs.push(`--trace-process-name=${processName}`);
         } else {
+          // We default to 3 if no other arguments are provided since this was the default
+          // behaviour of the Runner. Note this path never happens in the CodeQL Action
+          // because that always passes in a process name.
           extraArgs.push(`--trace-process-level=${processLevel || 3}`);
         }
       }
+      if (await util.codeQlVersionAbove(codeql, CODEQL_VERSION_CONFIG_FILES)) {
+        const configLocation = path.resolve(config.tempDir, "user-config.yaml");
+        fs.writeFileSync(configLocation, yaml.dump(config.originalUserInput));
+        extraArgs.push(`--codescanning-config=${configLocation}`);
+      }
       await runTool(cmd, [
         "database",
         "init",
         "--db-cluster",
-        databasePath,
+        config.dbLocation,
         `--source-root=${sourceRoot}`,
         ...extraArgs,
         ...getExtraOptionsFromEnv(["database", "init"]),
@@ -864,7 +897,9 @@ async function getCodeQLForCmd(
       if (extraSearchPath !== undefined) {
         codeqlArgs.push("--additional-packs", extraSearchPath);
       }
-      codeqlArgs.push(querySuitePath);
+      if (!(await util.codeQlVersionAbove(this, CODEQL_VERSION_CONFIG_FILES))) {
+        codeqlArgs.push(querySuitePath);
+      }
       await runTool(cmd, codeqlArgs);
     },
     async databaseInterpretResults(
@@ -899,7 +934,10 @@ async function getCodeQLForCmd(
       ) {
         codeqlArgs.push("--sarif-category", automationDetailsId);
       }
-      codeqlArgs.push(databasePath, ...querySuitePaths);
+      codeqlArgs.push(databasePath);
+      if (!(await util.codeQlVersionAbove(this, CODEQL_VERSION_CONFIG_FILES))) {
+        codeqlArgs.push(...querySuitePaths);
+      }
       // capture stdout, which contains analysis summaries
       return await runTool(cmd, codeqlArgs);
     },
@@ -982,6 +1020,14 @@ async function getCodeQLForCmd(
       await new toolrunner.ToolRunner(cmd, args).exec();
     },
   };
+  // To ensure that status reports include the CodeQL CLI version whereever
+  // possbile, we want to call getVersion(), which populates the version value
+  // used by status reporting, at the earliest opportunity. But invoking
+  // getVersion() directly here breaks tests that only pretend to create a
+  // CodeQL object. So instead we rely on the assumption that all non-test
+  // callers would set checkVersion to true, and util.codeQlVersionAbove()
+  // would call getVersion(), so the CLI version would be cached as soon as the
+  // CodeQL object is created.
   if (
     checkVersion &&
     !(await util.codeQlVersionAbove(codeql, CODEQL_MINIMUM_VERSION))

src/config-utils.test.ts

@@ -1811,7 +1811,7 @@ test(
   true,
   undefined,
   "security-extended",
-  "~0.0.2"
+  "~0.1.0"
 );
 test(
   mlPoweredQueriesMacro,
@@ -1819,7 +1819,7 @@ test(
   true,
   undefined,
   "security-and-quality",
-  "~0.0.2"
+  "~0.1.0"
 );
 test(
   mlPoweredQueriesMacro,

src/config-utils.ts

@@ -849,7 +849,7 @@ async function addQueriesAndPacksFromWorkflow(
 // should instead be added in addition
 function shouldAddConfigFileQueries(queriesInput: string | undefined): boolean {
   if (queriesInput) {
-    return queriesInput.trimStart().substr(0, 1) === "+";
+    return queriesInput.trimStart().slice(0, 1) === "+";
   }
 
   return true;

src/defaults.json

@@ -1,3 +1,3 @@
 {
-  "bundleVersion": "codeql-bundle-20220224"
+  "bundleVersion": "codeql-bundle-20220311"
 }

src/init-action.ts

@@ -13,6 +13,7 @@ import {
   StatusReportBase,
   validateWorkflow,
 } from "./actions-util";
+import { getGitHubVersionActionsOnly } from "./api-client";
 import { CodeQL, CODEQL_VERSION_NEW_TRACING } from "./codeql";
 import * as configUtils from "./config-utils";
 import { GitHubFeatureFlags } from "./feature-flags";
@@ -31,7 +32,6 @@ import {
   initializeEnvironment,
   Mode,
   checkGitHubVersionInRange,
-  getGitHubVersion,
   codeQlVersionAbove,
   enrichEnvironment,
   getMemoryFlagValue,
@@ -99,7 +99,7 @@ async function sendSuccessStatusReport(
   }
   if (queriesInput !== undefined) {
     queriesInput = queriesInput.startsWith("+")
-      ? queriesInput.substr(1)
+      ? queriesInput.slice(1)
       : queriesInput;
     queries.push(...queriesInput.split(","));
   }
@@ -135,7 +135,7 @@ async function run() {
     url: getRequiredEnvParam("GITHUB_SERVER_URL"),
   };
 
-  const gitHubVersion = await getGitHubVersion(apiDetails);
+  const gitHubVersion = await getGitHubVersionActionsOnly();
   checkGitHubVersionInRange(gitHubVersion, logger, Mode.actions);
 
   const repositoryNwo = parseRepositoryNwo(

src/init.ts

@@ -95,8 +95,7 @@ export async function runInit(
     if (await codeQlVersionAbove(codeql, CODEQL_VERSION_NEW_TRACING)) {
       // Init a database cluster
       await codeql.databaseInitCluster(
-        config.dbLocation,
-        config.languages,
+        config,
         sourceRoot,
         processName,
         processLevel

src/upload-sarif-action.ts

@@ -1,15 +1,11 @@
 import * as core from "@actions/core";
 
 import * as actionsUtil from "./actions-util";
+import { getGitHubVersionActionsOnly } from "./api-client";
 import { getActionsLogger } from "./logging";
 import { parseRepositoryNwo } from "./repository";
 import * as upload_lib from "./upload-lib";
-import {
-  getGitHubVersion,
-  getRequiredEnvParam,
-  initializeEnvironment,
-  Mode,
-} from "./util";
+import { getRequiredEnvParam, initializeEnvironment, Mode } from "./util";
 
 // eslint-disable-next-line import/no-commonjs
 const pkg = require("../package.json");
@@ -55,7 +51,7 @@ async function run() {
       url: getRequiredEnvParam("GITHUB_SERVER_URL"),
     };
 
-    const gitHubVersion = await getGitHubVersion(apiDetails);
+    const gitHubVersion = await getGitHubVersionActionsOnly();
 
     const uploadResult = await upload_lib.uploadFromActions(
       actionsUtil.getRequiredInput("sarif_file"),

src/util.test.ts

@@ -353,3 +353,30 @@ for (const [packs, expectedStatus] of ML_POWERED_JS_STATUS_TESTS) {
     });
   });
 }
+
+test("isGitHubGhesVersionBelow", async (t) => {
+  t.falsy(
+    util.isGitHubGhesVersionBelow({ type: util.GitHubVariant.DOTCOM }, "3.2.0")
+  );
+  t.falsy(
+    util.isGitHubGhesVersionBelow({ type: util.GitHubVariant.GHAE }, "3.2.0")
+  );
+  t.falsy(
+    util.isGitHubGhesVersionBelow(
+      { type: util.GitHubVariant.GHES, version: "3.3.0" },
+      "3.2.0"
+    )
+  );
+  t.falsy(
+    util.isGitHubGhesVersionBelow(
+      { type: util.GitHubVariant.GHES, version: "3.2.0" },
+      "3.2.0"
+    )
+  );
+  t.true(
+    util.isGitHubGhesVersionBelow(
+      { type: util.GitHubVariant.GHES, version: "3.1.2" },
+      "3.2.0"
+    )
+  );
+});

src/util.ts

@@ -594,6 +594,29 @@ export function isHTTPError(arg: any): arg is HTTPError {
   return arg?.status !== undefined && Number.isInteger(arg.status);
 }
 
+export function isGitHubGhesVersionBelow(
+  gitHubVersion: GitHubVersion,
+  expectedVersion: string
+): boolean {
+  return (
+    gitHubVersion.type === GitHubVariant.GHES &&
+    semver.lt(gitHubVersion.version, expectedVersion)
+  );
+}
+
+let cachedCodeQlVersion: undefined | string = undefined;
+
+export function cacheCodeQlVersion(version: string): void {
+  if (cachedCodeQlVersion !== undefined) {
+    throw new Error("cacheCodeQlVersion() should be called only once");
+  }
+  cachedCodeQlVersion = version;
+}
+
+export function getCachedCodeQlVersion(): undefined | string {
+  return cachedCodeQlVersion;
+}
+
 export async function codeQlVersionAbove(
   codeql: CodeQL,
   requiredVersion: string
@@ -636,7 +659,7 @@ export function isGoodVersion(versionSpec: string) {
  */
 export const ML_POWERED_JS_QUERIES_PACK: PackWithVersion = {
   packName: "codeql/javascript-experimental-atm-queries",
-  version: "~0.0.2",
+  version: "~0.1.0",
 };
 
 /**