Handle errors during database upload

CHANGELOG.md

@@ -2,12 +2,6 @@
 
 ## [UNRELEASED]
 
-## 1.0.2 - 17 Jun 2021
-
-- Fix out of memory in hash computation. [#550](https://github.com/github/codeql-action/pull/550)
-- Clean up logging during analyze results. [#557](https://github.com/github/codeql-action/pull/557)
-- Add `--finalize-dataset` to `database finalize` call, freeing up some disk space after database creation. [#558](https://github.com/github/codeql-action/pull/558)
-
 ## 1.0.1 - 07 Jun 2021
 
 - Pass the `--sarif-group-rules-by-pack` argument to CodeQL CLI invocations that generate SARIF. This means the SARIF rule object for each query will now be found underneath its corresponding query pack in `runs[].tool.extensions`. [#546](https://github.com/github/codeql-action/pull/546)

analyze/action.yml

@@ -34,6 +34,10 @@ inputs:
   category:
     description: String used by Code Scanning for matching the analyses
     required: false
+  upload-database:
+    description: Whether to upload the resulting CodeQL database
+    required: false
+    default: "true"
   token:
     default: ${{ github.token }}
   matrix:

lib/actions-util.js

@@ -448,10 +448,6 @@ async function createStatusReportBase(actionName, status, actionStartedAt, cause
     return statusReport;
 }
 exports.createStatusReportBase = createStatusReportBase;
-function isHTTPError(arg) {
-    var _a;
-    return ((_a = arg) === null || _a === void 0 ? void 0 : _a.status) !== undefined && Number.isInteger(arg.status);
-}
 const GENERIC_403_MSG = "The repo on which this action is running is not opted-in to CodeQL code scanning.";
 const GENERIC_404_MSG = "Not authorized to used the CodeQL code scanning feature on this repo.";
 const OUT_OF_DATE_MSG = "CodeQL Action is out-of-date. Please upgrade to the latest version of codeql-action.";
@@ -481,7 +477,7 @@ async function sendStatusReport(statusReport) {
     }
     catch (e) {
         console.log(e);
-        if (isHTTPError(e)) {
+        if (util_1.isHTTPError(e)) {
             switch (e.status) {
                 case 403:
                     if (workflowIsTriggeredByPushEvent() && isDependabotActor()) {
@@ -540,4 +536,28 @@ function getRelativeScriptPath() {
     return path.relative(actionsDirectory, __filename);
 }
 exports.getRelativeScriptPath = getRelativeScriptPath;
+// Reads the contents of GITHUB_EVENT_PATH as a JSON object
+function getWorkflowEvent() {
+    const eventJsonFile = util_1.getRequiredEnvParam("GITHUB_EVENT_PATH");
+    try {
+        return JSON.parse(fs.readFileSync(eventJsonFile, "utf-8"));
+    }
+    catch (e) {
+        throw new Error(`Unable to read workflow event JSON from ${eventJsonFile}: ${e}`);
+    }
+}
+// Is the version of the repository we are currently analyzing from the default branch,
+// or alternatively from another branch or a pull request.
+async function isAnalyzingDefaultBranch() {
+    var _a, _b;
+    // Get the current ref and trim and refs/heads/ prefix
+    let currentRef = await getRef();
+    currentRef = currentRef.startsWith("refs/heads/")
+        ? currentRef.substr("refs/heads/".length)
+        : currentRef;
+    const event = getWorkflowEvent();
+    const defaultBranch = (_b = (_a = event) === null || _a === void 0 ? void 0 : _a.repository) === null || _b === void 0 ? void 0 : _b.default_branch;
+    return currentRef === defaultBranch;
+}
+exports.isAnalyzingDefaultBranch = isAnalyzingDefaultBranch;
 //# sourceMappingURL=actions-util.js.map

lib/analyze-action.js

@@ -12,8 +12,11 @@ const path = __importStar(require("path"));
 const core = __importStar(require("@actions/core"));
 const actionsUtil = __importStar(require("./actions-util"));
 const analyze_1 = require("./analyze");
+const api_client_1 = require("./api-client");
+const codeql_1 = require("./codeql");
 const config_utils_1 = require("./config-utils");
 const logging_1 = require("./logging");
+const repository_1 = require("./repository");
 const upload_lib = __importStar(require("./upload-lib"));
 const util = __importStar(require("./util"));
 // eslint-disable-next-line import/no-commonjs
@@ -30,6 +33,63 @@ async function sendStatusReport(startedAt, stats, error) {
     };
     await actionsUtil.sendStatusReport(statusReport);
 }
+async function uploadDatabases(repositoryNwo, config, apiDetails, logger) {
+    if (actionsUtil.getRequiredInput("upload-database") !== "true") {
+        logger.debug("Database upload disabled in workflow. Skipping upload.");
+        return;
+    }
+    // Do nothing when not running against github.com
+    if (config.gitHubVersion.type !== util.GitHubVariant.DOTCOM) {
+        logger.debug("Not running against github.com. Skipping upload.");
+        return;
+    }
+    if (!(await actionsUtil.isAnalyzingDefaultBranch())) {
+        // We only want to upload a database if we are analyzing the default branch.
+        logger.debug("Not analyzing default branch. Skipping upload.");
+        return;
+    }
+    const client = api_client_1.getApiClient(apiDetails);
+    try {
+        await client.request("GET /repos/:owner/:repo/code-scanning/databases", {
+            owner: repositoryNwo.owner,
+            repo: repositoryNwo.repo,
+        });
+    }
+    catch (e) {
+        console.log(e);
+        if (util.isHTTPError(e)) {
+            if (e.status === 404) {
+                logger.debug("Repository is not opted in to database uploads. Skipping upload.");
+            }
+            else {
+                logger.debug(`Skipping database upload due to unknown error: ${e}`);
+            }
+            return;
+        }
+    }
+    const codeql = codeql_1.getCodeQL(config.codeQLCmd);
+    for (const language of config.languages) {
+        // Bundle the database up into a single zip file
+        const databasePath = util.getCodeQLDatabasePath(config, language);
+        const databaseBundlePath = `${databasePath}.zip`;
+        await codeql.databaseBundle(databasePath, databaseBundlePath);
+        // Upload the database bundle
+        const payload = fs.readFileSync(databaseBundlePath);
+        try {
+            await client.request(`PUT /repos/:owner/:repo/code-scanning/databases/${language}`, {
+                owner: repositoryNwo.owner,
+                repo: repositoryNwo.repo,
+                data: payload,
+            });
+            logger.debug(`Successfully uploaded database for ${language}`);
+        }
+        catch (e) {
+            console.log(e);
+            // Log a warning but don't fail the workflow
+            logger.warning(`Failed to upload database for ${language}: ${e}`);
+        }
+    }
+}
 async function run() {
     const startedAt = new Date();
     let stats = undefined;
@@ -66,6 +126,8 @@ async function run() {
             logger.info("Not uploading results");
             stats = { ...queriesStats };
         }
+        const repositoryNwo = repository_1.parseRepositoryNwo(util.getRequiredEnvParam("GITHUB_REPOSITORY"));
+        await uploadDatabases(repositoryNwo, config, apiDetails, logger);
     }
     catch (error) {
         core.setFailed(error.message);

lib/codeql.js

@@ -287,6 +287,7 @@ function setCodeQL(partialCodeql) {
         resolveQueries: resolveFunction(partialCodeql, "resolveQueries"),
         packDownload: resolveFunction(partialCodeql, "packDownload"),
         databaseCleanup: resolveFunction(partialCodeql, "databaseCleanup"),
+        databaseBundle: resolveFunction(partialCodeql, "databaseBundle"),
         databaseRunQueries: resolveFunction(partialCodeql, "databaseRunQueries"),
         databaseInterpretResults: resolveFunction(partialCodeql, "databaseInterpretResults"),
     };
@@ -535,6 +536,15 @@ function getCodeQLForCmd(cmd) {
             ];
             await runTool(cmd, codeqlArgs);
         },
+        async databaseBundle(databasePath, outputFilePath) {
+            const args = [
+                "database",
+                "bundle",
+                databasePath,
+                `--output=${outputFilePath}`,
+            ];
+            await new toolrunner.ToolRunner(cmd, args).exec();
+        },
     };
 }
 function packWithVersionToString(pack) {

lib/util.js

@@ -408,4 +408,9 @@ function getRequiredEnvParam(paramName) {
     return value;
 }
 exports.getRequiredEnvParam = getRequiredEnvParam;
+function isHTTPError(arg) {
+    var _a;
+    return ((_a = arg) === null || _a === void 0 ? void 0 : _a.status) !== undefined && Number.isInteger(arg.status);
+}
+exports.isHTTPError = isHTTPError;
 //# sourceMappingURL=util.js.map

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "codeql",
-  "version": "1.0.3",
+  "version": "1.0.2",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "codeql",
-  "version": "1.0.3",
+  "version": "1.0.2",
   "private": true,
   "description": "CodeQL action",
   "scripts": {

runner/package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "codeql-runner",
-  "version": "1.0.3",
+  "version": "1.0.2",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {

runner/package.json

@@ -1,6 +1,6 @@
 {
   "name": "codeql-runner",
-  "version": "1.0.3",
+  "version": "1.0.2",
   "private": true,
   "description": "CodeQL runner",
   "scripts": {

src/actions-util.ts

@@ -8,7 +8,7 @@ import * as yaml from "js-yaml";
 
 import * as api from "./api-client";
 import * as sharedEnv from "./shared-environment";
-import { getRequiredEnvParam, GITHUB_DOTCOM_URL } from "./util";
+import { getRequiredEnvParam, GITHUB_DOTCOM_URL, isHTTPError } from "./util";
 
 /**
  * The utils in this module are meant to be run inside of the action only.
@@ -576,15 +576,6 @@ export async function createStatusReportBase(
   return statusReport;
 }
 
-interface HTTPError {
-  status: number;
-  message?: string;
-}
-
-function isHTTPError(arg: any): arg is HTTPError {
-  return arg?.status !== undefined && Number.isInteger(arg.status);
-}
-
 const GENERIC_403_MSG =
   "The repo on which this action is running is not opted-in to CodeQL code scanning.";
 const GENERIC_404_MSG =
@@ -691,3 +682,30 @@ export function getRelativeScriptPath(): string {
   const actionsDirectory = path.join(path.dirname(runnerTemp), "_actions");
   return path.relative(actionsDirectory, __filename);
 }
+
+// Reads the contents of GITHUB_EVENT_PATH as a JSON object
+function getWorkflowEvent(): any {
+  const eventJsonFile = getRequiredEnvParam("GITHUB_EVENT_PATH");
+  try {
+    return JSON.parse(fs.readFileSync(eventJsonFile, "utf-8"));
+  } catch (e) {
+    throw new Error(
+      `Unable to read workflow event JSON from ${eventJsonFile}: ${e}`
+    );
+  }
+}
+
+// Is the version of the repository we are currently analyzing from the default branch,
+// or alternatively from another branch or a pull request.
+export async function isAnalyzingDefaultBranch(): Promise<boolean> {
+  // Get the current ref and trim and refs/heads/ prefix
+  let currentRef = await getRef();
+  currentRef = currentRef.startsWith("refs/heads/")
+    ? currentRef.substr("refs/heads/".length)
+    : currentRef;
+
+  const event = getWorkflowEvent();
+  const defaultBranch = event?.repository?.default_branch;
+
+  return currentRef === defaultBranch;
+}

src/analyze-action.ts

@@ -10,8 +10,11 @@ import {
   QueriesStatusReport,
   runCleanup,
 } from "./analyze";
+import { getApiClient, GitHubApiDetails } from "./api-client";
+import { getCodeQL } from "./codeql";
 import { Config, getConfig } from "./config-utils";
-import { getActionsLogger } from "./logging";
+import { getActionsLogger, Logger } from "./logging";
+import { parseRepositoryNwo, RepositoryNwo } from "./repository";
 import * as upload_lib from "./upload-lib";
 import * as util from "./util";
 
@@ -49,6 +52,76 @@ async function sendStatusReport(
   await actionsUtil.sendStatusReport(statusReport);
 }
 
+async function uploadDatabases(
+  repositoryNwo: RepositoryNwo,
+  config: Config,
+  apiDetails: GitHubApiDetails,
+  logger: Logger
+): Promise<void> {
+  if (actionsUtil.getRequiredInput("upload-database") !== "true") {
+    logger.debug("Database upload disabled in workflow. Skipping upload.");
+    return;
+  }
+
+  // Do nothing when not running against github.com
+  if (config.gitHubVersion.type !== util.GitHubVariant.DOTCOM) {
+    logger.debug("Not running against github.com. Skipping upload.");
+    return;
+  }
+
+  if (!(await actionsUtil.isAnalyzingDefaultBranch())) {
+    // We only want to upload a database if we are analyzing the default branch.
+    logger.debug("Not analyzing default branch. Skipping upload.");
+    return;
+  }
+
+  const client = getApiClient(apiDetails);
+  try {
+    await client.request("GET /repos/:owner/:repo/code-scanning/databases", {
+      owner: repositoryNwo.owner,
+      repo: repositoryNwo.repo,
+    });
+  } catch (e) {
+    console.log(e);
+    if (util.isHTTPError(e)) {
+      if (e.status === 404) {
+        logger.debug(
+          "Repository is not opted in to database uploads. Skipping upload."
+        );
+      } else {
+        logger.debug(`Skipping database upload due to unknown error: ${e}`);
+      }
+      return;
+    }
+  }
+
+  const codeql = getCodeQL(config.codeQLCmd);
+  for (const language of config.languages) {
+    // Bundle the database up into a single zip file
+    const databasePath = util.getCodeQLDatabasePath(config, language);
+    const databaseBundlePath = `${databasePath}.zip`;
+    await codeql.databaseBundle(databasePath, databaseBundlePath);
+
+    // Upload the database bundle
+    const payload = fs.readFileSync(databaseBundlePath);
+    try {
+      await client.request(
+        `PUT /repos/:owner/:repo/code-scanning/databases/${language}`,
+        {
+          owner: repositoryNwo.owner,
+          repo: repositoryNwo.repo,
+          data: payload,
+        }
+      );
+      logger.debug(`Successfully uploaded database for ${language}`);
+    } catch (e) {
+      console.log(e);
+      // Log a warning but don't fail the workflow
+      logger.warning(`Failed to upload database for ${language}: ${e}`);
+    }
+  }
+}
+
 async function run() {
   const startedAt = new Date();
   let stats: AnalysisStatusReport | undefined = undefined;
@@ -116,6 +189,11 @@ async function run() {
       logger.info("Not uploading results");
       stats = { ...queriesStats };
     }
+
+    const repositoryNwo = parseRepositoryNwo(
+      util.getRequiredEnvParam("GITHUB_REPOSITORY")
+    );
+    await uploadDatabases(repositoryNwo, config, apiDetails, logger);
   } catch (error) {
     core.setFailed(error.message);
     console.log(error);

src/codeql.ts

@@ -99,6 +99,10 @@ export interface CodeQL {
    * Run 'codeql database cleanup'.
    */
   databaseCleanup(databasePath: string, cleanupLevel: string): Promise<void>;
+  /**
+   * Run 'codeql database bundle'.
+   */
+  databaseBundle(databasePath: string, outputFilePath: string): Promise<void>;
   /**
    * Run 'codeql database run-queries'.
    */
@@ -512,6 +516,7 @@ export function setCodeQL(partialCodeql: Partial<CodeQL>): CodeQL {
     resolveQueries: resolveFunction(partialCodeql, "resolveQueries"),
     packDownload: resolveFunction(partialCodeql, "packDownload"),
     databaseCleanup: resolveFunction(partialCodeql, "databaseCleanup"),
+    databaseBundle: resolveFunction(partialCodeql, "databaseBundle"),
     databaseRunQueries: resolveFunction(partialCodeql, "databaseRunQueries"),
     databaseInterpretResults: resolveFunction(
       partialCodeql,
@@ -829,6 +834,18 @@ function getCodeQLForCmd(cmd: string): CodeQL {
       ];
       await runTool(cmd, codeqlArgs);
     },
+    async databaseBundle(
+      databasePath: string,
+      outputFilePath: string
+    ): Promise<void> {
+      const args = [
+        "database",
+        "bundle",
+        databasePath,
+        `--output=${outputFilePath}`,
+      ];
+      await new toolrunner.ToolRunner(cmd, args).exec();
+    },
   };
 }
 

src/util.ts

@@ -478,3 +478,12 @@ export function getRequiredEnvParam(paramName: string): string {
   }
   return value;
 }
+
+export interface HTTPError {
+  status: number;
+  message?: string;
+}
+
+export function isHTTPError(arg: any): arg is HTTPError {
+  return arg?.status !== undefined && Number.isInteger(arg.status);
+}