Merge branch 'main' into local-bundle

src/analyze-action.ts

@@ -5,16 +5,18 @@ import * as core from "@actions/core";
 
 import * as actionsUtil from "./actions-util";
 import {
-  runAnalyze,
   CodeQLAnalysisError,
   QueriesStatusReport,
   runCleanup,
+  runQueries,
+  runFinalize,
 } from "./analyze";
 import { Config, getConfig } from "./config-utils";
 import { uploadDatabases } from "./database-upload";
 import { getActionsLogger } from "./logging";
 import { parseRepositoryNwo } from "./repository";
 import * as upload_lib from "./upload-lib";
+import { UploadStatusReport } from "./upload-lib";
 import * as util from "./util";
 
 // eslint-disable-next-line import/no-commonjs
@@ -53,7 +55,8 @@ async function sendStatusReport(
 
 async function run() {
   const startedAt = new Date();
-  let stats: AnalysisStatusReport | undefined = undefined;
+  let uploadStats: UploadStatusReport | undefined = undefined;
+  let runStats: QueriesStatusReport | undefined = undefined;
   let config: Config | undefined = undefined;
   util.initializeEnvironment(util.Mode.actions, pkg.version);
 
@@ -82,15 +85,22 @@ async function run() {
       url: util.getRequiredEnvParam("GITHUB_SERVER_URL"),
     };
     const outputDir = actionsUtil.getRequiredInput("output");
-    const queriesStats = await runAnalyze(
-      outputDir,
-      util.getMemoryFlag(actionsUtil.getOptionalInput("ram")),
-      util.getAddSnippetsFlag(actionsUtil.getRequiredInput("add-snippets")),
-      util.getThreadsFlag(actionsUtil.getOptionalInput("threads"), logger),
-      actionsUtil.getOptionalInput("category"),
-      config,
+    const threads = util.getThreadsFlag(
+      actionsUtil.getOptionalInput("threads"),
       logger
     );
+    await runFinalize(outputDir, threads, config, logger);
+    if (actionsUtil.getRequiredInput("skip-queries") !== "true") {
+      runStats = await runQueries(
+        outputDir,
+        util.getMemoryFlag(actionsUtil.getOptionalInput("ram")),
+        util.getAddSnippetsFlag(actionsUtil.getRequiredInput("add-snippets")),
+        threads,
+        actionsUtil.getOptionalInput("category"),
+        config,
+        logger
+      );
+    }
 
     if (actionsUtil.getOptionalInput("cleanup-level") !== "none") {
       await runCleanup(
@@ -106,17 +116,15 @@ async function run() {
     }
     core.setOutput("db-locations", dbLocations);
 
-    if (actionsUtil.getRequiredInput("upload") === "true") {
-      const uploadStats = await upload_lib.uploadFromActions(
+    if (runStats && actionsUtil.getRequiredInput("upload") === "true") {
+      uploadStats = await upload_lib.uploadFromActions(
         outputDir,
         config.gitHubVersion,
         apiDetails,
         logger
       );
-      stats = { ...queriesStats, ...uploadStats };
     } else {
       logger.info("Not uploading results");
-      stats = { ...queriesStats };
     }
 
     const repositoryNwo = parseRepositoryNwo(
@@ -128,10 +136,12 @@ async function run() {
     console.log(error);
 
     if (error instanceof CodeQLAnalysisError) {
-      stats = { ...error.queriesStatusReport };
+      const stats = { ...error.queriesStatusReport };
+      await sendStatusReport(startedAt, stats, error);
+    } else {
+      await sendStatusReport(startedAt, undefined, error);
     }
 
-    await sendStatusReport(startedAt, stats, error);
     return;
   } finally {
     if (core.isDebug() && config !== undefined) {
@@ -161,7 +171,13 @@ async function run() {
     }
   }
 
-  await sendStatusReport(startedAt, stats);
+  if (runStats && uploadStats) {
+    await sendStatusReport(startedAt, { ...runStats, ...uploadStats });
+  } else if (runStats) {
+    await sendStatusReport(startedAt, { ...runStats });
+  } else {
+    await sendStatusReport(startedAt, undefined);
+  }
 }
 
 async function runWrapper() {

src/analyze.ts

@@ -2,6 +2,7 @@ import * as fs from "fs";
 import * as path from "path";
 
 import * as toolrunner from "@actions/exec/lib/toolrunner";
+import * as yaml from "js-yaml";
 
 import * as analysisPaths from "./analysis-paths";
 import { getCodeQL } from "./codeql";
@@ -117,7 +118,10 @@ async function createdDBForScannedLanguages(
 
   const codeql = getCodeQL(config.codeQLCmd);
   for (const language of config.languages) {
-    if (isScannedLanguage(language)) {
+    if (
+      isScannedLanguage(language) &&
+      !dbIsFinalized(config, language, logger)
+    ) {
       logger.startGroup(`Extracting ${language}`);
 
       if (language === Language.python) {
@@ -133,6 +137,25 @@ async function createdDBForScannedLanguages(
   }
 }
 
+function dbIsFinalized(
+  config: configUtils.Config,
+  language: Language,
+  logger: Logger
+) {
+  const dbPath = util.getCodeQLDatabasePath(config, language);
+  try {
+    const dbInfo = yaml.load(
+      fs.readFileSync(path.resolve(dbPath, "codeql-database.yml"), "utf8")
+    );
+    return !("inProgress" in dbInfo);
+  } catch (e) {
+    logger.warning(
+      `Could not check whether database for ${language} was finalized. Assuming it is not.`
+    );
+    return false;
+  }
+}
+
 async function finalizeDatabaseCreation(
   config: configUtils.Config,
   threadsFlag: string,
@@ -142,12 +165,18 @@ async function finalizeDatabaseCreation(
 
   const codeql = getCodeQL(config.codeQLCmd);
   for (const language of config.languages) {
-    logger.startGroup(`Finalizing ${language}`);
-    await codeql.finalizeDatabase(
-      util.getCodeQLDatabasePath(config, language),
-      threadsFlag
-    );
-    logger.endGroup();
+    if (dbIsFinalized(config, language, logger)) {
+      logger.info(
+        `There is already a finalized database for ${language} at the location where the CodeQL Action places databases, so we did not create one.`
+      );
+    } else {
+      logger.startGroup(`Finalizing ${language}`);
+      await codeql.finalizeDatabase(
+        util.getCodeQLDatabasePath(config, language),
+        threadsFlag
+      );
+      logger.endGroup();
+    }
   }
 }
 
@@ -349,33 +378,18 @@ function packWithVersionToQuerySuiteEntry(
   return text;
 }
 
-export async function runAnalyze(
+export async function runFinalize(
   outputDir: string,
-  memoryFlag: string,
-  addSnippetsFlag: string,
   threadsFlag: string,
-  automationDetailsId: string | undefined,
   config: configUtils.Config,
   logger: Logger
-): Promise<QueriesStatusReport> {
+) {
   // Delete the tracer config env var to avoid tracing ourselves
   delete process.env[sharedEnv.ODASA_TRACER_CONFIGURATION];
 
   fs.mkdirSync(outputDir, { recursive: true });
 
   await finalizeDatabaseCreation(config, threadsFlag, logger);
-
-  const queriesStats = await runQueries(
-    outputDir,
-    memoryFlag,
-    addSnippetsFlag,
-    threadsFlag,
-    automationDetailsId,
-    config,
-    logger
-  );
-
-  return { ...queriesStats };
 }
 
 export async function runCleanup(

src/runner.ts

@@ -4,7 +4,7 @@ import * as path from "path";
 
 import { Command } from "commander";
 
-import { runAnalyze } from "./analyze";
+import { runFinalize, runQueries } from "./analyze";
 import { determineAutobuildLanguage, runAutobuild } from "./autobuild";
 import { CodeQL, getCodeQL } from "./codeql";
 import { Config, getConfig } from "./config-utils";
@@ -431,11 +431,13 @@ program
 
       const outputDir =
         cmd.outputDir || path.join(config.tempDir, "codeql-sarif");
-      await runAnalyze(
+      const threads = getThreadsFlag(cmd.threads, logger);
+      await runFinalize(outputDir, threads, config, logger);
+      await runQueries(
         outputDir,
         getMemoryFlag(cmd.ram),
         getAddSnippetsFlag(cmd.addSnippets),
-        getThreadsFlag(cmd.threads, logger),
+        threads,
         cmd.category,
         config,
         logger