Merge pull request #3332 from github/kaspersv/overlay-memory-limit

Overlay: Fall back to full analysis if memory flag is low
2025-12-06 07:48:17 +08:00 · 2025-11-27 15:26:02 +01:00
parent d29b97960c bd8d26b618
commit c178e03ec8
4 changed files with 91 additions and 31 deletions
--- a/lib/init-action.js
+++ b/lib/init-action.js
@@ -86920,7 +86920,7 @@ async function isOverlayAnalysisFeatureEnabled(features, codeql, languages, code
  }
  return true;
 }
-async function getOverlayDatabaseMode(codeql, features, languages, sourceRoot, buildMode, codeScanningConfig, logger) {
+async function getOverlayDatabaseMode(codeql, features, languages, sourceRoot, buildMode, ramInput, codeScanningConfig, logger) {
  let overlayDatabaseMode = "none" /* None */;
  let useOverlayDatabaseCaching = false;
  const modeEnv = process.env.CODEQL_OVERLAY_DATABASE_MODE;
@@ -86936,6 +86936,7 @@ async function getOverlayDatabaseMode(codeql, features, languages, sourceRoot, b
    codeScanningConfig
  )) {
    const diskUsage = await checkDiskUsage(logger);
+    const memoryFlagValue = getMemoryFlagValue(ramInput, logger);
    if (diskUsage === void 0 || diskUsage.numAvailableBytes < OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_BYTES) {
      const diskSpaceMb = diskUsage === void 0 ? 0 : Math.round(diskUsage.numAvailableBytes / 1e6);
      overlayDatabaseMode = "none" /* None */;
@@ -86943,20 +86944,24 @@ async function getOverlayDatabaseMode(codeql, features, languages, sourceRoot, b
      logger.info(
        `Setting overlay database mode to ${overlayDatabaseMode} due to insufficient disk space (${diskSpaceMb} MB).`
      );
-    } else {
-      if (isAnalyzingPullRequest()) {
-        overlayDatabaseMode = "overlay" /* Overlay */;
-        useOverlayDatabaseCaching = true;
-        logger.info(
-          `Setting overlay database mode to ${overlayDatabaseMode} with caching because we are analyzing a pull request.`
-        );
-      } else if (await isAnalyzingDefaultBranch()) {
-        overlayDatabaseMode = "overlay-base" /* OverlayBase */;
-        useOverlayDatabaseCaching = true;
-        logger.info(
-          `Setting overlay database mode to ${overlayDatabaseMode} with caching because we are analyzing the default branch.`
-        );
-      }
+    } else if (memoryFlagValue < 5 * 1024) {
+      overlayDatabaseMode = "none" /* None */;
+      useOverlayDatabaseCaching = false;
+      logger.info(
+        `Setting overlay database mode to ${overlayDatabaseMode} due to insufficient memory for CodeQL analysis (${memoryFlagValue} MB).`
+      );
+    } else if (isAnalyzingPullRequest()) {
+      overlayDatabaseMode = "overlay" /* Overlay */;
+      useOverlayDatabaseCaching = true;
+      logger.info(
+        `Setting overlay database mode to ${overlayDatabaseMode} with caching because we are analyzing a pull request.`
+      );
+    } else if (await isAnalyzingDefaultBranch()) {
+      overlayDatabaseMode = "overlay-base" /* OverlayBase */;
+      useOverlayDatabaseCaching = true;
+      logger.info(
+        `Setting overlay database mode to ${overlayDatabaseMode} with caching because we are analyzing the default branch.`
+      );
    }
  }
  const nonOverlayAnalysis = {
@@ -87051,6 +87056,7 @@ async function initConfig(features, inputs) {
    config.languages,
    inputs.sourceRoot,
    config.buildMode,
+    inputs.ramInput,
    config.computedConfig,
    logger
  );
@@ -89998,6 +90004,7 @@ async function run() {
      queriesInput: getOptionalInput("queries"),
      packsInput: getOptionalInput("packs"),
      buildModeInput: getOptionalInput("build-mode"),
+      ramInput: getOptionalInput("ram"),
      configFile,
      dbLocation: getOptionalInput("db-location"),
      configInput: getOptionalInput("config"),
--- a/src/config-utils.test.ts
+++ b/src/config-utils.test.ts
@@ -59,6 +59,7 @@ function createTestInitConfigInputs(
      dbLocation: undefined,
      configInput: undefined,
      buildModeInput: undefined,
+      ramInput: undefined,
      trapCachingEnabled: false,
      dependencyCachingEnabled: CachingKind.None,
      debugMode: false,
@@ -979,6 +980,7 @@ interface OverlayDatabaseModeTestSetup {
  gitRoot: string | undefined;
  codeScanningConfig: configUtils.UserConfig;
  diskUsage: DiskUsage | undefined;
+  memoryFlagValue: number;
 }

 const defaultOverlayDatabaseModeTestSetup: OverlayDatabaseModeTestSetup = {
@@ -995,6 +997,7 @@ const defaultOverlayDatabaseModeTestSetup: OverlayDatabaseModeTestSetup = {
    numAvailableBytes: 50_000_000_000,
    numTotalBytes: 100_000_000_000,
  },
+  memoryFlagValue: 6920,
 };

 const getOverlayDatabaseModeMacro = test.macro({
@@ -1037,6 +1040,8 @@ const getOverlayDatabaseModeMacro = test.macro({
          .stub(actionsUtil, "isAnalyzingPullRequest")
          .returns(setup.isPullRequest);

+        sinon.stub(util, "getMemoryFlagValue").returns(setup.memoryFlagValue);
+
        // Set up CodeQL mock
        const codeql = mockCodeQLVersion(setup.codeqlVersion);

@@ -1063,6 +1068,7 @@ const getOverlayDatabaseModeMacro = test.macro({
          setup.languages,
          tempDir, // sourceRoot
          setup.buildMode,
+          undefined,
          setup.codeScanningConfig,
          logger,
        );
@@ -1225,6 +1231,24 @@ test(
  },
 );

+test(
+  getOverlayDatabaseModeMacro,
+  "No overlay-base database on default branch if memory flag is too low",
+  {
+    languages: [KnownLanguage.javascript],
+    features: [
+      Feature.OverlayAnalysis,
+      Feature.OverlayAnalysisCodeScanningJavascript,
+    ],
+    isDefaultBranch: true,
+    memoryFlagValue: 3072,
+  },
+  {
+    overlayDatabaseMode: OverlayDatabaseMode.None,
+    useOverlayDatabaseCaching: false,
+  },
+);
+
 test(
  getOverlayDatabaseModeMacro,
  "No overlay-base database on default branch when code-scanning feature enabled with disable-default-queries",
@@ -1434,6 +1458,24 @@ test(
  },
 );

+test(
+  getOverlayDatabaseModeMacro,
+  "No overlay analysis on PR if memory flag is too low",
+  {
+    languages: [KnownLanguage.javascript],
+    features: [
+      Feature.OverlayAnalysis,
+      Feature.OverlayAnalysisCodeScanningJavascript,
+    ],
+    isPullRequest: true,
+    memoryFlagValue: 3072,
+  },
+  {
+    overlayDatabaseMode: OverlayDatabaseMode.None,
+    useOverlayDatabaseCaching: false,
+  },
+);
+
 test(
  getOverlayDatabaseModeMacro,
  "No overlay analysis on PR when code-scanning feature enabled with disable-default-queries",
--- a/src/config-utils.ts
+++ b/src/config-utils.ts
@@ -44,6 +44,7 @@ import {
  cloneObject,
  isDefined,
  checkDiskUsage,
+  getMemoryFlagValue,
 } from "./util";

 export * from "./config/db-config";
@@ -393,6 +394,7 @@ export interface InitConfigInputs {
  dbLocation: string | undefined;
  configInput: string | undefined;
  buildModeInput: string | undefined;
+  ramInput: string | undefined;
  trapCachingEnabled: boolean;
  dependencyCachingEnabled: string | undefined;
  debugMode: boolean;
@@ -661,6 +663,7 @@ export async function getOverlayDatabaseMode(
  languages: Language[],
  sourceRoot: string,
  buildMode: BuildMode | undefined,
+  ramInput: string | undefined,
  codeScanningConfig: UserConfig,
  logger: Logger,
 ): Promise<{
@@ -692,6 +695,7 @@ export async function getOverlayDatabaseMode(
    )
  ) {
    const diskUsage = await checkDiskUsage(logger);
+    const memoryFlagValue = getMemoryFlagValue(ramInput, logger);
    if (
      diskUsage === undefined ||
      diskUsage.numAvailableBytes < OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_BYTES
@@ -706,22 +710,27 @@ export async function getOverlayDatabaseMode(
        `Setting overlay database mode to ${overlayDatabaseMode} ` +
          `due to insufficient disk space (${diskSpaceMb} MB).`,
      );
-    } else {
-      if (isAnalyzingPullRequest()) {
-        overlayDatabaseMode = OverlayDatabaseMode.Overlay;
-        useOverlayDatabaseCaching = true;
-        logger.info(
-          `Setting overlay database mode to ${overlayDatabaseMode} ` +
-            "with caching because we are analyzing a pull request.",
-        );
-      } else if (await isAnalyzingDefaultBranch()) {
-        overlayDatabaseMode = OverlayDatabaseMode.OverlayBase;
-        useOverlayDatabaseCaching = true;
-        logger.info(
-          `Setting overlay database mode to ${overlayDatabaseMode} ` +
-            "with caching because we are analyzing the default branch.",
-        );
-      }
+    } else if (memoryFlagValue < 5 * 1024) {
+      overlayDatabaseMode = OverlayDatabaseMode.None;
+      useOverlayDatabaseCaching = false;
+      logger.info(
+        `Setting overlay database mode to ${overlayDatabaseMode} ` +
+          `due to insufficient memory for CodeQL analysis (${memoryFlagValue} MB).`,
+      );
+    } else if (isAnalyzingPullRequest()) {
+      overlayDatabaseMode = OverlayDatabaseMode.Overlay;
+      useOverlayDatabaseCaching = true;
+      logger.info(
+        `Setting overlay database mode to ${overlayDatabaseMode} ` +
+          "with caching because we are analyzing a pull request.",
+      );
+    } else if (await isAnalyzingDefaultBranch()) {
+      overlayDatabaseMode = OverlayDatabaseMode.OverlayBase;
+      useOverlayDatabaseCaching = true;
+      logger.info(
+        `Setting overlay database mode to ${overlayDatabaseMode} ` +
+          "with caching because we are analyzing the default branch.",
+      );
    }
  }

@@ -875,6 +884,7 @@ export async function initConfig(
      config.languages,
      inputs.sourceRoot,
      config.buildMode,
+      inputs.ramInput,
      config.computedConfig,
      logger,
    );
--- a/src/init-action.ts
+++ b/src/init-action.ts
@@ -324,6 +324,7 @@ async function run() {
      queriesInput: getOptionalInput("queries"),
      packsInput: getOptionalInput("packs"),
      buildModeInput: getOptionalInput("build-mode"),
+      ramInput: getOptionalInput("ram"),
      configFile,
      dbLocation: getOptionalInput("db-location"),
      configInput: getOptionalInput("config"),