Respect scaling_reserved_ram feature flag

The amount of RAM given to the CodeQL evaluator is the machine's total
memory size, minus a reserved amount. Currently, the reserved amount is
fixed at 1 GB (or 1.5 GB on Windows). When the scaling_reserved_ram
feature flag is enabled, we also add 2% of the total memory size to the
reserved amount. This allows for the fact that the kernel will consume
more RAM (e.g. for page tables) on machines with more physical RAM.

lib/analyze-action.js

@@ -158,10 +158,10 @@ async function run() {
         const apiDetails = (0, api_client_1.getApiDetails)();
         const outputDir = actionsUtil.getRequiredInput("output");
         const threads = util.getThreadsFlag(actionsUtil.getOptionalInput("threads") || process.env["CODEQL_THREADS"], logger);
-        const memory = util.getMemoryFlag(actionsUtil.getOptionalInput("ram") || process.env["CODEQL_RAM"]);
         const repositoryNwo = (0, repository_1.parseRepositoryNwo)(util.getRequiredEnvParam("GITHUB_REPOSITORY"));
         const gitHubVersion = await (0, api_client_1.getGitHubVersion)();
         const features = new feature_flags_1.Features(gitHubVersion, repositoryNwo, actionsUtil.getTemporaryDirectory(), logger);
+        const memory = await util.getMemoryFlag(actionsUtil.getOptionalInput("ram") || process.env["CODEQL_RAM"], features);
         await runAutobuildIfLegacyGoWorkflow(config, logger);
         dbCreationTimings = await (0, analyze_1.runFinalize)(outputDir, threads, memory, config, logger, features);
         if (actionsUtil.getRequiredInput("skip-queries") !== "true") {

lib/feature-flags.js

@@ -42,6 +42,7 @@ var Feature;
     Feature["MlPoweredQueriesEnabled"] = "ml_powered_queries_enabled";
     Feature["NewAnalysisSummaryEnabled"] = "new_analysis_summary_enabled";
     Feature["QaTelemetryEnabled"] = "qa_telemetry_enabled";
+    Feature["ScalingReservedRam"] = "scaling_reserved_ram";
     Feature["UploadFailedSarifEnabled"] = "upload_failed_sarif_enabled";
 })(Feature = exports.Feature || (exports.Feature = {}));
 exports.featureConfig = {
@@ -75,6 +76,11 @@ exports.featureConfig = {
         minimumVersion: undefined,
         defaultValue: false,
     },
+    [Feature.ScalingReservedRam]: {
+        envVar: "CODEQL_ACTION_SCALING_RESERVED_RAM",
+        minimumVersion: undefined,
+        defaultValue: false,
+    },
     [Feature.UploadFailedSarifEnabled]: {
         envVar: "CODEQL_ACTION_UPLOAD_FAILED_SARIF",
         minimumVersion: "2.11.3",

lib/init-action.js

@@ -170,7 +170,7 @@ async function run() {
         // options at https://codeql.github.com/docs/codeql-cli/manual/database-trace-command/
         // for details.
         core.exportVariable("CODEQL_RAM", process.env["CODEQL_RAM"] ||
-            (0, util_1.getMemoryFlagValue)((0, actions_util_1.getOptionalInput)("ram")).toString());
+            (await (0, util_1.getMemoryFlagValue)((0, actions_util_1.getOptionalInput)("ram"), features)).toString());
         core.exportVariable("CODEQL_THREADS", (0, util_1.getThreadsFlagValue)((0, actions_util_1.getOptionalInput)("threads"), logger).toString());
         // Disable Kotlin extractor if feature flag set
         if (await features.getValue(feature_flags_1.Feature.DisableKotlinAnalysisEnabled)) {

lib/util.js

@@ -108,9 +108,18 @@ exports.withTmpDir = withTmpDir;
  * from committing too much of the available memory to CodeQL.
  * @returns number
  */
-function getSystemReservedMemoryMegaBytes() {
+async function getSystemReservedMemoryMegaBytes(totalMemoryMegaBytes, features) {
     // Windows needs more memory for OS processes.
-    return 1024 * (process.platform === "win32" ? 1.5 : 1);
+    const fixedAmount = 1024 * (process.platform === "win32" ? 1.5 : 1);
+    if (await features.getValue(feature_flags_1.Feature.ScalingReservedRam)) {
+        // Reserve an additional 2% of the total memory, since the amount used by
+        // the kernel for page tables scales with the size of physical memory.
+        const scaledAmount = 0.02 * totalMemoryMegaBytes;
+        return fixedAmount + scaledAmount;
+    }
+    else {
+        return fixedAmount;
+    }
 }
 /**
  * Get the value of the codeql `--ram` flag as configured by the `ram` input.
@@ -119,7 +128,7 @@ function getSystemReservedMemoryMegaBytes() {
  *
  * @returns {number} the amount of RAM to use, in megabytes
  */
-function getMemoryFlagValue(userInput) {
+async function getMemoryFlagValue(userInput, features) {
     let memoryToUseMegaBytes;
     if (userInput) {
         memoryToUseMegaBytes = Number(userInput);
@@ -130,7 +139,7 @@ function getMemoryFlagValue(userInput) {
     else {
         const totalMemoryBytes = os.totalmem();
         const totalMemoryMegaBytes = totalMemoryBytes / (1024 * 1024);
-        const reservedMemoryMegaBytes = getSystemReservedMemoryMegaBytes();
+        const reservedMemoryMegaBytes = await getSystemReservedMemoryMegaBytes(totalMemoryMegaBytes, features);
         memoryToUseMegaBytes = totalMemoryMegaBytes - reservedMemoryMegaBytes;
     }
     return Math.floor(memoryToUseMegaBytes);
@@ -143,8 +152,9 @@ exports.getMemoryFlagValue = getMemoryFlagValue;
  *
  * @returns string
  */
-function getMemoryFlag(userInput) {
-    return `--ram=${getMemoryFlagValue(userInput)}`;
+async function getMemoryFlag(userInput, features) {
+    const megabytes = await getMemoryFlagValue(userInput, features);
+    return `--ram=${megabytes}`;
 }
 exports.getMemoryFlag = getMemoryFlag;
 /**

lib/util.test.js

@@ -33,6 +33,7 @@ const github = __importStar(require("@actions/github"));
 const ava_1 = __importDefault(require("ava"));
 const sinon = __importStar(require("sinon"));
 const api = __importStar(require("./api-client"));
+const feature_flags_1 = require("./feature-flags");
 const logging_1 = require("./logging");
 const testing_utils_1 = require("./testing-utils");
 const util = __importStar(require("./util"));
@@ -42,22 +43,28 @@ const util = __importStar(require("./util"));
     const toolNames = util.getToolNames(JSON.parse(input));
     t.deepEqual(toolNames, ["CodeQL command-line toolchain", "ESLint"]);
 });
-(0, ava_1.default)("getMemoryFlag() should return the correct --ram flag", (t) => {
-    const totalMem = Math.floor(os.totalmem() / (1024 * 1024));
-    const expectedThreshold = process.platform === "win32" ? 1536 : 1024;
+(0, ava_1.default)("getMemoryFlag() should return the correct --ram flag", async (t) => {
+    const totalMem = os.totalmem() / (1024 * 1024);
+    const fixedAmount = process.platform === "win32" ? 1536 : 1024;
+    const scaledAmount = 0.02 * totalMem;
+    const expectedMemoryValue = Math.floor(totalMem - fixedAmount);
+    const expectedMemoryValueWithScaling = Math.floor(totalMem - fixedAmount - scaledAmount);
     const tests = [
-        [undefined, `--ram=${totalMem - expectedThreshold}`],
-        ["", `--ram=${totalMem - expectedThreshold}`],
-        ["512", "--ram=512"],
+        [undefined, false, `--ram=${expectedMemoryValue}`],
+        ["", false, `--ram=${expectedMemoryValue}`],
+        ["512", false, "--ram=512"],
+        [undefined, true, `--ram=${expectedMemoryValueWithScaling}`],
+        ["", true, `--ram=${expectedMemoryValueWithScaling}`],
     ];
-    for (const [input, expectedFlag] of tests) {
-        const flag = util.getMemoryFlag(input);
+    for (const [input, withScaling, expectedFlag] of tests) {
+        const features = (0, testing_utils_1.createFeatures)(withScaling ? [feature_flags_1.Feature.ScalingReservedRam] : []);
+        const flag = await util.getMemoryFlag(input, features);
         t.deepEqual(flag, expectedFlag);
     }
 });
-(0, ava_1.default)("getMemoryFlag() throws if the ram input is < 0 or NaN", (t) => {
+(0, ava_1.default)("getMemoryFlag() throws if the ram input is < 0 or NaN", async (t) => {
     for (const input of ["-1", "hello!"]) {
-        t.throws(() => util.getMemoryFlag(input));
+        await t.throwsAsync(async () => await util.getMemoryFlag(input, (0, testing_utils_1.createFeatures)([])));
     }
 });
 (0, ava_1.default)("getAddSnippetsFlag() should return the correct flag", (t) => {

src/analyze-action.ts

@@ -210,9 +210,6 @@ async function run() {
       actionsUtil.getOptionalInput("threads") || process.env["CODEQL_THREADS"],
       logger
     );
-    const memory = util.getMemoryFlag(
-      actionsUtil.getOptionalInput("ram") || process.env["CODEQL_RAM"]
-    );
 
     const repositoryNwo = parseRepositoryNwo(
       util.getRequiredEnvParam("GITHUB_REPOSITORY")
@@ -227,6 +224,11 @@ async function run() {
       logger
     );
 
+    const memory = await util.getMemoryFlag(
+      actionsUtil.getOptionalInput("ram") || process.env["CODEQL_RAM"],
+      features
+    );
+
     await runAutobuildIfLegacyGoWorkflow(config, logger);
 
     dbCreationTimings = await runFinalize(

src/feature-flags.ts

@@ -39,6 +39,7 @@ export enum Feature {
   MlPoweredQueriesEnabled = "ml_powered_queries_enabled",
   NewAnalysisSummaryEnabled = "new_analysis_summary_enabled",
   QaTelemetryEnabled = "qa_telemetry_enabled",
+  ScalingReservedRam = "scaling_reserved_ram",
   UploadFailedSarifEnabled = "upload_failed_sarif_enabled",
 }
 
@@ -76,6 +77,11 @@ export const featureConfig: Record<
     minimumVersion: undefined,
     defaultValue: false,
   },
+  [Feature.ScalingReservedRam]: {
+    envVar: "CODEQL_ACTION_SCALING_RESERVED_RAM",
+    minimumVersion: undefined,
+    defaultValue: false,
+  },
   [Feature.UploadFailedSarifEnabled]: {
     envVar: "CODEQL_ACTION_UPLOAD_FAILED_SARIF",
     minimumVersion: "2.11.3",

src/init-action.ts

@@ -329,7 +329,7 @@ async function run() {
     core.exportVariable(
       "CODEQL_RAM",
       process.env["CODEQL_RAM"] ||
-        getMemoryFlagValue(getOptionalInput("ram")).toString()
+        (await getMemoryFlagValue(getOptionalInput("ram"), features)).toString()
     );
     core.exportVariable(
       "CODEQL_THREADS",

src/util.test.ts

@@ -8,8 +8,14 @@ import * as sinon from "sinon";
 
 import * as api from "./api-client";
 import { Config } from "./config-utils";
+import { Feature } from "./feature-flags";
 import { getRunnerLogger } from "./logging";
-import { getRecordingLogger, LoggedMessage, setupTests } from "./testing-utils";
+import {
+  createFeatures,
+  getRecordingLogger,
+  LoggedMessage,
+  setupTests,
+} from "./testing-utils";
 import * as util from "./util";
 
 setupTests(test);
@@ -23,25 +29,37 @@ test("getToolNames", (t) => {
   t.deepEqual(toolNames, ["CodeQL command-line toolchain", "ESLint"]);
 });
 
-test("getMemoryFlag() should return the correct --ram flag", (t) => {
-  const totalMem = Math.floor(os.totalmem() / (1024 * 1024));
-  const expectedThreshold = process.platform === "win32" ? 1536 : 1024;
+test("getMemoryFlag() should return the correct --ram flag", async (t) => {
+  const totalMem = os.totalmem() / (1024 * 1024);
+  const fixedAmount = process.platform === "win32" ? 1536 : 1024;
+  const scaledAmount = 0.02 * totalMem;
+  const expectedMemoryValue = Math.floor(totalMem - fixedAmount);
+  const expectedMemoryValueWithScaling = Math.floor(
+    totalMem - fixedAmount - scaledAmount
+  );
 
-  const tests: Array<[string | undefined, string]> = [
-    [undefined, `--ram=${totalMem - expectedThreshold}`],
-    ["", `--ram=${totalMem - expectedThreshold}`],
-    ["512", "--ram=512"],
+  const tests: Array<[string | undefined, boolean, string]> = [
+    [undefined, false, `--ram=${expectedMemoryValue}`],
+    ["", false, `--ram=${expectedMemoryValue}`],
+    ["512", false, "--ram=512"],
+    [undefined, true, `--ram=${expectedMemoryValueWithScaling}`],
+    ["", true, `--ram=${expectedMemoryValueWithScaling}`],
   ];
 
-  for (const [input, expectedFlag] of tests) {
-    const flag = util.getMemoryFlag(input);
+  for (const [input, withScaling, expectedFlag] of tests) {
+    const features = createFeatures(
+      withScaling ? [Feature.ScalingReservedRam] : []
+    );
+    const flag = await util.getMemoryFlag(input, features);
     t.deepEqual(flag, expectedFlag);
   }
 });
 
-test("getMemoryFlag() throws if the ram input is < 0 or NaN", (t) => {
+test("getMemoryFlag() throws if the ram input is < 0 or NaN", async (t) => {
   for (const input of ["-1", "hello!"]) {
-    t.throws(() => util.getMemoryFlag(input));
+    await t.throwsAsync(
+      async () => await util.getMemoryFlag(input, createFeatures([]))
+    );
   }
 });
 

src/util.ts

@@ -157,9 +157,21 @@ export async function withTmpDir<T>(
  * from committing too much of the available memory to CodeQL.
  * @returns number
  */
-function getSystemReservedMemoryMegaBytes(): number {
+async function getSystemReservedMemoryMegaBytes(
+  totalMemoryMegaBytes: number,
+  features: FeatureEnablement
+): Promise<number> {
   // Windows needs more memory for OS processes.
-  return 1024 * (process.platform === "win32" ? 1.5 : 1);
+  const fixedAmount = 1024 * (process.platform === "win32" ? 1.5 : 1);
+
+  if (await features.getValue(Feature.ScalingReservedRam)) {
+    // Reserve an additional 2% of the total memory, since the amount used by
+    // the kernel for page tables scales with the size of physical memory.
+    const scaledAmount = 0.02 * totalMemoryMegaBytes;
+    return fixedAmount + scaledAmount;
+  } else {
+    return fixedAmount;
+  }
 }
 
 /**
@@ -169,7 +181,10 @@ function getSystemReservedMemoryMegaBytes(): number {
  *
  * @returns {number} the amount of RAM to use, in megabytes
  */
-export function getMemoryFlagValue(userInput: string | undefined): number {
+export async function getMemoryFlagValue(
+  userInput: string | undefined,
+  features: FeatureEnablement
+): Promise<number> {
   let memoryToUseMegaBytes: number;
   if (userInput) {
     memoryToUseMegaBytes = Number(userInput);
@@ -179,7 +194,10 @@ export function getMemoryFlagValue(userInput: string | undefined): number {
   } else {
     const totalMemoryBytes = os.totalmem();
     const totalMemoryMegaBytes = totalMemoryBytes / (1024 * 1024);
-    const reservedMemoryMegaBytes = getSystemReservedMemoryMegaBytes();
+    const reservedMemoryMegaBytes = await getSystemReservedMemoryMegaBytes(
+      totalMemoryMegaBytes,
+      features
+    );
     memoryToUseMegaBytes = totalMemoryMegaBytes - reservedMemoryMegaBytes;
   }
   return Math.floor(memoryToUseMegaBytes);
@@ -192,8 +210,12 @@ export function getMemoryFlagValue(userInput: string | undefined): number {
  *
  * @returns string
  */
-export function getMemoryFlag(userInput: string | undefined): string {
-  return `--ram=${getMemoryFlagValue(userInput)}`;
+export async function getMemoryFlag(
+  userInput: string | undefined,
+  features: FeatureEnablement
+): Promise<string> {
+  const megabytes = await getMemoryFlagValue(userInput, features);
+  return `--ram=${megabytes}`;
 }
 
 /**