WIP: Improve total memory detection

lib/util.js

@@ -22,7 +22,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.tryGetFolderBytes = exports.isGoExtractionReconciliationEnabled = exports.listFolder = exports.doesDirectoryExist = exports.useCodeScanningConfigInCli = exports.isInTestMode = exports.checkActionVersion = exports.getMlPoweredJsQueriesStatus = exports.getMlPoweredJsQueriesPack = exports.ML_POWERED_JS_QUERIES_PACK_NAME = exports.isGoodVersion = exports.delay = exports.bundleDb = exports.codeQlVersionAbove = exports.getCachedCodeQlVersion = exports.cacheCodeQlVersion = exports.isGitHubGhesVersionBelow = exports.isHTTPError = exports.UserError = exports.HTTPError = exports.getRequiredEnvParam = exports.isActions = exports.getMode = exports.enrichEnvironment = exports.initializeEnvironment = exports.EnvVar = exports.Mode = exports.assertNever = exports.getGitHubAuth = exports.apiVersionInRange = exports.DisallowedAPIVersionReason = exports.checkGitHubVersionInRange = exports.getGitHubVersion = exports.GitHubVariant = exports.parseGitHubUrl = exports.getCodeQLDatabasePath = exports.getThreadsFlag = exports.getThreadsFlagValue = exports.getAddSnippetsFlag = exports.getMemoryFlag = exports.getMemoryFlagValue = exports.withTmpDir = exports.getToolNames = exports.getExtraOptionsEnvParam = exports.DID_AUTOBUILD_GO_ENV_VAR_NAME = exports.DEFAULT_DEBUG_DATABASE_NAME = exports.DEFAULT_DEBUG_ARTIFACT_NAME = exports.GITHUB_DOTCOM_URL = void 0;
+exports.tryGetFolderBytes = exports.isGoExtractionReconciliationEnabled = exports.listFolder = exports.doesDirectoryExist = exports.useCodeScanningConfigInCli = exports.isInTestMode = exports.checkActionVersion = exports.getMlPoweredJsQueriesStatus = exports.getMlPoweredJsQueriesPack = exports.ML_POWERED_JS_QUERIES_PACK_NAME = exports.isGoodVersion = exports.delay = exports.bundleDb = exports.codeQlVersionAbove = exports.getCachedCodeQlVersion = exports.cacheCodeQlVersion = exports.isGitHubGhesVersionBelow = exports.isHTTPError = exports.UserError = exports.HTTPError = exports.getRequiredEnvParam = exports.isActions = exports.getMode = exports.enrichEnvironment = exports.initializeEnvironment = exports.EnvVar = exports.Mode = exports.assertNever = exports.getGitHubAuth = exports.apiVersionInRange = exports.DisallowedAPIVersionReason = exports.checkGitHubVersionInRange = exports.getGitHubVersion = exports.GitHubVariant = exports.parseGitHubUrl = exports.getCodeQLDatabasePath = exports.getThreadsFlag = exports.getThreadsFlagValue = exports.getAddSnippetsFlag = exports.getMemoryFlag = exports.getTotalMemoryBytes = exports.getMemoryFlagValue = exports.withTmpDir = exports.getToolNames = exports.getExtraOptionsEnvParam = exports.DID_AUTOBUILD_GO_ENV_VAR_NAME = exports.DEFAULT_DEBUG_DATABASE_NAME = exports.DEFAULT_DEBUG_ARTIFACT_NAME = exports.GITHUB_DOTCOM_URL = void 0;
 const fs = __importStar(require("fs"));
 const os = __importStar(require("os"));
 const path = __importStar(require("path"));
@@ -130,7 +130,7 @@ function getMemoryFlagValue(userInput) {
         }
     }
     else {
-        const totalMemoryBytes = os.totalmem();
+        const totalMemoryBytes = getTotalMemoryBytes();
         const totalMemoryMegaBytes = totalMemoryBytes / (1024 * 1024);
         const reservedMemoryMegaBytes = getSystemReservedMemoryMegaBytes();
         memoryToUseMegaBytes = totalMemoryMegaBytes - reservedMemoryMegaBytes;
@@ -138,6 +138,41 @@ function getMemoryFlagValue(userInput) {
     return Math.floor(memoryToUseMegaBytes);
 }
 exports.getMemoryFlagValue = getMemoryFlagValue;
+function getTotalMemoryBytes() {
+    const nodeReportedMemory = os.totalmem();
+    console.log(`Node reported ${nodeReportedMemory} bytes of memory.`);
+    if (process.platform === "win32") {
+        console.log("On Windows, so just returning the memory Node reported.");
+        return nodeReportedMemory;
+    }
+    let lowestMemorySeen = nodeReportedMemory;
+    try {
+        const dockerMemoryLimit = parseInt(fs.readFileSync("/sys/fs/cgroup/memory/memory.limit_in_bytes", "utf8"));
+        console.log(`Docker set a limit of ${dockerMemoryLimit} bytes of memory.`);
+        lowestMemorySeen = Math.min(lowestMemorySeen, dockerMemoryLimit);
+    }
+    catch (err) {
+        console.error(err);
+    }
+    try {
+        const memoryInfo = fs.readFileSync("/proc/meminfo", "utf8").split("\n");
+        const relevantLine = /^\s*MemTotal:\s*(\d+)\s*kB\s*$/;
+        for (const line of memoryInfo) {
+            const match = relevantLine.exec(line);
+            if (match) {
+                const memoryFromMemoryInfo = parseInt(match[1]) * 1024;
+                console.log(`Found total memory of ${memoryFromMemoryInfo} in memory info.`);
+                lowestMemorySeen = Math.min(lowestMemorySeen, memoryFromMemoryInfo);
+                break;
+            }
+        }
+    }
+    catch (err) {
+        console.error(err);
+    }
+    return lowestMemorySeen;
+}
+exports.getTotalMemoryBytes = getTotalMemoryBytes;
 /**
  * Get the codeql `--ram` flag as configured by the `ram` input. If no value was
  * specified, the total available memory will be used minus a threshold

src/util.ts

@@ -162,7 +162,7 @@ export function getMemoryFlagValue(userInput: string | undefined): number {
       throw new Error(`Invalid RAM setting "${userInput}", specified.`);
     }
   } else {
-    const totalMemoryBytes = os.totalmem();
+    const totalMemoryBytes = getTotalMemoryBytes();
     const totalMemoryMegaBytes = totalMemoryBytes / (1024 * 1024);
     const reservedMemoryMegaBytes = getSystemReservedMemoryMegaBytes();
     memoryToUseMegaBytes = totalMemoryMegaBytes - reservedMemoryMegaBytes;
@@ -170,6 +170,43 @@ export function getMemoryFlagValue(userInput: string | undefined): number {
   return Math.floor(memoryToUseMegaBytes);
 }
 
+export function getTotalMemoryBytes(): number {
+  const nodeReportedMemory = os.totalmem();
+  console.log(`Node reported ${nodeReportedMemory} bytes of memory.`);
+  if (process.platform === "win32") {
+    console.log("On Windows, so just returning the memory Node reported.");
+    return nodeReportedMemory;
+  }
+  let lowestMemorySeen = nodeReportedMemory;
+  try {
+    const dockerMemoryLimit = parseInt(
+      fs.readFileSync("/sys/fs/cgroup/memory/memory.limit_in_bytes", "utf8")
+    );
+    console.log(`Docker set a limit of ${dockerMemoryLimit} bytes of memory.`);
+    lowestMemorySeen = Math.min(lowestMemorySeen, dockerMemoryLimit);
+  } catch (err) {
+    console.error(err);
+  }
+  try {
+    const memoryInfo = fs.readFileSync("/proc/meminfo", "utf8").split("\n");
+    const relevantLine = /^\s*MemTotal:\s*(\d+)\s*kB\s*$/;
+    for (const line of memoryInfo) {
+      const match = relevantLine.exec(line);
+      if (match) {
+        const memoryFromMemoryInfo = parseInt(match[1]) * 1024;
+        console.log(
+          `Found total memory of ${memoryFromMemoryInfo} in memory info.`
+        );
+        lowestMemorySeen = Math.min(lowestMemorySeen, memoryFromMemoryInfo);
+        break;
+      }
+    }
+  } catch (err) {
+    console.error(err);
+  }
+  return lowestMemorySeen;
+}
+
 /**
  * Get the codeql `--ram` flag as configured by the `ram` input. If no value was
  * specified, the total available memory will be used minus a threshold