Add messy debug statements

This can lead to OOME in larger repositories. Unfortunately, running
the LoC serially will cause the action to take longer, but I don't think
there is any other way around this.

lib/analyze.js

@@ -78,23 +78,18 @@ async function finalizeDatabaseCreation(config, threadsFlag, logger) {
 // Runs queries and creates sarif files in the given folder
 async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag, automationDetailsId, config, logger) {
     const statusReport = {};
-    // count the number of lines in the background
-    const locPromise = count_loc_1.countLoc(path.resolve(), 
-    // config.paths specifies external directories. the current
-    // directory is included in the analysis by default. Replicate
-    // that here.
-    config.paths, config.pathsIgnore, config.languages, logger);
     for (const language of config.languages) {
         logger.startGroup(`Analyzing ${language}`);
         const queries = config.queries[language];
         if (queries.builtin.length === 0 && queries.custom.length === 0) {
             throw new Error(`Unable to analyse ${language} as no queries were selected for this language`);
         }
+        const allSarifFiles = [];
         try {
             if (queries["builtin"].length > 0) {
                 const startTimeBuliltIn = new Date().getTime();
                 const sarifFile = await runQueryGroup(language, "builtin", queries["builtin"], sarifFolder, undefined);
-                await injectLinesOfCode(sarifFile, language, locPromise);
+                allSarifFiles.push(sarifFile);
                 statusReport[`analyze_builtin_queries_${language}_duration_ms`] =
                     new Date().getTime() - startTimeBuliltIn;
             }
@@ -105,12 +100,28 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
                 if (queries["custom"][i].queries.length > 0) {
                     const sarifFile = await runQueryGroup(language, `custom-${i}`, queries["custom"][i].queries, temporarySarifDir, queries["custom"][i].searchPath);
                     temporarySarifFiles.push(sarifFile);
+                    allSarifFiles.push(sarifFile);
+                }
+            }
+            logger.info("111111111");
+            logger.info("About to start LoC");
+            logger.info("111111111");
+            if (allSarifFiles.length > 0) {
+                const linesOfCode = await count_loc_1.countLoc(path.resolve(), 
+                // config.paths specifies external directories. the current
+                // directory is included in the analysis by default. Replicate
+                // that here.
+                config.paths, config.pathsIgnore, config.languages, logger);
+                logger.info("22222222");
+                logger.info("Finished LoC");
+                logger.info("22222222");
+                for (const sarifFile of allSarifFiles) {
+                    injectLinesOfCode(sarifFile, language, linesOfCode);
                 }
             }
             if (temporarySarifFiles.length > 0) {
                 const sarifFile = path.join(sarifFolder, `${language}-custom.sarif`);
                 fs.writeFileSync(sarifFile, upload_lib_1.combineSarifFiles(temporarySarifFiles));
-                await injectLinesOfCode(sarifFile, language, locPromise);
                 statusReport[`analyze_custom_queries_${language}_duration_ms`] =
                     new Date().getTime() - startTimeCustom;
             }
@@ -133,8 +144,17 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
         fs.writeFileSync(querySuitePath, querySuiteContents);
         logger.debug(`Query suite file for ${language}...\n${querySuiteContents}`);
         const sarifFile = path.join(destinationFolder, `${language}-${type}.sarif`);
-        const codeql = codeql_1.getCodeQL(config.codeQLCmd);
-        await codeql.databaseAnalyze(databasePath, sarifFile, searchPath, querySuitePath, memoryFlag, addSnippetsFlag, threadsFlag, automationDetailsId);
+        // const codeql = getCodeQL(config.codeQLCmd);
+        // await codeql.databaseAnalyze(
+        //   databasePath,
+        //   sarifFile,
+        //   searchPath,
+        //   querySuitePath,
+        //   memoryFlag,
+        //   addSnippetsFlag,
+        //   threadsFlag,
+        //   automationDetailsId
+        // );
         logger.debug(`SARIF results for database ${language} created at "${sarifFile}"`);
         logger.endGroup();
         return sarifFile;
@@ -152,9 +172,8 @@ async function runAnalyze(outputDir, memoryFlag, addSnippetsFlag, threadsFlag, a
     return { ...queriesStats };
 }
 exports.runAnalyze = runAnalyze;
-async function injectLinesOfCode(sarifFile, language, locPromise) {
-    const lineCounts = await locPromise;
-    if (language in lineCounts) {
+function injectLinesOfCode(sarifFile, language, linesOfCode) {
+    if (language in linesOfCode) {
         const sarif = JSON.parse(fs.readFileSync(sarifFile, "utf8"));
         if (Array.isArray(sarif.runs)) {
             for (const run of sarif.runs) {
@@ -166,7 +185,7 @@ async function injectLinesOfCode(sarifFile, language, locPromise) {
                 (r) => { var _a; return r.ruleId === ruleId || ((_a = r.rule) === null || _a === void 0 ? void 0 : _a.id) === ruleId; });
                 // only add the baseline value if the rule already exists
                 if (rule) {
-                    rule.baseline = lineCounts[language];
+                    rule.baseline = linesOfCode[language];
                 }
             }
         }

node_modules/github-linguist/dist/directory.js

@@ -154,10 +154,13 @@ class LocDir {
             ignore: this.exclude,
             nodir: true
         });
+        console.log(`Excluding: ${this.exclude}`);
+
         const files = [];
         const info = { ...defaultInfo };
         let languages = {};
         await Promise.all(paths.map(async (pathItem) => {
+            console.log(`Processing ${pathItem}`);
             const fullPath = slash2_1.default(path_1.default.join(this.cwd, pathItem));
             if (!pathItem ||
                 this.ignoreLanguage(pathItem) ||
@@ -205,4 +208,4 @@ function ensureArray(arr, dfault) {
         ? arr
         : [arr];
 }
-//# sourceMappingURL=directory.js.map
+//# sourceMappingURL=directory.js.map

src/analyze.ts

@@ -146,18 +146,6 @@ export async function runQueries(
 ): Promise<QueriesStatusReport> {
   const statusReport: QueriesStatusReport = {};
 
-  // count the number of lines in the background
-  const locPromise = countLoc(
-    path.resolve(),
-    // config.paths specifies external directories. the current
-    // directory is included in the analysis by default. Replicate
-    // that here.
-    config.paths,
-    config.pathsIgnore,
-    config.languages,
-    logger
-  );
-
   for (const language of config.languages) {
     logger.startGroup(`Analyzing ${language}`);
 
@@ -168,6 +156,7 @@ export async function runQueries(
       );
     }
 
+    const allSarifFiles: string[] = [];
     try {
       if (queries["builtin"].length > 0) {
         const startTimeBuliltIn = new Date().getTime();
@@ -178,7 +167,8 @@ export async function runQueries(
           sarifFolder,
           undefined
         );
-        await injectLinesOfCode(sarifFile, language, locPromise);
+
+        allSarifFiles.push(sarifFile);
 
         statusReport[`analyze_builtin_queries_${language}_duration_ms`] =
           new Date().getTime() - startTimeBuliltIn;
@@ -196,12 +186,38 @@ export async function runQueries(
             queries["custom"][i].searchPath
           );
           temporarySarifFiles.push(sarifFile);
+          allSarifFiles.push(sarifFile);
         }
       }
+
+      logger.info("111111111");
+      logger.info("About to start LoC");
+      logger.info("111111111");
+
+      if (allSarifFiles.length > 0) {
+        const linesOfCode = await countLoc(
+          path.resolve(),
+          // config.paths specifies external directories. the current
+          // directory is included in the analysis by default. Replicate
+          // that here.
+          config.paths,
+          config.pathsIgnore,
+          config.languages,
+          logger
+        );
+
+        logger.info("22222222");
+        logger.info("Finished LoC");
+        logger.info("22222222");
+
+        for (const sarifFile of allSarifFiles) {
+          injectLinesOfCode(sarifFile, language, linesOfCode);
+        }
+      }
+
       if (temporarySarifFiles.length > 0) {
         const sarifFile = path.join(sarifFolder, `${language}-custom.sarif`);
         fs.writeFileSync(sarifFile, combineSarifFiles(temporarySarifFiles));
-        await injectLinesOfCode(sarifFile, language, locPromise);
 
         statusReport[`analyze_custom_queries_${language}_duration_ms`] =
           new Date().getTime() - startTimeCustom;
@@ -237,17 +253,17 @@ export async function runQueries(
 
     const sarifFile = path.join(destinationFolder, `${language}-${type}.sarif`);
 
-    const codeql = getCodeQL(config.codeQLCmd);
-    await codeql.databaseAnalyze(
-      databasePath,
-      sarifFile,
-      searchPath,
-      querySuitePath,
-      memoryFlag,
-      addSnippetsFlag,
-      threadsFlag,
-      automationDetailsId
-    );
+    // const codeql = getCodeQL(config.codeQLCmd);
+    // await codeql.databaseAnalyze(
+    //   databasePath,
+    //   sarifFile,
+    //   searchPath,
+    //   querySuitePath,
+    //   memoryFlag,
+    //   addSnippetsFlag,
+    //   threadsFlag,
+    //   automationDetailsId
+    // );
 
     logger.debug(
       `SARIF results for database ${language} created at "${sarifFile}"`
@@ -289,13 +305,12 @@ export async function runAnalyze(
   return { ...queriesStats };
 }
 
-async function injectLinesOfCode(
+function injectLinesOfCode(
   sarifFile: string,
   language: string,
-  locPromise: Promise<Partial<Record<IdPrefixes, number>>>
+  linesOfCode: Partial<Record<IdPrefixes, number>>
 ) {
-  const lineCounts = await locPromise;
-  if (language in lineCounts) {
+  if (language in linesOfCode) {
     const sarif = JSON.parse(fs.readFileSync(sarifFile, "utf8"));
     if (Array.isArray(sarif.runs)) {
       for (const run of sarif.runs) {
@@ -308,7 +323,7 @@ async function injectLinesOfCode(
         );
         // only add the baseline value if the rule already exists
         if (rule) {
-          rule.baseline = lineCounts[language];
+          rule.baseline = linesOfCode[language];
         }
       }
     }