Avoid counting lines of code in parallel with running queries

This can lead to OOME in larger repositories. Unfortunately, running
the LoC serially will cause the action to take longer, but I don't think
there is any other way around this.
This commit is contained in:
Andrew Eisenberg
2021-05-11 09:50:29 -07:00
parent 4c0671c518
commit a19f1a7b3c
3 changed files with 43 additions and 32 deletions

28
lib/analyze.js generated
View File

@@ -78,23 +78,18 @@ async function finalizeDatabaseCreation(config, threadsFlag, logger) {
// Runs queries and creates sarif files in the given folder
async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag, automationDetailsId, config, logger) {
const statusReport = {};
// count the number of lines in the background
const locPromise = count_loc_1.countLoc(path.resolve(),
// config.paths specifies external directories. the current
// directory is included in the analysis by default. Replicate
// that here.
config.paths, config.pathsIgnore, config.languages, logger);
for (const language of config.languages) {
logger.startGroup(`Analyzing ${language}`);
const queries = config.queries[language];
if (queries.builtin.length === 0 && queries.custom.length === 0) {
throw new Error(`Unable to analyse ${language} as no queries were selected for this language`);
}
const allSarifFiles = [];
try {
if (queries["builtin"].length > 0) {
const startTimeBuliltIn = new Date().getTime();
const sarifFile = await runQueryGroup(language, "builtin", queries["builtin"], sarifFolder, undefined);
await injectLinesOfCode(sarifFile, language, locPromise);
allSarifFiles.push(sarifFile);
statusReport[`analyze_builtin_queries_${language}_duration_ms`] =
new Date().getTime() - startTimeBuliltIn;
}
@@ -105,12 +100,22 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
if (queries["custom"][i].queries.length > 0) {
const sarifFile = await runQueryGroup(language, `custom-${i}`, queries["custom"][i].queries, temporarySarifDir, queries["custom"][i].searchPath);
temporarySarifFiles.push(sarifFile);
allSarifFiles.push(sarifFile);
}
}
if (allSarifFiles.length > 0) {
const linesOfCode = await count_loc_1.countLoc(path.resolve(),
// config.paths specifies external directories. the current
// directory is included in the analysis by default. Replicate
// that here.
config.paths, config.pathsIgnore, config.languages, logger);
for (const sarifFile of allSarifFiles) {
injectLinesOfCode(sarifFile, language, linesOfCode);
}
}
if (temporarySarifFiles.length > 0) {
const sarifFile = path.join(sarifFolder, `${language}-custom.sarif`);
fs.writeFileSync(sarifFile, upload_lib_1.combineSarifFiles(temporarySarifFiles));
await injectLinesOfCode(sarifFile, language, locPromise);
statusReport[`analyze_custom_queries_${language}_duration_ms`] =
new Date().getTime() - startTimeCustom;
}
@@ -152,9 +157,8 @@ async function runAnalyze(outputDir, memoryFlag, addSnippetsFlag, threadsFlag, a
return { ...queriesStats };
}
exports.runAnalyze = runAnalyze;
async function injectLinesOfCode(sarifFile, language, locPromise) {
const lineCounts = await locPromise;
if (language in lineCounts) {
function injectLinesOfCode(sarifFile, language, linesOfCode) {
if (language in linesOfCode) {
const sarif = JSON.parse(fs.readFileSync(sarifFile, "utf8"));
if (Array.isArray(sarif.runs)) {
for (const run of sarif.runs) {
@@ -166,7 +170,7 @@ async function injectLinesOfCode(sarifFile, language, locPromise) {
(r) => { var _a; return r.ruleId === ruleId || ((_a = r.rule) === null || _a === void 0 ? void 0 : _a.id) === ruleId; });
// only add the baseline value if the rule already exists
if (rule) {
rule.baseline = lineCounts[language];
rule.baseline = linesOfCode[language];
}
}
}

File diff suppressed because one or more lines are too long