From a19f1a7b3c0becba01120877c8889168ec73a701 Mon Sep 17 00:00:00 2001 From: Andrew Eisenberg Date: Tue, 11 May 2021 09:50:29 -0700 Subject: [PATCH] Avoid counting lines of code in parallel with running queries This can lead to OOME in larger repositories. Unfortunately, running the LoC serially will cause the action to take longer, but I don't think there is any other way around this. --- lib/analyze.js | 28 ++++++++++++++++------------ lib/analyze.js.map | 2 +- src/analyze.ts | 45 ++++++++++++++++++++++++++------------------- 3 files changed, 43 insertions(+), 32 deletions(-) diff --git a/lib/analyze.js b/lib/analyze.js index 580b3a6a0..88038ede1 100644 --- a/lib/analyze.js +++ b/lib/analyze.js @@ -78,23 +78,18 @@ async function finalizeDatabaseCreation(config, threadsFlag, logger) { // Runs queries and creates sarif files in the given folder async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag, automationDetailsId, config, logger) { const statusReport = {}; - // count the number of lines in the background - const locPromise = count_loc_1.countLoc(path.resolve(), - // config.paths specifies external directories. the current - // directory is included in the analysis by default. Replicate - // that here. - config.paths, config.pathsIgnore, config.languages, logger); for (const language of config.languages) { logger.startGroup(`Analyzing ${language}`); const queries = config.queries[language]; if (queries.builtin.length === 0 && queries.custom.length === 0) { throw new Error(`Unable to analyse ${language} as no queries were selected for this language`); } + const allSarifFiles = []; try { if (queries["builtin"].length > 0) { const startTimeBuliltIn = new Date().getTime(); const sarifFile = await runQueryGroup(language, "builtin", queries["builtin"], sarifFolder, undefined); - await injectLinesOfCode(sarifFile, language, locPromise); + allSarifFiles.push(sarifFile); statusReport[`analyze_builtin_queries_${language}_duration_ms`] = new Date().getTime() - startTimeBuliltIn; } @@ -105,12 +100,22 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag, if (queries["custom"][i].queries.length > 0) { const sarifFile = await runQueryGroup(language, `custom-${i}`, queries["custom"][i].queries, temporarySarifDir, queries["custom"][i].searchPath); temporarySarifFiles.push(sarifFile); + allSarifFiles.push(sarifFile); + } + } + if (allSarifFiles.length > 0) { + const linesOfCode = await count_loc_1.countLoc(path.resolve(), + // config.paths specifies external directories. the current + // directory is included in the analysis by default. Replicate + // that here. + config.paths, config.pathsIgnore, config.languages, logger); + for (const sarifFile of allSarifFiles) { + injectLinesOfCode(sarifFile, language, linesOfCode); } } if (temporarySarifFiles.length > 0) { const sarifFile = path.join(sarifFolder, `${language}-custom.sarif`); fs.writeFileSync(sarifFile, upload_lib_1.combineSarifFiles(temporarySarifFiles)); - await injectLinesOfCode(sarifFile, language, locPromise); statusReport[`analyze_custom_queries_${language}_duration_ms`] = new Date().getTime() - startTimeCustom; } @@ -152,9 +157,8 @@ async function runAnalyze(outputDir, memoryFlag, addSnippetsFlag, threadsFlag, a return { ...queriesStats }; } exports.runAnalyze = runAnalyze; -async function injectLinesOfCode(sarifFile, language, locPromise) { - const lineCounts = await locPromise; - if (language in lineCounts) { +function injectLinesOfCode(sarifFile, language, linesOfCode) { + if (language in linesOfCode) { const sarif = JSON.parse(fs.readFileSync(sarifFile, "utf8")); if (Array.isArray(sarif.runs)) { for (const run of sarif.runs) { @@ -166,7 +170,7 @@ async function injectLinesOfCode(sarifFile, language, locPromise) { (r) => { var _a; return r.ruleId === ruleId || ((_a = r.rule) === null || _a === void 0 ? void 0 : _a.id) === ruleId; }); // only add the baseline value if the rule already exists if (rule) { - rule.baseline = lineCounts[language]; + rule.baseline = linesOfCode[language]; } } } diff --git a/lib/analyze.js.map b/lib/analyze.js.map index b6a3da534..b55746319 100644 --- a/lib/analyze.js.map +++ b/lib/analyze.js.map @@ -1 +1 @@ -{"version":3,"file":"analyze.js","sourceRoot":"","sources":["../src/analyze.ts"],"names":[],"mappings":";;;;;;;;;AAAA,uCAAyB;AACzB,2CAA6B;AAE7B,yEAA2D;AAE3D,gEAAkD;AAClD,qCAAqC;AAErC,2CAAmD;AACnD,2CAA0D;AAE1D,gEAAkD;AAClD,6CAAiD;AACjD,6CAA+B;AAE/B,MAAa,mBAAoB,SAAQ,KAAK;IAG5C,YAAY,mBAAwC,EAAE,OAAe;QACnE,KAAK,CAAC,OAAO,CAAC,CAAC;QAEf,IAAI,CAAC,IAAI,GAAG,qBAAqB,CAAC;QAClC,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAC;IACjD,CAAC;CACF;AATD,kDASC;AA+BD,KAAK,UAAU,oBAAoB,CAAC,MAAc;IAChD,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;IAClD,IAAI,YAAY,KAAK,SAAS,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE;QAC3D,+FAA+F;QAC/F,OAAO;KACR;IAED,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,MAAM,OAAO,GAAG;QACd,SAAS,EAAE;YACT,MAAM,EAAE,CAAC,IAAY,EAAE,EAAE;gBACvB,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC5B,CAAC;SACF;KACF,CAAC;IAEF,MAAM,IAAI,UAAU,CAAC,UAAU,CAC7B,YAAY,EACZ;QACE,IAAI;QACJ,8EAA8E;KAC/E,EACD,OAAO,CACR,CAAC,IAAI,EAAE,CAAC;IACT,MAAM,CAAC,IAAI,CAAC,kCAAkC,MAAM,EAAE,CAAC,CAAC;IACxD,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,GAAG,MAAM,CAAC;IAE/C,MAAM,GAAG,EAAE,CAAC;IACZ,MAAM,IAAI,UAAU,CAAC,UAAU,CAC7B,YAAY,EACZ,CAAC,IAAI,EAAE,wCAAwC,CAAC,EAChD,OAAO,CACR,CAAC,IAAI,EAAE,CAAC;IACT,MAAM,CAAC,IAAI,CAAC,qCAAqC,MAAM,EAAE,CAAC,CAAC;IAC3D,OAAO,CAAC,GAAG,CAAC,2BAA2B,CAAC,GAAG,MAAM,CAAC;AACpD,CAAC;AAED,KAAK,UAAU,4BAA4B,CACzC,MAA0B,EAC1B,MAAc;IAEd,sEAAsE;IACtE,oCAAoC;IACpC,aAAa,CAAC,8BAA8B,CAAC,MAAM,CAAC,CAAC;IAErD,MAAM,MAAM,GAAG,kBAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAC3C,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACvC,IAAI,6BAAiB,CAAC,QAAQ,CAAC,EAAE;YAC/B,MAAM,CAAC,UAAU,CAAC,cAAc,QAAQ,EAAE,CAAC,CAAC;YAE5C,IAAI,QAAQ,KAAK,oBAAQ,CAAC,MAAM,EAAE;gBAChC,MAAM,oBAAoB,CAAC,MAAM,CAAC,CAAC;aACpC;YAED,MAAM,MAAM,CAAC,sBAAsB,CACjC,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,EACpD,QAAQ,CACT,CAAC;YACF,MAAM,CAAC,QAAQ,EAAE,CAAC;SACnB;KACF;AACH,CAAC;AAED,KAAK,UAAU,wBAAwB,CACrC,MAA0B,EAC1B,WAAmB,EACnB,MAAc;IAEd,MAAM,4BAA4B,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEnD,MAAM,MAAM,GAAG,kBAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAC3C,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACvC,MAAM,CAAC,UAAU,CAAC,cAAc,QAAQ,EAAE,CAAC,CAAC;QAC5C,MAAM,MAAM,CAAC,gBAAgB,CAC3B,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,EACpD,WAAW,CACZ,CAAC;QACF,MAAM,CAAC,QAAQ,EAAE,CAAC;KACnB;AACH,CAAC;AAED,2DAA2D;AACpD,KAAK,UAAU,UAAU,CAC9B,WAAmB,EACnB,UAAkB,EAClB,eAAuB,EACvB,WAAmB,EACnB,mBAAuC,EACvC,MAA0B,EAC1B,MAAc;IAEd,MAAM,YAAY,GAAwB,EAAE,CAAC;IAE7C,8CAA8C;IAC9C,MAAM,UAAU,GAAG,oBAAQ,CACzB,IAAI,CAAC,OAAO,EAAE;IACd,2DAA2D;IAC3D,8DAA8D;IAC9D,aAAa;IACb,MAAM,CAAC,KAAK,EACZ,MAAM,CAAC,WAAW,EAClB,MAAM,CAAC,SAAS,EAChB,MAAM,CACP,CAAC;IAEF,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACvC,MAAM,CAAC,UAAU,CAAC,aAAa,QAAQ,EAAE,CAAC,CAAC;QAE3C,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACzC,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE;YAC/D,MAAM,IAAI,KAAK,CACb,qBAAqB,QAAQ,gDAAgD,CAC9E,CAAC;SACH;QAED,IAAI;YACF,IAAI,OAAO,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE;gBACjC,MAAM,iBAAiB,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;gBAC/C,MAAM,SAAS,GAAG,MAAM,aAAa,CACnC,QAAQ,EACR,SAAS,EACT,OAAO,CAAC,SAAS,CAAC,EAClB,WAAW,EACX,SAAS,CACV,CAAC;gBACF,MAAM,iBAAiB,CAAC,SAAS,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;gBAEzD,YAAY,CAAC,2BAA2B,QAAQ,cAAc,CAAC;oBAC7D,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,iBAAiB,CAAC;aAC5C;YACD,MAAM,eAAe,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;YAC7C,MAAM,iBAAiB,GAAG,MAAM,CAAC,OAAO,CAAC;YACzC,MAAM,mBAAmB,GAAa,EAAE,CAAC;YACzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,EAAE;gBACjD,IAAI,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE;oBAC3C,MAAM,SAAS,GAAG,MAAM,aAAa,CACnC,QAAQ,EACR,UAAU,CAAC,EAAE,EACb,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,EAC5B,iBAAiB,EACjB,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAChC,CAAC;oBACF,mBAAmB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;iBACrC;aACF;YACD,IAAI,mBAAmB,CAAC,MAAM,GAAG,CAAC,EAAE;gBAClC,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,QAAQ,eAAe,CAAC,CAAC;gBACrE,EAAE,CAAC,aAAa,CAAC,SAAS,EAAE,8BAAiB,CAAC,mBAAmB,CAAC,CAAC,CAAC;gBACpE,MAAM,iBAAiB,CAAC,SAAS,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;gBAEzD,YAAY,CAAC,0BAA0B,QAAQ,cAAc,CAAC;oBAC5D,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,eAAe,CAAC;aAC1C;SACF;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACf,YAAY,CAAC,wBAAwB,GAAG,QAAQ,CAAC;YACjD,MAAM,IAAI,mBAAmB,CAC3B,YAAY,EACZ,8BAA8B,QAAQ,KAAK,CAAC,EAAE,CAC/C,CAAC;SACH;KACF;IAED,OAAO,YAAY,CAAC;IAEpB,KAAK,UAAU,aAAa,CAC1B,QAAkB,EAClB,IAAY,EACZ,OAAiB,EACjB,iBAAyB,EACzB,UAA8B;QAE9B,MAAM,YAAY,GAAG,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;QAC1E,uEAAuE;QACvE,2EAA2E;QAC3E,MAAM,cAAc,GAAG,GAAG,YAAY,YAAY,IAAI,MAAM,CAAC;QAC7D,MAAM,kBAAkB,GAAG,OAAO;aAC/B,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,YAAY,CAAC,EAAE,CAAC;aACnC,IAAI,CAAC,IAAI,CAAC,CAAC;QACd,EAAE,CAAC,aAAa,CAAC,cAAc,EAAE,kBAAkB,CAAC,CAAC;QACrD,MAAM,CAAC,KAAK,CAAC,wBAAwB,QAAQ,QAAQ,kBAAkB,EAAE,CAAC,CAAC;QAE3E,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,GAAG,QAAQ,IAAI,IAAI,QAAQ,CAAC,CAAC;QAE5E,MAAM,MAAM,GAAG,kBAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAC3C,MAAM,MAAM,CAAC,eAAe,CAC1B,YAAY,EACZ,SAAS,EACT,UAAU,EACV,cAAc,EACd,UAAU,EACV,eAAe,EACf,WAAW,EACX,mBAAmB,CACpB,CAAC;QAEF,MAAM,CAAC,KAAK,CACV,8BAA8B,QAAQ,gBAAgB,SAAS,GAAG,CACnE,CAAC;QACF,MAAM,CAAC,QAAQ,EAAE,CAAC;QAElB,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAzHD,gCAyHC;AAEM,KAAK,UAAU,UAAU,CAC9B,SAAiB,EACjB,UAAkB,EAClB,eAAuB,EACvB,WAAmB,EACnB,mBAAuC,EACvC,MAA0B,EAC1B,MAAc;IAEd,8DAA8D;IAC9D,OAAO,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,0BAA0B,CAAC,CAAC;IAEzD,EAAE,CAAC,SAAS,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE7C,MAAM,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;IAC5C,MAAM,wBAAwB,CAAC,MAAM,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;IAE5D,MAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IAClC,MAAM,YAAY,GAAG,MAAM,UAAU,CACnC,SAAS,EACT,UAAU,EACV,eAAe,EACf,WAAW,EACX,mBAAmB,EACnB,MAAM,EACN,MAAM,CACP,CAAC;IAEF,OAAO,EAAE,GAAG,YAAY,EAAE,CAAC;AAC7B,CAAC;AA7BD,gCA6BC;AAED,KAAK,UAAU,iBAAiB,CAC9B,SAAiB,EACjB,QAAgB,EAChB,UAAwD;IAExD,MAAM,UAAU,GAAG,MAAM,UAAU,CAAC;IACpC,IAAI,QAAQ,IAAI,UAAU,EAAE;QAC1B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC;QAC7D,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE;YAC7B,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE;gBAC5B,MAAM,MAAM,GAAG,GAAG,QAAQ,wBAAwB,CAAC;gBACnD,GAAG,CAAC,UAAU,GAAG,GAAG,CAAC,UAAU,IAAI,EAAE,CAAC;gBACtC,GAAG,CAAC,UAAU,CAAC,aAAa,GAAG,GAAG,CAAC,UAAU,CAAC,aAAa,IAAI,EAAE,CAAC;gBAClE,MAAM,IAAI,GAAG,GAAG,CAAC,UAAU,CAAC,aAAa,CAAC,IAAI;gBAC5C,6CAA6C;gBAC7C,CAAC,CAAC,EAAE,EAAE,WAAC,OAAA,CAAC,CAAC,MAAM,KAAK,MAAM,IAAI,OAAA,CAAC,CAAC,IAAI,0CAAE,EAAE,MAAK,MAAM,CAAA,EAAA,CACpD,CAAC;gBACF,yDAAyD;gBACzD,IAAI,IAAI,EAAE;oBACR,IAAI,CAAC,QAAQ,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;iBACtC;aACF;SACF;QACD,EAAE,CAAC,aAAa,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;KACpD;AACH,CAAC"} \ No newline at end of file +{"version":3,"file":"analyze.js","sourceRoot":"","sources":["../src/analyze.ts"],"names":[],"mappings":";;;;;;;;;AAAA,uCAAyB;AACzB,2CAA6B;AAE7B,yEAA2D;AAE3D,gEAAkD;AAClD,qCAAqC;AAErC,2CAAmD;AACnD,2CAA0D;AAE1D,gEAAkD;AAClD,6CAAiD;AACjD,6CAA+B;AAE/B,MAAa,mBAAoB,SAAQ,KAAK;IAG5C,YAAY,mBAAwC,EAAE,OAAe;QACnE,KAAK,CAAC,OAAO,CAAC,CAAC;QAEf,IAAI,CAAC,IAAI,GAAG,qBAAqB,CAAC;QAClC,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAC;IACjD,CAAC;CACF;AATD,kDASC;AA+BD,KAAK,UAAU,oBAAoB,CAAC,MAAc;IAChD,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;IAClD,IAAI,YAAY,KAAK,SAAS,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE;QAC3D,+FAA+F;QAC/F,OAAO;KACR;IAED,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,MAAM,OAAO,GAAG;QACd,SAAS,EAAE;YACT,MAAM,EAAE,CAAC,IAAY,EAAE,EAAE;gBACvB,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC5B,CAAC;SACF;KACF,CAAC;IAEF,MAAM,IAAI,UAAU,CAAC,UAAU,CAC7B,YAAY,EACZ;QACE,IAAI;QACJ,8EAA8E;KAC/E,EACD,OAAO,CACR,CAAC,IAAI,EAAE,CAAC;IACT,MAAM,CAAC,IAAI,CAAC,kCAAkC,MAAM,EAAE,CAAC,CAAC;IACxD,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,GAAG,MAAM,CAAC;IAE/C,MAAM,GAAG,EAAE,CAAC;IACZ,MAAM,IAAI,UAAU,CAAC,UAAU,CAC7B,YAAY,EACZ,CAAC,IAAI,EAAE,wCAAwC,CAAC,EAChD,OAAO,CACR,CAAC,IAAI,EAAE,CAAC;IACT,MAAM,CAAC,IAAI,CAAC,qCAAqC,MAAM,EAAE,CAAC,CAAC;IAC3D,OAAO,CAAC,GAAG,CAAC,2BAA2B,CAAC,GAAG,MAAM,CAAC;AACpD,CAAC;AAED,KAAK,UAAU,4BAA4B,CACzC,MAA0B,EAC1B,MAAc;IAEd,sEAAsE;IACtE,oCAAoC;IACpC,aAAa,CAAC,8BAA8B,CAAC,MAAM,CAAC,CAAC;IAErD,MAAM,MAAM,GAAG,kBAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAC3C,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACvC,IAAI,6BAAiB,CAAC,QAAQ,CAAC,EAAE;YAC/B,MAAM,CAAC,UAAU,CAAC,cAAc,QAAQ,EAAE,CAAC,CAAC;YAE5C,IAAI,QAAQ,KAAK,oBAAQ,CAAC,MAAM,EAAE;gBAChC,MAAM,oBAAoB,CAAC,MAAM,CAAC,CAAC;aACpC;YAED,MAAM,MAAM,CAAC,sBAAsB,CACjC,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,EACpD,QAAQ,CACT,CAAC;YACF,MAAM,CAAC,QAAQ,EAAE,CAAC;SACnB;KACF;AACH,CAAC;AAED,KAAK,UAAU,wBAAwB,CACrC,MAA0B,EAC1B,WAAmB,EACnB,MAAc;IAEd,MAAM,4BAA4B,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEnD,MAAM,MAAM,GAAG,kBAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAC3C,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACvC,MAAM,CAAC,UAAU,CAAC,cAAc,QAAQ,EAAE,CAAC,CAAC;QAC5C,MAAM,MAAM,CAAC,gBAAgB,CAC3B,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,EACpD,WAAW,CACZ,CAAC;QACF,MAAM,CAAC,QAAQ,EAAE,CAAC;KACnB;AACH,CAAC;AAED,2DAA2D;AACpD,KAAK,UAAU,UAAU,CAC9B,WAAmB,EACnB,UAAkB,EAClB,eAAuB,EACvB,WAAmB,EACnB,mBAAuC,EACvC,MAA0B,EAC1B,MAAc;IAEd,MAAM,YAAY,GAAwB,EAAE,CAAC;IAE7C,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACvC,MAAM,CAAC,UAAU,CAAC,aAAa,QAAQ,EAAE,CAAC,CAAC;QAE3C,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACzC,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE;YAC/D,MAAM,IAAI,KAAK,CACb,qBAAqB,QAAQ,gDAAgD,CAC9E,CAAC;SACH;QAED,MAAM,aAAa,GAAa,EAAE,CAAC;QACnC,IAAI;YACF,IAAI,OAAO,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE;gBACjC,MAAM,iBAAiB,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;gBAC/C,MAAM,SAAS,GAAG,MAAM,aAAa,CACnC,QAAQ,EACR,SAAS,EACT,OAAO,CAAC,SAAS,CAAC,EAClB,WAAW,EACX,SAAS,CACV,CAAC;gBAEF,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBAE9B,YAAY,CAAC,2BAA2B,QAAQ,cAAc,CAAC;oBAC7D,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,iBAAiB,CAAC;aAC5C;YACD,MAAM,eAAe,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;YAC7C,MAAM,iBAAiB,GAAG,MAAM,CAAC,OAAO,CAAC;YACzC,MAAM,mBAAmB,GAAa,EAAE,CAAC;YACzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,EAAE;gBACjD,IAAI,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE;oBAC3C,MAAM,SAAS,GAAG,MAAM,aAAa,CACnC,QAAQ,EACR,UAAU,CAAC,EAAE,EACb,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,EAC5B,iBAAiB,EACjB,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAChC,CAAC;oBACF,mBAAmB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;oBACpC,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;iBAC/B;aACF;YAED,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE;gBAC5B,MAAM,WAAW,GAAG,MAAM,oBAAQ,CAChC,IAAI,CAAC,OAAO,EAAE;gBACd,2DAA2D;gBAC3D,8DAA8D;gBAC9D,aAAa;gBACb,MAAM,CAAC,KAAK,EACZ,MAAM,CAAC,WAAW,EAClB,MAAM,CAAC,SAAS,EAChB,MAAM,CACP,CAAC;gBAEF,KAAK,MAAM,SAAS,IAAI,aAAa,EAAE;oBACrC,iBAAiB,CAAC,SAAS,EAAE,QAAQ,EAAE,WAAW,CAAC,CAAC;iBACrD;aACF;YAED,IAAI,mBAAmB,CAAC,MAAM,GAAG,CAAC,EAAE;gBAClC,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,QAAQ,eAAe,CAAC,CAAC;gBACrE,EAAE,CAAC,aAAa,CAAC,SAAS,EAAE,8BAAiB,CAAC,mBAAmB,CAAC,CAAC,CAAC;gBAEpE,YAAY,CAAC,0BAA0B,QAAQ,cAAc,CAAC;oBAC5D,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,eAAe,CAAC;aAC1C;SACF;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACf,YAAY,CAAC,wBAAwB,GAAG,QAAQ,CAAC;YACjD,MAAM,IAAI,mBAAmB,CAC3B,YAAY,EACZ,8BAA8B,QAAQ,KAAK,CAAC,EAAE,CAC/C,CAAC;SACH;KACF;IAED,OAAO,YAAY,CAAC;IAEpB,KAAK,UAAU,aAAa,CAC1B,QAAkB,EAClB,IAAY,EACZ,OAAiB,EACjB,iBAAyB,EACzB,UAA8B;QAE9B,MAAM,YAAY,GAAG,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;QAC1E,uEAAuE;QACvE,2EAA2E;QAC3E,MAAM,cAAc,GAAG,GAAG,YAAY,YAAY,IAAI,MAAM,CAAC;QAC7D,MAAM,kBAAkB,GAAG,OAAO;aAC/B,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,YAAY,CAAC,EAAE,CAAC;aACnC,IAAI,CAAC,IAAI,CAAC,CAAC;QACd,EAAE,CAAC,aAAa,CAAC,cAAc,EAAE,kBAAkB,CAAC,CAAC;QACrD,MAAM,CAAC,KAAK,CAAC,wBAAwB,QAAQ,QAAQ,kBAAkB,EAAE,CAAC,CAAC;QAE3E,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,GAAG,QAAQ,IAAI,IAAI,QAAQ,CAAC,CAAC;QAE5E,MAAM,MAAM,GAAG,kBAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAC3C,MAAM,MAAM,CAAC,eAAe,CAC1B,YAAY,EACZ,SAAS,EACT,UAAU,EACV,cAAc,EACd,UAAU,EACV,eAAe,EACf,WAAW,EACX,mBAAmB,CACpB,CAAC;QAEF,MAAM,CAAC,KAAK,CACV,8BAA8B,QAAQ,gBAAgB,SAAS,GAAG,CACnE,CAAC;QACF,MAAM,CAAC,QAAQ,EAAE,CAAC;QAElB,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAjID,gCAiIC;AAEM,KAAK,UAAU,UAAU,CAC9B,SAAiB,EACjB,UAAkB,EAClB,eAAuB,EACvB,WAAmB,EACnB,mBAAuC,EACvC,MAA0B,EAC1B,MAAc;IAEd,8DAA8D;IAC9D,OAAO,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,0BAA0B,CAAC,CAAC;IAEzD,EAAE,CAAC,SAAS,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE7C,MAAM,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;IAC5C,MAAM,wBAAwB,CAAC,MAAM,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;IAE5D,MAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IAClC,MAAM,YAAY,GAAG,MAAM,UAAU,CACnC,SAAS,EACT,UAAU,EACV,eAAe,EACf,WAAW,EACX,mBAAmB,EACnB,MAAM,EACN,MAAM,CACP,CAAC;IAEF,OAAO,EAAE,GAAG,YAAY,EAAE,CAAC;AAC7B,CAAC;AA7BD,gCA6BC;AAED,SAAS,iBAAiB,CACxB,SAAiB,EACjB,QAAgB,EAChB,WAAgD;IAEhD,IAAI,QAAQ,IAAI,WAAW,EAAE;QAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC;QAC7D,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE;YAC7B,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE;gBAC5B,MAAM,MAAM,GAAG,GAAG,QAAQ,wBAAwB,CAAC;gBACnD,GAAG,CAAC,UAAU,GAAG,GAAG,CAAC,UAAU,IAAI,EAAE,CAAC;gBACtC,GAAG,CAAC,UAAU,CAAC,aAAa,GAAG,GAAG,CAAC,UAAU,CAAC,aAAa,IAAI,EAAE,CAAC;gBAClE,MAAM,IAAI,GAAG,GAAG,CAAC,UAAU,CAAC,aAAa,CAAC,IAAI;gBAC5C,6CAA6C;gBAC7C,CAAC,CAAC,EAAE,EAAE,WAAC,OAAA,CAAC,CAAC,MAAM,KAAK,MAAM,IAAI,OAAA,CAAC,CAAC,IAAI,0CAAE,EAAE,MAAK,MAAM,CAAA,EAAA,CACpD,CAAC;gBACF,yDAAyD;gBACzD,IAAI,IAAI,EAAE;oBACR,IAAI,CAAC,QAAQ,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC;iBACvC;aACF;SACF;QACD,EAAE,CAAC,aAAa,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;KACpD;AACH,CAAC"} \ No newline at end of file diff --git a/src/analyze.ts b/src/analyze.ts index 3f317f68c..f0e88d121 100644 --- a/src/analyze.ts +++ b/src/analyze.ts @@ -146,18 +146,6 @@ export async function runQueries( ): Promise { const statusReport: QueriesStatusReport = {}; - // count the number of lines in the background - const locPromise = countLoc( - path.resolve(), - // config.paths specifies external directories. the current - // directory is included in the analysis by default. Replicate - // that here. - config.paths, - config.pathsIgnore, - config.languages, - logger - ); - for (const language of config.languages) { logger.startGroup(`Analyzing ${language}`); @@ -168,6 +156,7 @@ export async function runQueries( ); } + const allSarifFiles: string[] = []; try { if (queries["builtin"].length > 0) { const startTimeBuliltIn = new Date().getTime(); @@ -178,7 +167,8 @@ export async function runQueries( sarifFolder, undefined ); - await injectLinesOfCode(sarifFile, language, locPromise); + + allSarifFiles.push(sarifFile); statusReport[`analyze_builtin_queries_${language}_duration_ms`] = new Date().getTime() - startTimeBuliltIn; @@ -196,12 +186,30 @@ export async function runQueries( queries["custom"][i].searchPath ); temporarySarifFiles.push(sarifFile); + allSarifFiles.push(sarifFile); } } + + if (allSarifFiles.length > 0) { + const linesOfCode = await countLoc( + path.resolve(), + // config.paths specifies external directories. the current + // directory is included in the analysis by default. Replicate + // that here. + config.paths, + config.pathsIgnore, + config.languages, + logger + ); + + for (const sarifFile of allSarifFiles) { + injectLinesOfCode(sarifFile, language, linesOfCode); + } + } + if (temporarySarifFiles.length > 0) { const sarifFile = path.join(sarifFolder, `${language}-custom.sarif`); fs.writeFileSync(sarifFile, combineSarifFiles(temporarySarifFiles)); - await injectLinesOfCode(sarifFile, language, locPromise); statusReport[`analyze_custom_queries_${language}_duration_ms`] = new Date().getTime() - startTimeCustom; @@ -289,13 +297,12 @@ export async function runAnalyze( return { ...queriesStats }; } -async function injectLinesOfCode( +function injectLinesOfCode( sarifFile: string, language: string, - locPromise: Promise>> + linesOfCode: Partial> ) { - const lineCounts = await locPromise; - if (language in lineCounts) { + if (language in linesOfCode) { const sarif = JSON.parse(fs.readFileSync(sarifFile, "utf8")); if (Array.isArray(sarif.runs)) { for (const run of sarif.runs) { @@ -308,7 +315,7 @@ async function injectLinesOfCode( ); // only add the baseline value if the rule already exists if (rule) { - rule.baseline = lineCounts[language]; + rule.baseline = linesOfCode[language]; } } }