Merge 800a951427 into eeaefb6054

Merge pull request #517 from github/aibaars-additional-packs
Use --additional-packs instead of --search-path
2025-12-07 00:08:06 +08:00 · 2021-05-21 18:04:50 +00:00 · 2021-05-21 09:31:17 -07:00 · 2021-05-21 09:21:54 -07:00 · 2021-05-21 12:54:23 +02:00 · 2021-05-21 12:41:35 +02:00
21 changed files with 147 additions and 59 deletions
--- a/.gitattributes
+++ b/.gitattributes
@@ -1 +1,8 @@
 lib/*.js linguist-generated=true
+
+# Reduce incidence of needless merge conflicts on CHANGELOG.md
+# The man page at
+# https://mirrors.edge.kernel.org/pub/software/scm/git/docs/gitattributes.html
+# suggests that this might interleave lines arbitrarily, but empirically
+# it keeps added chunks contiguous
+CHANGELOG.md                merge=union
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,6 @@
 # CodeQL Action and CodeQL Runner Changelog

 ## [UNRELEASED]
+
+- Add this changelog file. [#507](https://github.com/github/codeql-action/pull/507)
+- Improve grouping of analysis logs. Add a new log group containing a summary of metrics and diagnostics, if they were produced by CodeQL builtin queries. [#515](https://github.com/github/codeql-action/pull/515)
--- a/README.md
+++ b/README.md
@@ -2,6 +2,8 @@

 This action runs GitHub's industry-leading static analysis engine, CodeQL, against a repository's source code to find security vulnerabilities. It then automatically uploads the results to GitHub so they can be displayed in the repository's security tab. CodeQL runs an extensible set of [queries](https://github.com/github/codeql), which have been developed by the community and the [GitHub Security Lab](https://securitylab.github.com/) to find common vulnerabilities in your code.

+For a list of recent changes, see the CodeQL Action's [changelog](CHANGELOG.md).
+
 ## License

 This project is released under the [MIT License](LICENSE).
--- a/lib/analyze.js
+++ b/lib/analyze.js
@@ -87,13 +87,16 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
    for (const language of config.languages) {
        logger.startGroup(`Analyzing ${language}`);
        const queries = config.queries[language];
-        if (queries.builtin.length === 0 && queries.custom.length === 0) {
+        if (queries === undefined ||
+            (queries.builtin.length === 0 && queries.custom.length === 0)) {
            throw new Error(`Unable to analyse ${language} as no queries were selected for this language`);
        }
        try {
+            let analysisSummary = "";
            if (queries["builtin"].length > 0) {
                const startTimeBuliltIn = new Date().getTime();
-                const sarifFile = await runQueryGroup(language, "builtin", queries["builtin"], sarifFolder, undefined);
+                const { sarifFile, stdout } = await runQueryGroup(language, "builtin", queries["builtin"], sarifFolder, undefined);
+                analysisSummary = stdout;
                await injectLinesOfCode(sarifFile, language, locPromise);
                statusReport[`analyze_builtin_queries_${language}_duration_ms`] =
                    new Date().getTime() - startTimeBuliltIn;
@@ -103,7 +106,7 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
            const temporarySarifFiles = [];
            for (let i = 0; i < queries["custom"].length; ++i) {
                if (queries["custom"][i].queries.length > 0) {
-                    const sarifFile = await runQueryGroup(language, `custom-${i}`, queries["custom"][i].queries, temporarySarifDir, queries["custom"][i].searchPath);
+                    const { sarifFile } = await runQueryGroup(language, `custom-${i}`, queries["custom"][i].queries, temporarySarifDir, queries["custom"][i].searchPath);
                    temporarySarifFiles.push(sarifFile);
                }
            }
@@ -114,7 +117,12 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
                statusReport[`analyze_custom_queries_${language}_duration_ms`] =
                    new Date().getTime() - startTimeCustom;
            }
+            logger.endGroup();
+            // Print the LoC baseline and the summary results from database analyze.
+            logger.startGroup(`Analysis summary for ${language}`);
            printLinesOfCodeSummary(logger, language, await locPromise);
+            logger.info(analysisSummary);
+            logger.endGroup();
        }
        catch (e) {
            logger.info(e);
@@ -135,10 +143,9 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
        logger.debug(`Query suite file for ${language}...\n${querySuiteContents}`);
        const sarifFile = path.join(destinationFolder, `${language}-${type}.sarif`);
        const codeql = codeql_1.getCodeQL(config.codeQLCmd);
-        await codeql.databaseAnalyze(databasePath, sarifFile, searchPath, querySuitePath, memoryFlag, addSnippetsFlag, threadsFlag, automationDetailsId);
+        const databaseAnalyzeStdout = await codeql.databaseAnalyze(databasePath, sarifFile, searchPath, querySuitePath, memoryFlag, addSnippetsFlag, threadsFlag, automationDetailsId);
        logger.debug(`SARIF results for database ${language} created at "${sarifFile}"`);
-        logger.endGroup();
-        return sarifFile;
+        return { sarifFile, stdout: databaseAnalyzeStdout };
    }
 }
 exports.runQueries = runQueries;
--- a/lib/analyze.js.map
+++ b/lib/analyze.js.map
--- a/lib/analyze.test.js
+++ b/lib/analyze.test.js
@@ -72,6 +72,7 @@ ava_1.default("status report fields and search path setting", async (t) => {
                        ],
                    }));
                    searchPathsUsed.push(searchPath);
+                    return "";
                },
            });
            searchPathsUsed = [];
--- a/lib/analyze.test.js.map
+++ b/lib/analyze.test.js.map
@@ -1 +1 @@
-{"version":3,"file":"analyze.test.js","sourceRoot":"","sources":["../src/analyze.test.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,uCAAyB;AACzB,2CAA6B;AAE7B,8CAAuB;AACvB,kDAA0B;AAE1B,uCAAuC;AACvC,qCAAqC;AAErC,2CAA0C;AAC1C,mDAAqC;AACrC,2CAAuC;AACvC,uCAA4C;AAC5C,mDAA+D;AAC/D,6CAA+B;AAE/B,0BAAU,CAAC,aAAI,CAAC,CAAC;AAEjB,yEAAyE;AACzE,wEAAwE;AACxE,oDAAoD;AACpD,aAAI,CAAC,8CAA8C,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IAC/D,MAAM,eAAe,GAAG,MAAM,CAAC,MAAM,CAAC,oBAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE;QACtE,+CAA+C;QAC/C,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAClB,OAAO,GAAG,CAAC;IACb,CAAC,EAAE,EAAE,CAAC,CAAC;IACP,eAAK,CAAC,IAAI,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC;IACxD,IAAI,eAAe,GAAa,EAAE,CAAC;IACnC,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,gCAAgB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAEjC,MAAM,UAAU,GAAG,EAAE,CAAC;QACtB,MAAM,eAAe,GAAG,EAAE,CAAC;QAC3B,MAAM,WAAW,GAAG,EAAE,CAAC;QAEvB,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,MAAM,CAAC,oBAAQ,CAAC,EAAE;YAC9C,kBAAS,CAAC;gBACR,eAAe,EAAE,KAAK,EACpB,CAAC,EACD,SAAiB,EACjB,UAA8B,EAC9B,EAAE;oBACF,EAAE,CAAC,aAAa,CACd,SAAS,EACT,IAAI,CAAC,SAAS,CAAC;wBACb,IAAI,EAAE;4BACJ,wBAAwB;4BACxB;gCACE,UAAU,EAAE;oCACV,aAAa,EAAE;wCACb;4CACE,MAAM,EAAE,GAAG,uBAAW,CACpB,QAAQ,CACT,wBAAwB;4CACzB,KAAK,EAAE,GAAG;yCACX;qCACF;iCACF;6BACF;4BACD,yBAAyB;4BACzB;gCACE,UAAU,EAAE;oCACV,aAAa,EAAE;wCACb;4CACE,IAAI,EAAE;gDACJ,EAAE,EAAE,GAAG,uBAAW,CAAC,QAAQ,CAAC,wBAAwB;6CACrD;4CACD,KAAK,EAAE,GAAG;yCACX;qCACF;iCACF;6BACF;4BACD,EAAE;yBACH;qBACF,CAAC,CACH,CAAC;oBACF,eAAe,CAAC,IAAI,CAAC,UAAW,CAAC,CAAC;gBACpC,CAAC;aACF,CAAC,CAAC;YAEH,eAAe,GAAG,EAAE,CAAC;YACrB,MAAM,MAAM,GAAW;gBACrB,SAAS,EAAE,CAAC,QAAQ,CAAC;gBACrB,OAAO,EAAE,EAAE;gBACX,WAAW,EAAE,EAAE;gBACf,KAAK,EAAE,EAAE;gBACT,iBAAiB,EAAE,EAAE;gBACrB,OAAO,EAAE,MAAM;gBACf,YAAY,EAAE,MAAM;gBACpB,SAAS,EAAE,EAAE;gBACb,aAAa,EAAE;oBACb,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM;iBACV;gBACvB,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,kBAAkB,CAAC;aACrD,CAAC;YACF,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,qBAAqB,CAAC,MAAM,EAAE,QAAQ,CAAC,EAAE;gBACzD,SAAS,EAAE,IAAI;aAChB,CAAC,CAAC;YAEH,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG;gBACzB,OAAO,EAAE,CAAC,QAAQ,CAAC;gBACnB,MAAM,EAAE,EAAE;aACX,CAAC;YACF,MAAM,mBAAmB,GAAG,MAAM,oBAAU,CAC1C,MAAM,EACN,UAAU,EACV,eAAe,EACf,WAAW,EACX,SAAS,EACT,MAAM,EACN,yBAAe,CAAC,IAAI,CAAC,CACtB,CAAC;YACF,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YACxD,CAAC,CAAC,IAAI,CACJ,2BAA2B,QAAQ,cAAc,IAAI,mBAAmB,CACzE,CAAC;YAEF,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG;gBACzB,OAAO,EAAE,EAAE;gBACX,MAAM,EAAE;oBACN;wBACE,OAAO,EAAE,CAAC,QAAQ,CAAC;wBACnB,UAAU,EAAE,IAAI;qBACjB;oBACD;wBACE,OAAO,EAAE,CAAC,QAAQ,CAAC;wBACnB,UAAU,EAAE,IAAI;qBACjB;iBACF;aACF,CAAC;YACF,MAAM,kBAAkB,GAAG,MAAM,oBAAU,CACzC,MAAM,EACN,UAAU,EACV,eAAe,EACf,WAAW,EACX,SAAS,EACT,MAAM,EACN,yBAAe,CAAC,IAAI,CAAC,CACtB,CAAC;YACF,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YACvD,CAAC,CAAC,IAAI,CACJ,0BAA0B,QAAQ,cAAc,IAAI,kBAAkB,CACvE,CAAC;YACF,CAAC,CAAC,SAAS,CAAC,eAAe,EAAE,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;SACvD;QAED,gBAAgB,CAAC,MAAM,CAAC,CAAC;IAC3B,CAAC,CAAC,CAAC;IAEH,SAAS,gBAAgB,CAAC,MAAc;QACtC,gDAAgD;QAChD,MAAM,CAAC,IAAI,CAAC,oBAAQ,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE;YACxC,sBAAsB,CACpB,IAAgB,EAChB,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,gBAAgB,CAAC,EAC1C,CAAC,GAAG,CAAC,CACN,CAAC;YACF,sBAAsB,CACpB,IAAgB,EAChB,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,eAAe,CAAC,EACzC,CAAC,GAAG,CAAC,CACN,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;IAED,SAAS,sBAAsB,CAC7B,IAAc,EACd,QAAgB,EAChB,SAAiB;QAEjB,MAAM,QAAQ,GAAG,uBAAW,CAAC,IAAI,CAAC,CAAC;QACnC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;QAC5D,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,aAAa,EAAE;YAClD;gBACE,MAAM,EAAE,GAAG,QAAQ,wBAAwB;gBAC3C,KAAK,EAAE,GAAG;gBACV,QAAQ,EAAE,SAAS;aACpB;SACF,CAAC,CAAC;QACH,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,aAAa,EAAE;YAClD;gBACE,IAAI,EAAE;oBACJ,EAAE,EAAE,GAAG,QAAQ,wBAAwB;iBACxC;gBACD,KAAK,EAAE,GAAG;gBACV,QAAQ,EAAE,SAAS;aACpB;SACF,CAAC,CAAC;QACH,uDAAuD;QACvD,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC;IAC1D,CAAC;AACH,CAAC,CAAC,CAAC"}
+{"version":3,"file":"analyze.test.js","sourceRoot":"","sources":["../src/analyze.test.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,uCAAyB;AACzB,2CAA6B;AAE7B,8CAAuB;AACvB,kDAA0B;AAE1B,uCAAuC;AACvC,qCAAqC;AAErC,2CAA0C;AAC1C,mDAAqC;AACrC,2CAAuC;AACvC,uCAA4C;AAC5C,mDAA+D;AAC/D,6CAA+B;AAE/B,0BAAU,CAAC,aAAI,CAAC,CAAC;AAEjB,yEAAyE;AACzE,wEAAwE;AACxE,oDAAoD;AACpD,aAAI,CAAC,8CAA8C,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IAC/D,MAAM,eAAe,GAAG,MAAM,CAAC,MAAM,CAAC,oBAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE;QACtE,+CAA+C;QAC/C,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAClB,OAAO,GAAG,CAAC;IACb,CAAC,EAAE,EAAE,CAAC,CAAC;IACP,eAAK,CAAC,IAAI,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC;IACxD,IAAI,eAAe,GAAa,EAAE,CAAC;IACnC,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,gCAAgB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAEjC,MAAM,UAAU,GAAG,EAAE,CAAC;QACtB,MAAM,eAAe,GAAG,EAAE,CAAC;QAC3B,MAAM,WAAW,GAAG,EAAE,CAAC;QAEvB,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,MAAM,CAAC,oBAAQ,CAAC,EAAE;YAC9C,kBAAS,CAAC;gBACR,eAAe,EAAE,KAAK,EACpB,CAAC,EACD,SAAiB,EACjB,UAA8B,EAC9B,EAAE;oBACF,EAAE,CAAC,aAAa,CACd,SAAS,EACT,IAAI,CAAC,SAAS,CAAC;wBACb,IAAI,EAAE;4BACJ,wBAAwB;4BACxB;gCACE,UAAU,EAAE;oCACV,aAAa,EAAE;wCACb;4CACE,MAAM,EAAE,GAAG,uBAAW,CACpB,QAAQ,CACT,wBAAwB;4CACzB,KAAK,EAAE,GAAG;yCACX;qCACF;iCACF;6BACF;4BACD,yBAAyB;4BACzB;gCACE,UAAU,EAAE;oCACV,aAAa,EAAE;wCACb;4CACE,IAAI,EAAE;gDACJ,EAAE,EAAE,GAAG,uBAAW,CAAC,QAAQ,CAAC,wBAAwB;6CACrD;4CACD,KAAK,EAAE,GAAG;yCACX;qCACF;iCACF;6BACF;4BACD,EAAE;yBACH;qBACF,CAAC,CACH,CAAC;oBACF,eAAe,CAAC,IAAI,CAAC,UAAW,CAAC,CAAC;oBAClC,OAAO,EAAE,CAAC;gBACZ,CAAC;aACF,CAAC,CAAC;YAEH,eAAe,GAAG,EAAE,CAAC;YACrB,MAAM,MAAM,GAAW;gBACrB,SAAS,EAAE,CAAC,QAAQ,CAAC;gBACrB,OAAO,EAAE,EAAE;gBACX,WAAW,EAAE,EAAE;gBACf,KAAK,EAAE,EAAE;gBACT,iBAAiB,EAAE,EAAE;gBACrB,OAAO,EAAE,MAAM;gBACf,YAAY,EAAE,MAAM;gBACpB,SAAS,EAAE,EAAE;gBACb,aAAa,EAAE;oBACb,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM;iBACV;gBACvB,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,kBAAkB,CAAC;aACrD,CAAC;YACF,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,qBAAqB,CAAC,MAAM,EAAE,QAAQ,CAAC,EAAE;gBACzD,SAAS,EAAE,IAAI;aAChB,CAAC,CAAC;YAEH,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG;gBACzB,OAAO,EAAE,CAAC,QAAQ,CAAC;gBACnB,MAAM,EAAE,EAAE;aACX,CAAC;YACF,MAAM,mBAAmB,GAAG,MAAM,oBAAU,CAC1C,MAAM,EACN,UAAU,EACV,eAAe,EACf,WAAW,EACX,SAAS,EACT,MAAM,EACN,yBAAe,CAAC,IAAI,CAAC,CACtB,CAAC;YACF,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YACxD,CAAC,CAAC,IAAI,CACJ,2BAA2B,QAAQ,cAAc,IAAI,mBAAmB,CACzE,CAAC;YAEF,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG;gBACzB,OAAO,EAAE,EAAE;gBACX,MAAM,EAAE;oBACN;wBACE,OAAO,EAAE,CAAC,QAAQ,CAAC;wBACnB,UAAU,EAAE,IAAI;qBACjB;oBACD;wBACE,OAAO,EAAE,CAAC,QAAQ,CAAC;wBACnB,UAAU,EAAE,IAAI;qBACjB;iBACF;aACF,CAAC;YACF,MAAM,kBAAkB,GAAG,MAAM,oBAAU,CACzC,MAAM,EACN,UAAU,EACV,eAAe,EACf,WAAW,EACX,SAAS,EACT,MAAM,EACN,yBAAe,CAAC,IAAI,CAAC,CACtB,CAAC;YACF,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YACvD,CAAC,CAAC,IAAI,CACJ,0BAA0B,QAAQ,cAAc,IAAI,kBAAkB,CACvE,CAAC;YACF,CAAC,CAAC,SAAS,CAAC,eAAe,EAAE,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;SACvD;QAED,gBAAgB,CAAC,MAAM,CAAC,CAAC;IAC3B,CAAC,CAAC,CAAC;IAEH,SAAS,gBAAgB,CAAC,MAAc;QACtC,gDAAgD;QAChD,MAAM,CAAC,IAAI,CAAC,oBAAQ,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE;YACxC,sBAAsB,CACpB,IAAgB,EAChB,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,gBAAgB,CAAC,EAC1C,CAAC,GAAG,CAAC,CACN,CAAC;YACF,sBAAsB,CACpB,IAAgB,EAChB,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,eAAe,CAAC,EACzC,CAAC,GAAG,CAAC,CACN,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;IAED,SAAS,sBAAsB,CAC7B,IAAc,EACd,QAAgB,EAChB,SAAiB;QAEjB,MAAM,QAAQ,GAAG,uBAAW,CAAC,IAAI,CAAC,CAAC;QACnC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;QAC5D,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,aAAa,EAAE;YAClD;gBACE,MAAM,EAAE,GAAG,QAAQ,wBAAwB;gBAC3C,KAAK,EAAE,GAAG;gBACV,QAAQ,EAAE,SAAS;aACpB;SACF,CAAC,CAAC;QACH,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,aAAa,EAAE;YAClD;gBACE,IAAI,EAAE;oBACJ,EAAE,EAAE,GAAG,QAAQ,wBAAwB;iBACxC;gBACD,KAAK,EAAE,GAAG;gBACV,QAAQ,EAAE,SAAS;aACpB;SACF,CAAC,CAAC;QACH,uDAAuD;QACvD,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC;IAC1D,CAAC;AACH,CAAC,CAAC,CAAC"}
--- a/lib/codeql.js
+++ b/lib/codeql.js
@@ -427,7 +427,7 @@ function getCodeQLForCmd(cmd) {
                ...getExtraOptionsFromEnv(["resolve", "queries"]),
            ];
            if (extraSearchPath !== undefined) {
-                codeqlArgs.push("--search-path", extraSearchPath);
+                codeqlArgs.push("--additional-packs", extraSearchPath);
            }
            let output = "";
            await new toolrunner.ToolRunner(cmd, codeqlArgs, {
@@ -457,13 +457,22 @@ function getCodeQLForCmd(cmd) {
                ...getExtraOptionsFromEnv(["database", "analyze"]),
            ];
            if (extraSearchPath !== undefined) {
-                args.push("--search-path", extraSearchPath);
+                args.push("--additional-packs", extraSearchPath);
            }
            if (automationDetailsId !== undefined) {
                args.push("--sarif-category", automationDetailsId);
            }
            args.push(querySuite);
-            await new toolrunner.ToolRunner(cmd, args).exec();
+            // capture stdout, which contains analysis summaries
+            let output = "";
+            await new toolrunner.ToolRunner(cmd, args, {
+                listeners: {
+                    stdout: (data) => {
+                        output += data.toString("utf8");
+                    },
+                },
+            }).exec();
+            return output;
        },
    };
 }
--- a/lib/codeql.js.map
+++ b/lib/codeql.js.map
--- a/lib/config-utils.js
+++ b/lib/config-utils.js
@@ -394,6 +394,12 @@ function shouldAddConfigFileQueries(queriesInput) {
 async function getDefaultConfig(languagesInput, queriesInput, dbLocation, repository, tempDir, toolCacheDir, codeQL, checkoutPath, gitHubVersion, apiDetails, logger) {
    const languages = await getLanguages(languagesInput, repository, apiDetails, logger);
    const queries = {};
+    for (const language of languages) {
+        queries[language] = {
+            builtin: [],
+            custom: [],
+        };
+    }
    await addDefaultQueries(codeQL, languages, queries);
    if (queriesInput) {
        await addQueriesFromWorkflow(codeQL, queriesInput, languages, queries, tempDir, checkoutPath, apiDetails, logger);
@@ -437,6 +443,12 @@ async function loadConfig(languagesInput, queriesInput, configFile, dbLocation,
    }
    const languages = await getLanguages(languagesInput, repository, apiDetails, logger);
    const queries = {};
+    for (const language of languages) {
+        queries[language] = {
+            builtin: [],
+            custom: [],
+        };
+    }
    const pathsIgnore = [];
    const paths = [];
    let disableDefaultQueries = false;
@@ -491,16 +503,6 @@ async function loadConfig(languagesInput, queriesInput, configFile, dbLocation,
            paths.push(validateAndSanitisePath(includePath, PATHS_PROPERTY, configFile, logger));
        }
    }
-    // The list of queries should not be empty for any language. If it is then
-    // it is a user configuration error.
-    for (const language of languages) {
-        if (queries[language] === undefined ||
-            (queries[language].builtin.length === 0 &&
-                queries[language].custom.length === 0)) {
-            throw new Error(`Did not detect any queries to run for ${language}. ` +
-                "Please make sure that the default queries are enabled, or you are specifying queries to run.");
-        }
-    }
    return {
        languages,
        queries,
@@ -533,6 +535,16 @@ async function initConfig(languagesInput, queriesInput, configFile, dbLocation,
    else {
        config = await loadConfig(languagesInput, queriesInput, configFile, dbLocation, repository, tempDir, toolCacheDir, codeQL, checkoutPath, gitHubVersion, apiDetails, logger);
    }
+    // The list of queries should not be empty for any language. If it is then
+    // it is a user configuration error.
+    for (const language of config.languages) {
+        if (config.queries[language] === undefined ||
+            (config.queries[language].builtin.length === 0 &&
+                config.queries[language].custom.length === 0)) {
+            throw new Error(`Did not detect any queries to run for ${language}. ` +
+                "Please make sure that the default queries are enabled, or you are specifying queries to run.");
+        }
+    }
    // Save the config so we can easily access it again in the future
    await saveConfig(config, logger);
    return config;
--- a/lib/config-utils.js.map
+++ b/lib/config-utils.js.map
--- a/lib/config-utils.test.js
+++ b/lib/config-utils.test.js
@@ -66,7 +66,10 @@ ava_1.default("load empty config", async (t) => {
        const codeQL = codeql_1.setCodeQL({
            async resolveQueries() {
                return {
-                    byLanguage: {},
+                    byLanguage: {
+                        javascript: { queries: ["query1.ql"] },
+                        python: { queries: ["query2.ql"] },
+                    },
                    noDeclaredLanguage: {},
                    multipleDeclaredLanguages: {},
                };
@@ -82,7 +85,10 @@ ava_1.default("loading config saves config", async (t) => {
        const codeQL = codeql_1.setCodeQL({
            async resolveQueries() {
                return {
-                    byLanguage: {},
+                    byLanguage: {
+                        javascript: { queries: ["query1.ql"] },
+                        python: { queries: ["query2.ql"] },
+                    },
                    noDeclaredLanguage: {},
                    multipleDeclaredLanguages: {},
                };
--- a/lib/config-utils.test.js.map
+++ b/lib/config-utils.test.js.map
--- a/lib/upload-lib.js
+++ b/lib/upload-lib.js
@@ -228,7 +228,8 @@ exports.buildPayload = buildPayload;
 // Uploads the given set of sarif files.
 // Returns true iff the upload occurred and succeeded
 async function uploadFiles(sarifFiles, repositoryNwo, commitOid, ref, analysisKey, category, analysisName, workflowRunID, checkoutPath, environment, gitHubVersion, apiDetails, mode, logger) {
-    logger.info(`Uploading sarif files: ${JSON.stringify(sarifFiles)}`);
+    logger.startGroup("Uploading results");
+    logger.info(`Processing sarif files: ${JSON.stringify(sarifFiles)}`);
    if (mode === "actions") {
        // This check only works on actions as env vars don't persist between calls to the runner
        const sentinelEnvVar = "CODEQL_UPLOAD_SARIF";
@@ -257,6 +258,7 @@ async function uploadFiles(sarifFiles, repositoryNwo, commitOid, ref, analysisKe
    logger.debug(`Number of results in upload: ${numResultInSarif}`);
    // Make the upload
    await uploadPayload(payload, repositoryNwo, apiDetails, mode, logger);
+    logger.endGroup();
    return {
        raw_upload_size_bytes: rawUploadSizeBytes,
        zipped_upload_size_bytes: zippedUploadSizeBytes,
--- a/lib/upload-lib.js.map
+++ b/lib/upload-lib.js.map
--- a/src/analyze.test.ts
+++ b/src/analyze.test.ts
@@ -76,6 +76,7 @@ test("status report fields and search path setting", async (t) => {
            })
          );
          searchPathsUsed.push(searchPath!);
+          return "";
        },
      });

--- a/src/analyze.ts
+++ b/src/analyze.ts
@@ -162,22 +162,27 @@ export async function runQueries(
    logger.startGroup(`Analyzing ${language}`);

    const queries = config.queries[language];
-    if (queries.builtin.length === 0 && queries.custom.length === 0) {
+    if (
+      queries === undefined ||
+      (queries.builtin.length === 0 && queries.custom.length === 0)
+    ) {
      throw new Error(
        `Unable to analyse ${language} as no queries were selected for this language`
      );
    }

    try {
+      let analysisSummary = "";
      if (queries["builtin"].length > 0) {
        const startTimeBuliltIn = new Date().getTime();
-        const sarifFile = await runQueryGroup(
+        const { sarifFile, stdout } = await runQueryGroup(
          language,
          "builtin",
          queries["builtin"],
          sarifFolder,
          undefined
        );
+        analysisSummary = stdout;
        await injectLinesOfCode(sarifFile, language, locPromise);

        statusReport[`analyze_builtin_queries_${language}_duration_ms`] =
@@ -188,7 +193,7 @@ export async function runQueries(
      const temporarySarifFiles: string[] = [];
      for (let i = 0; i < queries["custom"].length; ++i) {
        if (queries["custom"][i].queries.length > 0) {
-          const sarifFile = await runQueryGroup(
+          const { sarifFile } = await runQueryGroup(
            language,
            `custom-${i}`,
            queries["custom"][i].queries,
@@ -206,8 +211,13 @@ export async function runQueries(
        statusReport[`analyze_custom_queries_${language}_duration_ms`] =
          new Date().getTime() - startTimeCustom;
      }
+      logger.endGroup();

+      // Print the LoC baseline and the summary results from database analyze.
+      logger.startGroup(`Analysis summary for ${language}`);
      printLinesOfCodeSummary(logger, language, await locPromise);
+      logger.info(analysisSummary);
+      logger.endGroup();
    } catch (e) {
      logger.info(e);
      statusReport.analyze_failure_language = language;
@@ -226,7 +236,7 @@ export async function runQueries(
    queries: string[],
    destinationFolder: string,
    searchPath: string | undefined
-  ): Promise<string> {
+  ): Promise<{ sarifFile: string; stdout: string }> {
    const databasePath = util.getCodeQLDatabasePath(config, language);
    // Pass the queries to codeql using a file instead of using the command
    // line to avoid command line length restrictions, particularly on windows.
@@ -240,7 +250,7 @@ export async function runQueries(
    const sarifFile = path.join(destinationFolder, `${language}-${type}.sarif`);

    const codeql = getCodeQL(config.codeQLCmd);
-    await codeql.databaseAnalyze(
+    const databaseAnalyzeStdout = await codeql.databaseAnalyze(
      databasePath,
      sarifFile,
      searchPath,
@@ -254,9 +264,7 @@ export async function runQueries(
    logger.debug(
      `SARIF results for database ${language} created at "${sarifFile}"`
    );
-    logger.endGroup();
-
-    return sarifFile;
+    return { sarifFile, stdout: databaseAnalyzeStdout };
  }
 }

--- a/src/codeql.ts
+++ b/src/codeql.ts
@@ -96,7 +96,7 @@ export interface CodeQL {
    addSnippetsFlag: string,
    threadsFlag: string,
    automationDetailsId: string | undefined
-  ): Promise<void>;
+  ): Promise<string>;
 }

 export interface ResolveQueriesOutput {
@@ -666,7 +666,7 @@ function getCodeQLForCmd(cmd: string): CodeQL {
        ...getExtraOptionsFromEnv(["resolve", "queries"]),
      ];
      if (extraSearchPath !== undefined) {
-        codeqlArgs.push("--search-path", extraSearchPath);
+        codeqlArgs.push("--additional-packs", extraSearchPath);
      }
      let output = "";
      await new toolrunner.ToolRunner(cmd, codeqlArgs, {
@@ -688,7 +688,7 @@ function getCodeQLForCmd(cmd: string): CodeQL {
      addSnippetsFlag: string,
      threadsFlag: string,
      automationDetailsId: string | undefined
-    ) {
+    ): Promise<string> {
      const args = [
        "database",
        "analyze",
@@ -706,13 +706,22 @@ function getCodeQLForCmd(cmd: string): CodeQL {
        ...getExtraOptionsFromEnv(["database", "analyze"]),
      ];
      if (extraSearchPath !== undefined) {
-        args.push("--search-path", extraSearchPath);
+        args.push("--additional-packs", extraSearchPath);
      }
      if (automationDetailsId !== undefined) {
        args.push("--sarif-category", automationDetailsId);
      }
      args.push(querySuite);
-      await new toolrunner.ToolRunner(cmd, args).exec();
+      // capture stdout, which contains analysis summaries
+      let output = "";
+      await new toolrunner.ToolRunner(cmd, args, {
+        listeners: {
+          stdout: (data: Buffer) => {
+            output += data.toString("utf8");
+          },
+        },
+      }).exec();
+      return output;
    },
  };
 }
--- a/src/config-utils.test.ts
+++ b/src/config-utils.test.ts
@@ -68,7 +68,10 @@ test("load empty config", async (t) => {
    const codeQL = setCodeQL({
      async resolveQueries() {
        return {
-          byLanguage: {},
+          byLanguage: {
+            javascript: { queries: ["query1.ql"] },
+            python: { queries: ["query2.ql"] },
+          },
          noDeclaredLanguage: {},
          multipleDeclaredLanguages: {},
        };
@@ -116,7 +119,10 @@ test("loading config saves config", async (t) => {
    const codeQL = setCodeQL({
      async resolveQueries() {
        return {
-          byLanguage: {},
+          byLanguage: {
+            javascript: { queries: ["query1.ql"] },
+            python: { queries: ["query2.ql"] },
+          },
          noDeclaredLanguage: {},
          multipleDeclaredLanguages: {},
        };
--- a/src/config-utils.ts
+++ b/src/config-utils.ts
@@ -760,6 +760,12 @@ export async function getDefaultConfig(
    logger
  );
  const queries: Queries = {};
+  for (const language of languages) {
+    queries[language] = {
+      builtin: [],
+      custom: [],
+    };
+  }
  await addDefaultQueries(codeQL, languages, queries);
  if (queriesInput) {
    await addQueriesFromWorkflow(
@@ -834,6 +840,12 @@ async function loadConfig(
  );

  const queries: Queries = {};
+  for (const language of languages) {
+    queries[language] = {
+      builtin: [],
+      custom: [],
+    };
+  }
  const pathsIgnore: string[] = [];
  const paths: string[] = [];

@@ -925,21 +937,6 @@ async function loadConfig(
    }
  }

-  // The list of queries should not be empty for any language. If it is then
-  // it is a user configuration error.
-  for (const language of languages) {
-    if (
-      queries[language] === undefined ||
-      (queries[language].builtin.length === 0 &&
-        queries[language].custom.length === 0)
-    ) {
-      throw new Error(
-        `Did not detect any queries to run for ${language}. ` +
-          "Please make sure that the default queries are enabled, or you are specifying queries to run."
-      );
-    }
-  }
-
  return {
    languages,
    queries,
@@ -1016,6 +1013,21 @@ export async function initConfig(
    );
  }

+  // The list of queries should not be empty for any language. If it is then
+  // it is a user configuration error.
+  for (const language of config.languages) {
+    if (
+      config.queries[language] === undefined ||
+      (config.queries[language].builtin.length === 0 &&
+        config.queries[language].custom.length === 0)
+    ) {
+      throw new Error(
+        `Did not detect any queries to run for ${language}. ` +
+          "Please make sure that the default queries are enabled, or you are specifying queries to run."
+      );
+    }
+  }
+
  // Save the config so we can easily access it again in the future
  await saveConfig(config, logger);
  return config;
--- a/src/upload-lib.ts
+++ b/src/upload-lib.ts
@@ -342,7 +342,8 @@ async function uploadFiles(
  mode: util.Mode,
  logger: Logger
 ): Promise<UploadStatusReport> {
-  logger.info(`Uploading sarif files: ${JSON.stringify(sarifFiles)}`);
+  logger.startGroup("Uploading results");
+  logger.info(`Processing sarif files: ${JSON.stringify(sarifFiles)}`);

  if (mode === "actions") {
    // This check only works on actions as env vars don't persist between calls to the runner
@@ -403,6 +404,8 @@ async function uploadFiles(
  // Make the upload
  await uploadPayload(payload, repositoryNwo, apiDetails, mode, logger);

+  logger.endGroup();
+
  return {
    raw_upload_size_bytes: rawUploadSizeBytes,
    zipped_upload_size_bytes: zippedUploadSizeBytes,
Author	SHA1	Message	Date
Andrew Eisenberg	b7dd4a6f2c	Merge `800a951427` into `eeaefb6054`	2021-05-21 18:04:50 +00:00
Andrew Eisenberg	eeaefb6054	Merge pull request #517 from github/aibaars-additional-packs Use --additional-packs instead of --search-path	2021-05-21 09:31:17 -07:00
Andrew Eisenberg	c3e0f887ab	Merge branch 'main' into aibaars-additional-packs	2021-05-21 09:21:54 -07:00
Arthur Baars	1ad5a6c1be	Merge pull request #518 from github/aibaars-no-queries Ensure queries[language] objects are initialized	2021-05-21 12:54:23 +02:00
Arthur Baars	6a14accb41	Fix tests	2021-05-21 12:41:35 +02:00
Arthur Baars	84bec4d116	Check queries in initConfig	2021-05-21 12:23:00 +02:00
Arthur Baars	9aca271fbb	Remove superfluous if	2021-05-21 12:07:30 +02:00
Arthur Baars	ec011ddfdb	Ensure queries[language] objects are initialized	2021-05-21 10:42:14 +02:00
Arthur Baars	69cc3550c8	Use --additional-packs instead of --search-path	2021-05-21 10:40:06 +02:00
Aditya Sharad	8fa2ef886e	Merge pull request #515 from adityasharad/analyze/summary-logging Analyze: Improve log grouping and log the analysis summary in its own group	2021-05-20 15:49:26 -07:00
Aditya Sharad	f84cc5e6f1	Fix trailing newline in package.json As produced by npm ci && npm run removeNPMAbsolutePaths.	2021-05-20 14:12:37 -07:00
Aditya Sharad	512d4671bc	Decode CodeQL stdout as UTF-8	2021-05-20 14:08:29 -07:00
Aditya Sharad	741bd73e8e	Changelog: Add note for grouping of analysis logs	2021-05-20 12:32:10 -07:00
Aditya Sharad	031ec8753a	Group log messages while uploading results	2021-05-20 12:32:10 -07:00
Aditya Sharad	19fe63f821	Analyze: Log the analysis summary in its own group Fix grouping of the analysis logs, so that custom query logs also get grouped. Capture the stdout of codeql database analyze, which contains the analysis summary from summary and diagnostic queries. Log this output in its own group, along with the baseline computed in the Action.	2021-05-20 12:32:09 -07:00
Andrew Eisenberg	6a9815718a	Merge pull request #514 from github/aeisenberg/first-changelog-entry Add the first changelog entry	2021-05-20 10:47:51 -07:00
Andrew Eisenberg	7ae8d165a6	Add the first changelog entry (Adding the changelog itself.)	2021-05-20 09:24:30 -07:00