Handle errors during database upload

Upload CodeQL databases

.github/update-release-branch.py

@@ -122,7 +122,7 @@ def get_commit_difference(repo):
 
 # Is the given commit the automatic merge commit from when merging a PR
 def is_pr_merge_commit(commit):
-  return commit.committer.login == 'web-flow' and len(commit.parents) > 1
+  return commit.committer is not None and commit.committer.login == 'web-flow' and len(commit.parents) > 1
 
 # Gets a copy of the commit message that should display nicely
 def get_truncated_commit_message(commit):

.github/workflows/post-release-mergeback.yml

@@ -16,10 +16,6 @@ on:
     branches:
       - v1
 
-  pull_request:
-    paths:
-      - .github/workflows/post-release-mergeback.yml
-
 jobs:
   merge-back:
     runs-on: ubuntu-latest

.github/workflows/pr-checks.yml

@@ -100,6 +100,49 @@ jobs:
           exit 1
         fi
 
+  # Packaging test that runs against a javascript database
+  test-packaging-javascript:
+    needs: [check-js, check-node-modules]
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Move codeql-action
+      shell: bash
+      run: |
+        mkdir ../action
+        mv * .github ../action/
+        mv ../action/tests/multi-language-repo/{*,.github} .
+        mv ../action/.github/workflows .github
+    - uses: ./../action/init
+      with:
+        config-file: ".github/codeql/codeql-config-packaging.yml"
+        languages: javascript
+        # TODO: this can be removed when cli v2.5.6 is released and available in the tool cache
+        tools: https://github.com/dsp-testing/aeisenberg-codeql-action-packaging/releases/download/codeql-bundle-20210615/codeql-bundle-linux64.tar.gz
+
+    - name: Build code
+      shell: bash
+      run: ./build.sh
+    - uses: ./../action/analyze
+      with:
+        output: "${{ runner.temp }}/results"
+      env:
+        TEST_MODE: true
+    - name: Assert Results
+      run: |
+        cd "$RUNNER_TEMP/results"
+        # We should have 3 hits from these rules
+        EXPECTED_RULES="javascript/example/empty-or-one-block javascript/example/empty-or-one-block javascript/example/two-block"
+
+        # use tr to replace newlines with spaces and xargs to trim leading and trailing whitespace
+        RULES="$(cat javascript.sarif | jq -r '.runs[0].results[].ruleId' | sort | tr "\n" " " | xargs)"
+        echo "Found matching rules '$RULES'"
+        if [ "$RULES" != "$EXPECTED_RULES" ]; then
+          echo "Did not match expected rules '$EXPECTED_RULES'."
+          exit 1
+        fi
+
   # Identify the CodeQL tool versions to integration test against.
   check-codeql-versions:
     needs: [check-js, check-node-modules]

CHANGELOG.md

@@ -1,5 +1,7 @@
 # CodeQL Action and CodeQL Runner Changelog
 
+## [UNRELEASED]
+
 ## 1.0.1 - 07 Jun 2021
 
 - Pass the `--sarif-group-rules-by-pack` argument to CodeQL CLI invocations that generate SARIF. This means the SARIF rule object for each query will now be found underneath its corresponding query pack in `runs[].tool.extensions`. [#546](https://github.com/github/codeql-action/pull/546)

CODEOWNERS

@@ -0,0 +1 @@
+**/* @github/codeql-action-reviewers

analyze/action.yml

@@ -34,6 +34,10 @@ inputs:
   category:
     description: String used by Code Scanning for matching the analyses
     required: false
+  upload-database:
+    description: Whether to upload the resulting CodeQL database
+    required: false
+    default: "true"
   token:
     default: ${{ github.token }}
   matrix:

lib/actions-util.js

@@ -448,10 +448,6 @@ async function createStatusReportBase(actionName, status, actionStartedAt, cause
     return statusReport;
 }
 exports.createStatusReportBase = createStatusReportBase;
-function isHTTPError(arg) {
-    var _a;
-    return ((_a = arg) === null || _a === void 0 ? void 0 : _a.status) !== undefined && Number.isInteger(arg.status);
-}
 const GENERIC_403_MSG = "The repo on which this action is running is not opted-in to CodeQL code scanning.";
 const GENERIC_404_MSG = "Not authorized to used the CodeQL code scanning feature on this repo.";
 const OUT_OF_DATE_MSG = "CodeQL Action is out-of-date. Please upgrade to the latest version of codeql-action.";
@@ -481,7 +477,7 @@ async function sendStatusReport(statusReport) {
     }
     catch (e) {
         console.log(e);
-        if (isHTTPError(e)) {
+        if (util_1.isHTTPError(e)) {
             switch (e.status) {
                 case 403:
                     if (workflowIsTriggeredByPushEvent() && isDependabotActor()) {
@@ -540,4 +536,28 @@ function getRelativeScriptPath() {
     return path.relative(actionsDirectory, __filename);
 }
 exports.getRelativeScriptPath = getRelativeScriptPath;
+// Reads the contents of GITHUB_EVENT_PATH as a JSON object
+function getWorkflowEvent() {
+    const eventJsonFile = util_1.getRequiredEnvParam("GITHUB_EVENT_PATH");
+    try {
+        return JSON.parse(fs.readFileSync(eventJsonFile, "utf-8"));
+    }
+    catch (e) {
+        throw new Error(`Unable to read workflow event JSON from ${eventJsonFile}: ${e}`);
+    }
+}
+// Is the version of the repository we are currently analyzing from the default branch,
+// or alternatively from another branch or a pull request.
+async function isAnalyzingDefaultBranch() {
+    var _a, _b;
+    // Get the current ref and trim and refs/heads/ prefix
+    let currentRef = await getRef();
+    currentRef = currentRef.startsWith("refs/heads/")
+        ? currentRef.substr("refs/heads/".length)
+        : currentRef;
+    const event = getWorkflowEvent();
+    const defaultBranch = (_b = (_a = event) === null || _a === void 0 ? void 0 : _a.repository) === null || _b === void 0 ? void 0 : _b.default_branch;
+    return currentRef === defaultBranch;
+}
+exports.isAnalyzingDefaultBranch = isAnalyzingDefaultBranch;
 //# sourceMappingURL=actions-util.js.map

lib/analysis-paths.test.js

@@ -29,6 +29,7 @@ ava_1.default("emptyPaths", async (t) => {
             codeQLCmd: "",
             gitHubVersion: { type: util.GitHubVariant.DOTCOM },
             dbLocation: path.resolve(tmpDir, "codeql_databases"),
+            packs: {},
         };
         analysisPaths.includeAndExcludeAnalysisPaths(config);
         t.is(process.env["LGTM_INDEX_INCLUDE"], undefined);
@@ -49,6 +50,7 @@ ava_1.default("nonEmptyPaths", async (t) => {
             codeQLCmd: "",
             gitHubVersion: { type: util.GitHubVariant.DOTCOM },
             dbLocation: path.resolve(tmpDir, "codeql_databases"),
+            packs: {},
         };
         analysisPaths.includeAndExcludeAnalysisPaths(config);
         t.is(process.env["LGTM_INDEX_INCLUDE"], "path1\npath2");
@@ -70,6 +72,7 @@ ava_1.default("exclude temp dir", async (t) => {
             codeQLCmd: "",
             gitHubVersion: { type: util.GitHubVariant.DOTCOM },
             dbLocation: path.resolve(tempDir, "codeql_databases"),
+            packs: {},
         };
         analysisPaths.includeAndExcludeAnalysisPaths(config);
         t.is(process.env["LGTM_INDEX_INCLUDE"], undefined);

lib/analysis-paths.test.js.map

@@ -1 +1 @@
-{"version":3,"file":"analysis-paths.test.js","sourceRoot":"","sources":["../src/analysis-paths.test.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,2CAA6B;AAE7B,8CAAuB;AAEvB,gEAAkD;AAClD,mDAA6C;AAC7C,6CAA+B;AAE/B,0BAAU,CAAC,aAAI,CAAC,CAAC;AAEjB,aAAI,CAAC,YAAY,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IAC7B,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,MAAM,GAAG;YACb,SAAS,EAAE,EAAE;YACb,OAAO,EAAE,EAAE;YACX,WAAW,EAAE,EAAE;YACf,KAAK,EAAE,EAAE;YACT,iBAAiB,EAAE,EAAE;YACrB,OAAO,EAAE,MAAM;YACf,YAAY,EAAE,MAAM;YACpB,SAAS,EAAE,EAAE;YACb,aAAa,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM,EAAwB;YACxE,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,kBAAkB,CAAC;SACrD,CAAC;QACF,aAAa,CAAC,8BAA8B,CAAC,MAAM,CAAC,CAAC;QACrD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,SAAS,CAAC,CAAC;QACnD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,SAAS,CAAC,CAAC;QACnD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,SAAS,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,aAAI,CAAC,eAAe,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IAChC,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,MAAM,GAAG;YACb,SAAS,EAAE,EAAE;YACb,OAAO,EAAE,EAAE;YACX,KAAK,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,UAAU,CAAC;YACrC,WAAW,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,UAAU,CAAC;YAC3C,iBAAiB,EAAE,EAAE;YACrB,OAAO,EAAE,MAAM;YACf,YAAY,EAAE,MAAM;YACpB,SAAS,EAAE,EAAE;YACb,aAAa,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM,EAAwB;YACxE,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,kBAAkB,CAAC;SACrD,CAAC;QACF,aAAa,CAAC,8BAA8B,CAAC,MAAM,CAAC,CAAC;QACrD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,cAAc,CAAC,CAAC;QACxD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,cAAc,CAAC,CAAC;QACxD,CAAC,CAAC,EAAE,CACF,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EACjC,gGAAgG,CACjG,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,aAAI,CAAC,kBAAkB,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IACnC,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,YAAY,EAAE,EAAE;QAClD,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,oBAAoB,CAAC,CAAC;QAC/D,MAAM,MAAM,GAAG;YACb,SAAS,EAAE,EAAE;YACb,OAAO,EAAE,EAAE;YACX,WAAW,EAAE,EAAE;YACf,KAAK,EAAE,EAAE;YACT,iBAAiB,EAAE,EAAE;YACrB,OAAO;YACP,YAAY;YACZ,SAAS,EAAE,EAAE;YACb,aAAa,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM,EAAwB;YACxE,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,kBAAkB,CAAC;SACtD,CAAC;QACF,aAAa,CAAC,8BAA8B,CAAC,MAAM,CAAC,CAAC;QACrD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,SAAS,CAAC,CAAC;QACnD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,oBAAoB,CAAC,CAAC;QAC9D,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,SAAS,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
+{"version":3,"file":"analysis-paths.test.js","sourceRoot":"","sources":["../src/analysis-paths.test.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,2CAA6B;AAE7B,8CAAuB;AAEvB,gEAAkD;AAClD,mDAA6C;AAC7C,6CAA+B;AAE/B,0BAAU,CAAC,aAAI,CAAC,CAAC;AAEjB,aAAI,CAAC,YAAY,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IAC7B,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,MAAM,GAAG;YACb,SAAS,EAAE,EAAE;YACb,OAAO,EAAE,EAAE;YACX,WAAW,EAAE,EAAE;YACf,KAAK,EAAE,EAAE;YACT,iBAAiB,EAAE,EAAE;YACrB,OAAO,EAAE,MAAM;YACf,YAAY,EAAE,MAAM;YACpB,SAAS,EAAE,EAAE;YACb,aAAa,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM,EAAwB;YACxE,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,kBAAkB,CAAC;YACpD,KAAK,EAAE,EAAE;SACV,CAAC;QACF,aAAa,CAAC,8BAA8B,CAAC,MAAM,CAAC,CAAC;QACrD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,SAAS,CAAC,CAAC;QACnD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,SAAS,CAAC,CAAC;QACnD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,SAAS,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,aAAI,CAAC,eAAe,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IAChC,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,MAAM,GAAG;YACb,SAAS,EAAE,EAAE;YACb,OAAO,EAAE,EAAE;YACX,KAAK,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,UAAU,CAAC;YACrC,WAAW,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,UAAU,CAAC;YAC3C,iBAAiB,EAAE,EAAE;YACrB,OAAO,EAAE,MAAM;YACf,YAAY,EAAE,MAAM;YACpB,SAAS,EAAE,EAAE;YACb,aAAa,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM,EAAwB;YACxE,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,kBAAkB,CAAC;YACpD,KAAK,EAAE,EAAE;SACV,CAAC;QACF,aAAa,CAAC,8BAA8B,CAAC,MAAM,CAAC,CAAC;QACrD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,cAAc,CAAC,CAAC;QACxD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,cAAc,CAAC,CAAC;QACxD,CAAC,CAAC,EAAE,CACF,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EACjC,gGAAgG,CACjG,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,aAAI,CAAC,kBAAkB,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IACnC,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,YAAY,EAAE,EAAE;QAClD,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,oBAAoB,CAAC,CAAC;QAC/D,MAAM,MAAM,GAAG;YACb,SAAS,EAAE,EAAE;YACb,OAAO,EAAE,EAAE;YACX,WAAW,EAAE,EAAE;YACf,KAAK,EAAE,EAAE;YACT,iBAAiB,EAAE,EAAE;YACrB,OAAO;YACP,YAAY;YACZ,SAAS,EAAE,EAAE;YACb,aAAa,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM,EAAwB;YACxE,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,kBAAkB,CAAC;YACrD,KAAK,EAAE,EAAE;SACV,CAAC;QACF,aAAa,CAAC,8BAA8B,CAAC,MAAM,CAAC,CAAC;QACrD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,SAAS,CAAC,CAAC;QACnD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,oBAAoB,CAAC,CAAC;QAC9D,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,SAAS,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

lib/analyze-action.js

@@ -12,8 +12,11 @@ const path = __importStar(require("path"));
 const core = __importStar(require("@actions/core"));
 const actionsUtil = __importStar(require("./actions-util"));
 const analyze_1 = require("./analyze");
+const api_client_1 = require("./api-client");
+const codeql_1 = require("./codeql");
 const config_utils_1 = require("./config-utils");
 const logging_1 = require("./logging");
+const repository_1 = require("./repository");
 const upload_lib = __importStar(require("./upload-lib"));
 const util = __importStar(require("./util"));
 // eslint-disable-next-line import/no-commonjs
@@ -30,6 +33,63 @@ async function sendStatusReport(startedAt, stats, error) {
     };
     await actionsUtil.sendStatusReport(statusReport);
 }
+async function uploadDatabases(repositoryNwo, config, apiDetails, logger) {
+    if (actionsUtil.getRequiredInput("upload-database") !== "true") {
+        logger.debug("Database upload disabled in workflow. Skipping upload.");
+        return;
+    }
+    // Do nothing when not running against github.com
+    if (config.gitHubVersion.type !== util.GitHubVariant.DOTCOM) {
+        logger.debug("Not running against github.com. Skipping upload.");
+        return;
+    }
+    if (!(await actionsUtil.isAnalyzingDefaultBranch())) {
+        // We only want to upload a database if we are analyzing the default branch.
+        logger.debug("Not analyzing default branch. Skipping upload.");
+        return;
+    }
+    const client = api_client_1.getApiClient(apiDetails);
+    try {
+        await client.request("GET /repos/:owner/:repo/code-scanning/databases", {
+            owner: repositoryNwo.owner,
+            repo: repositoryNwo.repo,
+        });
+    }
+    catch (e) {
+        console.log(e);
+        if (util.isHTTPError(e)) {
+            if (e.status === 404) {
+                logger.debug("Repository is not opted in to database uploads. Skipping upload.");
+            }
+            else {
+                logger.debug(`Skipping database upload due to unknown error: ${e}`);
+            }
+            return;
+        }
+    }
+    const codeql = codeql_1.getCodeQL(config.codeQLCmd);
+    for (const language of config.languages) {
+        // Bundle the database up into a single zip file
+        const databasePath = util.getCodeQLDatabasePath(config, language);
+        const databaseBundlePath = `${databasePath}.zip`;
+        await codeql.databaseBundle(databasePath, databaseBundlePath);
+        // Upload the database bundle
+        const payload = fs.readFileSync(databaseBundlePath);
+        try {
+            await client.request(`PUT /repos/:owner/:repo/code-scanning/databases/${language}`, {
+                owner: repositoryNwo.owner,
+                repo: repositoryNwo.repo,
+                data: payload,
+            });
+            logger.debug(`Successfully uploaded database for ${language}`);
+        }
+        catch (e) {
+            console.log(e);
+            // Log a warning but don't fail the workflow
+            logger.warning(`Failed to upload database for ${language}: ${e}`);
+        }
+    }
+}
 async function run() {
     const startedAt = new Date();
     let stats = undefined;
@@ -66,6 +126,8 @@ async function run() {
             logger.info("Not uploading results");
             stats = { ...queriesStats };
         }
+        const repositoryNwo = repository_1.parseRepositoryNwo(util.getRequiredEnvParam("GITHUB_REPOSITORY"));
+        await uploadDatabases(repositoryNwo, config, apiDetails, logger);
     }
     catch (error) {
         core.setFailed(error.message);

lib/analyze.js

@@ -15,7 +15,6 @@ const codeql_1 = require("./codeql");
 const count_loc_1 = require("./count-loc");
 const languages_1 = require("./languages");
 const sharedEnv = __importStar(require("./shared-environment"));
-const upload_lib_1 = require("./upload-lib");
 const util = __importStar(require("./util"));
 class CodeQLAnalysisError extends Error {
     constructor(queriesStatusReport, message) {
@@ -77,6 +76,7 @@ async function finalizeDatabaseCreation(config, threadsFlag, logger) {
 }
 // Runs queries and creates sarif files in the given folder
 async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag, automationDetailsId, config, logger) {
+    var _a, _b;
     const statusReport = {};
     // count the number of lines in the background
     const locPromise = count_loc_1.countLoc(path.resolve(), 
@@ -85,103 +85,121 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
     // that here.
     config.paths, config.pathsIgnore, config.languages, logger);
     for (const language of config.languages) {
-        logger.startGroup(`Analyzing ${language}`);
         const queries = config.queries[language];
-        if (queries === undefined ||
-            (queries.builtin.length === 0 && queries.custom.length === 0)) {
+        const packsWithVersion = config.packs[language] || [];
+        const hasBuiltinQueries = ((_a = queries) === null || _a === void 0 ? void 0 : _a.builtin.length) > 0;
+        const hasCustomQueries = ((_b = queries) === null || _b === void 0 ? void 0 : _b.custom.length) > 0;
+        const hasPackWithCustomQueries = packsWithVersion.length > 0;
+        if (!hasBuiltinQueries && !hasCustomQueries && !hasPackWithCustomQueries) {
             throw new Error(`Unable to analyse ${language} as no queries were selected for this language`);
         }
         try {
-            let analysisSummaryBuiltIn = "";
-            const customAnalysisSummaries = [];
+            if (hasPackWithCustomQueries) {
+                logger.info("*************");
+                logger.info("Performing analysis with custom QL Packs. QL Packs are an experimental feature.");
+                logger.info("And should not be used in production yet.");
+                logger.info("*************");
+                logger.startGroup(`Downloading custom packs for ${language}`);
+                const codeql = codeql_1.getCodeQL(config.codeQLCmd);
+                const results = await codeql.packDownload(packsWithVersion);
+                logger.info(`Downloaded packs: ${results.packs
+                    .map((r) => `${r.name}@${r.version || "latest"}`)
+                    .join(", ")}`);
+                logger.endGroup();
+            }
+            logger.startGroup(`Running queries for ${language}`);
+            const querySuitePaths = [];
             if (queries["builtin"].length > 0) {
                 const startTimeBuiltIn = new Date().getTime();
-                const { sarifFile, stdout } = await runQueryGroup(language, "builtin", queries["builtin"], sarifFolder, undefined);
-                analysisSummaryBuiltIn = stdout;
-                await injectLinesOfCode(sarifFile, language, locPromise);
+                querySuitePaths.push(await runQueryGroup(language, "builtin", createQuerySuiteContents(queries["builtin"]), undefined));
                 statusReport[`analyze_builtin_queries_${language}_duration_ms`] =
                     new Date().getTime() - startTimeBuiltIn;
             }
             const startTimeCustom = new Date().getTime();
-            const temporarySarifDir = config.tempDir;
-            const temporarySarifFiles = [];
+            let ranCustom = false;
             for (let i = 0; i < queries["custom"].length; ++i) {
                 if (queries["custom"][i].queries.length > 0) {
-                    const { sarifFile, stdout } = await runQueryGroup(language, `custom-${i}`, queries["custom"][i].queries, temporarySarifDir, queries["custom"][i].searchPath);
-                    customAnalysisSummaries.push(stdout);
-                    temporarySarifFiles.push(sarifFile);
+                    querySuitePaths.push(await runQueryGroup(language, `custom-${i}`, createQuerySuiteContents(queries["custom"][i].queries), queries["custom"][i].searchPath));
+                    ranCustom = true;
                 }
             }
-            if (temporarySarifFiles.length > 0) {
-                const sarifFile = path.join(sarifFolder, `${language}-custom.sarif`);
-                fs.writeFileSync(sarifFile, upload_lib_1.combineSarifFiles(temporarySarifFiles));
-                await injectLinesOfCode(sarifFile, language, locPromise);
+            if (packsWithVersion.length > 0) {
+                querySuitePaths.push(await runQueryGroup(language, "packs", createPackSuiteContents(packsWithVersion), undefined));
+                ranCustom = true;
+            }
+            if (ranCustom) {
                 statusReport[`analyze_custom_queries_${language}_duration_ms`] =
                     new Date().getTime() - startTimeCustom;
             }
             logger.endGroup();
-            // Print the LoC baseline and the summary results from database analyze for the standard
-            // query suite and (if appropriate) each custom query suite.
-            logger.startGroup(`Analysis summary for ${language}`);
-            printLinesOfCodeSummary(logger, language, await locPromise);
-            logger.info(analysisSummaryBuiltIn);
-            for (const [i, customSummary] of customAnalysisSummaries.entries()) {
-                if (customSummary.trim() === "") {
-                    continue;
-                }
-                const description = customAnalysisSummaries.length === 1
-                    ? "custom queries"
-                    : `custom query suite ${i + 1}/${customAnalysisSummaries.length}`;
-                logger.info(`Analysis summary for ${description}:`);
-                logger.info("");
-                logger.info(customSummary);
-                logger.info("");
-            }
+            logger.startGroup(`Interpreting results for ${language}`);
+            const startTimeInterpretResults = new Date().getTime();
+            const sarifFile = path.join(sarifFolder, `${language}.sarif`);
+            const analysisSummary = await runInterpretResults(language, querySuitePaths, sarifFile);
+            await injectLinesOfCode(sarifFile, language, locPromise);
+            statusReport[`interpret_results_${language}_duration_ms`] =
+                new Date().getTime() - startTimeInterpretResults;
             logger.endGroup();
+            logger.info(analysisSummary);
+            printLinesOfCodeSummary(logger, language, await locPromise);
         }
         catch (e) {
             logger.info(e);
+            logger.info(e.stack);
             statusReport.analyze_failure_language = language;
             throw new CodeQLAnalysisError(statusReport, `Error running analysis for ${language}: ${e}`);
         }
     }
     return statusReport;
-    async function runQueryGroup(language, type, queries, destinationFolder, searchPath) {
+    async function runInterpretResults(language, queries, sarifFile) {
+        const databasePath = util.getCodeQLDatabasePath(config, language);
+        const codeql = codeql_1.getCodeQL(config.codeQLCmd);
+        return await codeql.databaseInterpretResults(databasePath, queries, sarifFile, addSnippetsFlag, threadsFlag, automationDetailsId);
+    }
+    async function runQueryGroup(language, type, querySuiteContents, searchPath) {
         const databasePath = util.getCodeQLDatabasePath(config, language);
         // Pass the queries to codeql using a file instead of using the command
         // line to avoid command line length restrictions, particularly on windows.
         const querySuitePath = `${databasePath}-queries-${type}.qls`;
-        const querySuiteContents = queries
-            .map((q) => `- query: ${q}`)
-            .join("\n");
         fs.writeFileSync(querySuitePath, querySuiteContents);
-        logger.debug(`Query suite file for ${language}...\n${querySuiteContents}`);
-        const sarifFile = path.join(destinationFolder, `${language}-${type}.sarif`);
+        logger.debug(`Query suite file for ${language}-${type}...\n${querySuiteContents}`);
         const codeql = codeql_1.getCodeQL(config.codeQLCmd);
-        const databaseAnalyzeStdout = await codeql.databaseAnalyze(databasePath, sarifFile, searchPath, querySuitePath, memoryFlag, addSnippetsFlag, threadsFlag, automationDetailsId);
-        logger.debug(`SARIF results for database ${language} created at "${sarifFile}"`);
-        return { sarifFile, stdout: databaseAnalyzeStdout };
+        await codeql.databaseRunQueries(databasePath, searchPath, querySuitePath, memoryFlag, threadsFlag);
+        logger.debug(`BQRS results produced for ${language} (queries: ${type})"`);
+        return querySuitePath;
     }
 }
 exports.runQueries = runQueries;
+function createQuerySuiteContents(queries) {
+    return queries.map((q) => `- query: ${q}`).join("\n");
+}
+function createPackSuiteContents(packsWithVersion) {
+    return packsWithVersion.map(packWithVersionToQuerySuiteEntry).join("\n");
+}
+function packWithVersionToQuerySuiteEntry(pack) {
+    let text = `- qlpack: ${pack.packName}`;
+    if (pack.version) {
+        text += `\n  version: ${pack.version}`;
+    }
+    return text;
+}
 async function runAnalyze(outputDir, memoryFlag, addSnippetsFlag, threadsFlag, automationDetailsId, config, logger) {
     // Delete the tracer config env var to avoid tracing ourselves
     delete process.env[sharedEnv.ODASA_TRACER_CONFIGURATION];
     fs.mkdirSync(outputDir, { recursive: true });
-    logger.info("Finalizing database creation");
     await finalizeDatabaseCreation(config, threadsFlag, logger);
-    logger.info("Analyzing database");
     const queriesStats = await runQueries(outputDir, memoryFlag, addSnippetsFlag, threadsFlag, automationDetailsId, config, logger);
     return { ...queriesStats };
 }
 exports.runAnalyze = runAnalyze;
 async function runCleanup(config, cleanupLevel, logger) {
-    logger.info("Cleaning up databases...");
+    logger.startGroup("Cleaning up databases");
     for (const language of config.languages) {
         const codeql = codeql_1.getCodeQL(config.codeQLCmd);
         const databasePath = util.getCodeQLDatabasePath(config, language);
         await codeql.databaseCleanup(databasePath, cleanupLevel);
     }
+    logger.endGroup();
 }
 exports.runCleanup = runCleanup;
 async function injectLinesOfCode(sarifFile, language, locPromise) {
@@ -208,7 +226,7 @@ async function injectLinesOfCode(sarifFile, language, locPromise) {
 }
 function printLinesOfCodeSummary(logger, language, lineCounts) {
     if (language in lineCounts) {
-        logger.info(`Counted ${lineCounts[language]} lines of code for ${language} as a baseline.`);
+        logger.info(`Counted a baseline of ${lineCounts[language]} lines of code for ${language}.`);
     }
 }
 //# sourceMappingURL=analyze.js.map

lib/analyze.test.js

@@ -13,6 +13,8 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
 const ava_1 = __importDefault(require("ava"));
+const yaml = __importStar(require("js-yaml"));
+const semver_1 = require("semver");
 const sinon_1 = __importDefault(require("sinon"));
 const analyze_1 = require("./analyze");
 const codeql_1 = require("./codeql");
@@ -39,9 +41,27 @@ ava_1.default("status report fields and search path setting", async (t) => {
         const memoryFlag = "";
         const addSnippetsFlag = "";
         const threadsFlag = "";
+        const packs = {
+            [languages_1.Language.cpp]: [
+                {
+                    packName: "a/b",
+                    version: semver_1.clean("1.0.0"),
+                },
+            ],
+            [languages_1.Language.java]: [
+                {
+                    packName: "c/d",
+                    version: semver_1.clean("2.0.0"),
+                },
+            ],
+        };
         for (const language of Object.values(languages_1.Language)) {
             codeql_1.setCodeQL({
-                databaseAnalyze: async (_, sarifFile, searchPath) => {
+                packDownload: async () => ({ packs: [] }),
+                databaseRunQueries: async (_db, searchPath) => {
+                    searchPathsUsed.push(searchPath);
+                },
+                databaseInterpretResults: async (_db, _queriesRun, sarifFile) => {
                     fs.writeFileSync(sarifFile, JSON.stringify({
                         runs: [
                             // variant 1 uses ruleId
@@ -71,7 +91,6 @@ ava_1.default("status report fields and search path setting", async (t) => {
                             {},
                         ],
                     }));
-                    searchPathsUsed.push(searchPath);
                     return "";
                 },
             });
@@ -89,6 +108,7 @@ ava_1.default("status report fields and search path setting", async (t) => {
                     type: util.GitHubVariant.DOTCOM,
                 },
                 dbLocation: path.resolve(tmpDir, "codeql_databases"),
+                packs,
             };
             fs.mkdirSync(util.getCodeQLDatabasePath(config, language), {
                 recursive: true,
@@ -98,8 +118,18 @@ ava_1.default("status report fields and search path setting", async (t) => {
                 custom: [],
             };
             const builtinStatusReport = await analyze_1.runQueries(tmpDir, memoryFlag, addSnippetsFlag, threadsFlag, undefined, config, logging_1.getRunnerLogger(true));
-            t.deepEqual(Object.keys(builtinStatusReport).length, 1);
-            t.true(`analyze_builtin_queries_${language}_duration_ms` in builtinStatusReport);
+            const hasPacks = language in packs;
+            const statusReportKeys = Object.keys(builtinStatusReport).sort();
+            if (hasPacks) {
+                t.deepEqual(statusReportKeys.length, 3, statusReportKeys.toString());
+                t.deepEqual(statusReportKeys[0], `analyze_builtin_queries_${language}_duration_ms`);
+                t.deepEqual(statusReportKeys[1], `analyze_custom_queries_${language}_duration_ms`);
+                t.deepEqual(statusReportKeys[2], `interpret_results_${language}_duration_ms`);
+            }
+            else {
+                t.deepEqual(statusReportKeys[0], `analyze_builtin_queries_${language}_duration_ms`);
+                t.deepEqual(statusReportKeys[1], `interpret_results_${language}_duration_ms`);
+            }
             config.queries[language] = {
                 builtin: [],
                 custom: [
@@ -114,17 +144,21 @@ ava_1.default("status report fields and search path setting", async (t) => {
                 ],
             };
             const customStatusReport = await analyze_1.runQueries(tmpDir, memoryFlag, addSnippetsFlag, threadsFlag, undefined, config, logging_1.getRunnerLogger(true));
-            t.deepEqual(Object.keys(customStatusReport).length, 1);
+            t.deepEqual(Object.keys(customStatusReport).length, 2);
             t.true(`analyze_custom_queries_${language}_duration_ms` in customStatusReport);
-            t.deepEqual(searchPathsUsed, [undefined, "/1", "/2"]);
+            const expectedSearchPathsUsed = hasPacks
+                ? [undefined, undefined, "/1", "/2", undefined]
+                : [undefined, "/1", "/2"];
+            t.deepEqual(searchPathsUsed, expectedSearchPathsUsed);
+            t.true(`interpret_results_${language}_duration_ms` in customStatusReport);
         }
         verifyLineCounts(tmpDir);
+        verifyQuerySuites(tmpDir);
     });
     function verifyLineCounts(tmpDir) {
         // eslint-disable-next-line github/array-foreach
         Object.keys(languages_1.Language).forEach((lang, i) => {
-            verifyLineCountForFile(lang, path.join(tmpDir, `${lang}-builtin.sarif`), i + 1);
-            verifyLineCountForFile(lang, path.join(tmpDir, `${lang}-custom.sarif`), i + 1);
+            verifyLineCountForFile(lang, path.join(tmpDir, `${lang}.sarif`), i + 1);
         });
     }
     function verifyLineCountForFile(lang, filePath, lineCount) {
@@ -146,8 +180,52 @@ ava_1.default("status report fields and search path setting", async (t) => {
                 baseline: lineCount,
             },
         ]);
-        // when the rule doesn't exists, it should not be added
+        // when the rule doesn't exist, it should not be added
         t.deepEqual(sarif.runs[2].properties.metricResults, []);
     }
+    function verifyQuerySuites(tmpDir) {
+        const qlsContent = [
+            {
+                query: "foo.ql",
+            },
+        ];
+        const qlsContent2 = [
+            {
+                query: "bar.ql",
+            },
+        ];
+        const qlsPackContentCpp = [
+            {
+                qlpack: "a/b",
+                version: "1.0.0",
+            },
+        ];
+        const qlsPackContentJava = [
+            {
+                qlpack: "c/d",
+                version: "2.0.0",
+            },
+        ];
+        for (const lang of Object.values(languages_1.Language)) {
+            t.deepEqual(readContents(`${lang}-queries-builtin.qls`), qlsContent);
+            t.deepEqual(readContents(`${lang}-queries-custom-0.qls`), qlsContent);
+            t.deepEqual(readContents(`${lang}-queries-custom-1.qls`), qlsContent2);
+            const packSuiteName = `${lang}-queries-packs.qls`;
+            if (lang === languages_1.Language.cpp) {
+                t.deepEqual(readContents(packSuiteName), qlsPackContentCpp);
+            }
+            else if (lang === languages_1.Language.java) {
+                t.deepEqual(readContents(packSuiteName), qlsPackContentJava);
+            }
+            else {
+                t.false(fs.existsSync(path.join(tmpDir, "codeql_databases", packSuiteName)));
+            }
+        }
+        function readContents(name) {
+            const x = fs.readFileSync(path.join(tmpDir, "codeql_databases", name), "utf8");
+            console.log(x);
+            return yaml.safeLoad(fs.readFileSync(path.join(tmpDir, "codeql_databases", name), "utf8"));
+        }
+    }
 });
 //# sourceMappingURL=analyze.test.js.map

lib/codeql.js

@@ -285,8 +285,11 @@ function setCodeQL(partialCodeql) {
         finalizeDatabase: resolveFunction(partialCodeql, "finalizeDatabase"),
         resolveLanguages: resolveFunction(partialCodeql, "resolveLanguages"),
         resolveQueries: resolveFunction(partialCodeql, "resolveQueries"),
-        databaseAnalyze: resolveFunction(partialCodeql, "databaseAnalyze"),
+        packDownload: resolveFunction(partialCodeql, "packDownload"),
         databaseCleanup: resolveFunction(partialCodeql, "databaseCleanup"),
+        databaseBundle: resolveFunction(partialCodeql, "databaseBundle"),
+        databaseRunQueries: resolveFunction(partialCodeql, "databaseRunQueries"),
+        databaseInterpretResults: resolveFunction(partialCodeql, "databaseInterpretResults"),
     };
     return cachedCodeQL;
 }
@@ -311,7 +314,7 @@ function getCodeQLForCmd(cmd) {
             return cmd;
         },
         async printVersion() {
-            await new toolrunner.ToolRunner(cmd, ["version", "--format=json"]).exec();
+            await runTool(cmd, ["version", "--format=json"]);
         },
         async getTracerEnv(databasePath) {
             // Write tracer-env.js to a temp location.
@@ -342,7 +345,7 @@ function getCodeQLForCmd(cmd) {
             // action/runner has been implemented in `codeql database trace-command`
             // _and_ is present in the latest supported CLI release.)
             const envFile = path.resolve(databasePath, "working", "env.tmp");
-            await new toolrunner.ToolRunner(cmd, [
+            await runTool(cmd, [
                 "database",
                 "trace-command",
                 databasePath,
@@ -350,18 +353,18 @@ function getCodeQLForCmd(cmd) {
                 process.execPath,
                 tracerEnvJs,
                 envFile,
-            ]).exec();
+            ]);
             return JSON.parse(fs.readFileSync(envFile, "utf-8"));
         },
         async databaseInit(databasePath, language, sourceRoot) {
-            await new toolrunner.ToolRunner(cmd, [
+            await runTool(cmd, [
                 "database",
                 "init",
                 databasePath,
                 `--language=${language}`,
                 `--source-root=${sourceRoot}`,
                 ...getExtraOptionsFromEnv(["database", "init"]),
-            ]).exec();
+            ]);
         },
         async runAutobuild(language) {
             const cmdName = process.platform === "win32" ? "autobuild.cmd" : "autobuild.sh";
@@ -377,7 +380,7 @@ function getCodeQLForCmd(cmd) {
                 "-Dhttp.keepAlive=false",
                 "-Dmaven.wagon.http.pool=false",
             ].join(" ");
-            await new toolrunner.ToolRunner(autobuildCmd).exec();
+            await runTool(autobuildCmd);
         },
         async extractScannedLanguage(databasePath, language) {
             // Get extractor location
@@ -416,6 +419,7 @@ function getCodeQLForCmd(cmd) {
             await toolrunner_error_catcher_1.toolrunnerErrorCatcher(cmd, [
                 "database",
                 "finalize",
+                "--finalize-dataset",
                 threadsFlag,
                 ...getExtraOptionsFromEnv(["database", "finalize"]),
                 databasePath,
@@ -423,14 +427,7 @@ function getCodeQLForCmd(cmd) {
         },
         async resolveLanguages() {
             const codeqlArgs = ["resolve", "languages", "--format=json"];
-            let output = "";
-            await new toolrunner.ToolRunner(cmd, codeqlArgs, {
-                listeners: {
-                    stdout: (data) => {
-                        output += data.toString();
-                    },
-                },
-            }).exec();
+            const output = await runTool(cmd, codeqlArgs);
             try {
                 return JSON.parse(output);
             }
@@ -449,14 +446,7 @@ function getCodeQLForCmd(cmd) {
             if (extraSearchPath !== undefined) {
                 codeqlArgs.push("--additional-packs", extraSearchPath);
             }
-            let output = "";
-            await new toolrunner.ToolRunner(cmd, codeqlArgs, {
-                listeners: {
-                    stdout: (data) => {
-                        output += data.toString();
-                    },
-                },
-            }).exec();
+            const output = await runTool(cmd, codeqlArgs);
             try {
                 return JSON.parse(output);
             }
@@ -464,53 +454,102 @@ function getCodeQLForCmd(cmd) {
                 throw new Error(`Unexpected output from codeql resolve queries: ${e}`);
             }
         },
-        async databaseAnalyze(databasePath, sarifFile, extraSearchPath, querySuite, memoryFlag, addSnippetsFlag, threadsFlag, automationDetailsId) {
-            const args = [
+        async databaseRunQueries(databasePath, extraSearchPath, querySuitePath, memoryFlag, threadsFlag) {
+            const codeqlArgs = [
                 "database",
-                "analyze",
+                "run-queries",
                 memoryFlag,
                 threadsFlag,
                 databasePath,
                 "--min-disk-free=1024",
-                "--format=sarif-latest",
-                "--sarif-multicause-markdown",
-                "--sarif-group-rules-by-pack",
-                `--output=${sarifFile}`,
-                addSnippetsFlag,
-                // Enable progress verbosity so we log each query as it's interpreted. This aids debugging
-                // when interpretation takes a while for one of the queries being analyzed.
                 "-v",
-                ...getExtraOptionsFromEnv(["database", "analyze"]),
+                ...getExtraOptionsFromEnv(["database", "run-queries"]),
             ];
             if (extraSearchPath !== undefined) {
-                args.push("--additional-packs", extraSearchPath);
+                codeqlArgs.push("--additional-packs", extraSearchPath);
             }
+            codeqlArgs.push(querySuitePath);
+            await runTool(cmd, codeqlArgs);
+        },
+        async databaseInterpretResults(databasePath, querySuitePaths, sarifFile, addSnippetsFlag, threadsFlag, automationDetailsId) {
+            const codeqlArgs = [
+                "database",
+                "interpret-results",
+                threadsFlag,
+                "--format=sarif-latest",
+                "--print-metrics-summary",
+                "--sarif-group-rules-by-pack",
+                "-v",
+                `--output=${sarifFile}`,
+                addSnippetsFlag,
+                ...getExtraOptionsFromEnv(["database", "interpret-results"]),
+            ];
             if (automationDetailsId !== undefined) {
-                args.push("--sarif-category", automationDetailsId);
+                codeqlArgs.push("--sarif-category", automationDetailsId);
             }
-            args.push(querySuite);
+            codeqlArgs.push(databasePath, ...querySuitePaths);
             // capture stdout, which contains analysis summaries
-            let output = "";
-            await new toolrunner.ToolRunner(cmd, args, {
-                listeners: {
-                    stdout: (data) => {
-                        output += data.toString("utf8");
-                    },
-                },
-            }).exec();
-            return output;
+            return await runTool(cmd, codeqlArgs);
+        },
+        /**
+         * Download specified packs into the package cache. If the specified
+         * package and version already exists (e.g., from a previous analysis run),
+         * then it is not downloaded again (unless the extra option `--force` is
+         * specified).
+         *
+         * If no version is specified, then the latest version is
+         * downloaded. The check to determine what the latest version is is done
+         * each time this package is requested.
+         */
+        async packDownload(packs) {
+            const codeqlArgs = [
+                "pack",
+                "download",
+                "--format=json",
+                ...getExtraOptionsFromEnv(["pack", "download"]),
+                ...packs.map(packWithVersionToString),
+            ];
+            const output = await runTool(cmd, codeqlArgs);
+            try {
+                const parsedOutput = JSON.parse(output);
+                if (Array.isArray(parsedOutput.packs) &&
+                    // TODO PackDownloadOutput will not include the version if it is not specified
+                    // in the input. The version is always the latest version available.
+                    // It should be added to the output, but this requires a CLI change
+                    parsedOutput.packs.every((p) => p.name /* && p.version */)) {
+                    return parsedOutput;
+                }
+                else {
+                    throw new Error("Unexpected output from pack download");
+                }
+            }
+            catch (e) {
+                throw new Error(`Attempted to download specified packs but got an error:\n${output}\n${e}`);
+            }
         },
         async databaseCleanup(databasePath, cleanupLevel) {
-            const args = [
+            const codeqlArgs = [
                 "database",
                 "cleanup",
                 databasePath,
                 `--mode=${cleanupLevel}`,
             ];
+            await runTool(cmd, codeqlArgs);
+        },
+        async databaseBundle(databasePath, outputFilePath) {
+            const args = [
+                "database",
+                "bundle",
+                databasePath,
+                `--output=${outputFilePath}`,
+            ];
             await new toolrunner.ToolRunner(cmd, args).exec();
         },
     };
 }
+function packWithVersionToString(pack) {
+    return pack.version ? `${pack.packName}@${pack.version}` : pack.packName;
+}
 /**
  * Gets the options for `path` of `options` as an array of extra option strings.
  */
@@ -557,4 +596,15 @@ function getExtraOptions(options, paths, pathInfo) {
     return all.concat(specific);
 }
 exports.getExtraOptions = getExtraOptions;
+async function runTool(cmd, args = []) {
+    let output = "";
+    await new toolrunner.ToolRunner(cmd, args, {
+        listeners: {
+            stdout: (data) => {
+                output += data.toString();
+            },
+        },
+    }).exec();
+    return output;
+}
 //# sourceMappingURL=codeql.js.map

lib/config-utils.js

@@ -10,6 +10,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
 const yaml = __importStar(require("js-yaml"));
+const semver = __importStar(require("semver"));
 const api = __importStar(require("./api-client"));
 const externalQueries = __importStar(require("./external-queries"));
 const languages_1 = require("./languages");
@@ -20,6 +21,7 @@ const QUERIES_PROPERTY = "queries";
 const QUERIES_USES_PROPERTY = "uses";
 const PATHS_IGNORE_PROPERTY = "paths-ignore";
 const PATHS_PROPERTY = "paths";
+const PACKS_PROPERTY = "packs";
 /**
  * A list of queries from https://github.com/github/codeql that
  * we don't want to run. Disabling them here is a quicker alternative to
@@ -254,6 +256,22 @@ function getPathsInvalid(configFile) {
     return getConfigFilePropertyError(configFile, PATHS_PROPERTY, "must be an array of non-empty strings");
 }
 exports.getPathsInvalid = getPathsInvalid;
+function getPacksRequireLanguage(lang, configFile) {
+    return getConfigFilePropertyError(configFile, PACKS_PROPERTY, `has "${lang}", but it is not one of the languages to analyze`);
+}
+exports.getPacksRequireLanguage = getPacksRequireLanguage;
+function getPacksInvalidSplit(configFile) {
+    return getConfigFilePropertyError(configFile, PACKS_PROPERTY, "must split packages by language");
+}
+exports.getPacksInvalidSplit = getPacksInvalidSplit;
+function getPacksInvalid(configFile) {
+    return getConfigFilePropertyError(configFile, PACKS_PROPERTY, "must be an array of non-empty strings");
+}
+exports.getPacksInvalid = getPacksInvalid;
+function getPacksStrInvalid(packStr, configFile) {
+    return getConfigFilePropertyError(configFile, PACKS_PROPERTY, `"${packStr}" is not a valid pack`);
+}
+exports.getPacksStrInvalid = getPacksStrInvalid;
 function getLocalPathOutsideOfRepository(configFile, localPath) {
     return getConfigFilePropertyError(configFile, `${QUERIES_PROPERTY}.${QUERIES_USES_PROPERTY}`, `is invalid as the local path "${localPath}" is outside of the repository`);
 }
@@ -409,6 +427,7 @@ async function getDefaultConfig(languagesInput, queriesInput, dbLocation, reposi
         queries,
         pathsIgnore: [],
         paths: [],
+        packs: {},
         originalUserInput: {},
         tempDir,
         toolCacheDir,
@@ -422,6 +441,7 @@ exports.getDefaultConfig = getDefaultConfig;
  * Load the config from the given file.
  */
 async function loadConfig(languagesInput, queriesInput, configFile, dbLocation, repository, tempDir, toolCacheDir, codeQL, checkoutPath, gitHubVersion, apiDetails, logger) {
+    var _a;
     let parsedYAML;
     if (isLocal(configFile)) {
         // Treat the config file as relative to the workspace
@@ -470,10 +490,11 @@ async function loadConfig(languagesInput, queriesInput, configFile, dbLocation,
     }
     if (shouldAddConfigFileQueries(queriesInput) &&
         QUERIES_PROPERTY in parsedYAML) {
-        if (!(parsedYAML[QUERIES_PROPERTY] instanceof Array)) {
+        const queriesArr = parsedYAML[QUERIES_PROPERTY];
+        if (!Array.isArray(queriesArr)) {
             throw new Error(getQueriesInvalid(configFile));
         }
-        for (const query of parsedYAML[QUERIES_PROPERTY]) {
+        for (const query of queriesArr) {
             if (!(QUERIES_USES_PROPERTY in query) ||
                 typeof query[QUERIES_USES_PROPERTY] !== "string") {
                 throw new Error(getQueryUsesInvalid(configFile));
@@ -482,7 +503,7 @@ async function loadConfig(languagesInput, queriesInput, configFile, dbLocation,
         }
     }
     if (PATHS_IGNORE_PROPERTY in parsedYAML) {
-        if (!(parsedYAML[PATHS_IGNORE_PROPERTY] instanceof Array)) {
+        if (!Array.isArray(parsedYAML[PATHS_IGNORE_PROPERTY])) {
             throw new Error(getPathsIgnoreInvalid(configFile));
         }
         for (const ignorePath of parsedYAML[PATHS_IGNORE_PROPERTY]) {
@@ -493,7 +514,7 @@ async function loadConfig(languagesInput, queriesInput, configFile, dbLocation,
         }
     }
     if (PATHS_PROPERTY in parsedYAML) {
-        if (!(parsedYAML[PATHS_PROPERTY] instanceof Array)) {
+        if (!Array.isArray(parsedYAML[PATHS_PROPERTY])) {
             throw new Error(getPathsInvalid(configFile));
         }
         for (const includePath of parsedYAML[PATHS_PROPERTY]) {
@@ -503,11 +524,13 @@ async function loadConfig(languagesInput, queriesInput, configFile, dbLocation,
             paths.push(validateAndSanitisePath(includePath, PATHS_PROPERTY, configFile, logger));
         }
     }
+    const packs = parsePacks((_a = parsedYAML[PACKS_PROPERTY], (_a !== null && _a !== void 0 ? _a : {})), languages, configFile);
     return {
         languages,
         queries,
         pathsIgnore,
         paths,
+        packs,
         originalUserInput: parsedYAML,
         tempDir,
         toolCacheDir,
@@ -516,6 +539,68 @@ async function loadConfig(languagesInput, queriesInput, configFile, dbLocation,
         dbLocation: dbLocationOrDefault(dbLocation, tempDir),
     };
 }
+/**
+ * Pack names must be in the form of `scope/name`, with only alpha-numeric characters,
+ * and `-` allowed as long as not the first or last char.
+ **/
+const PACK_IDENTIFIER_PATTERN = (function () {
+    const alphaNumeric = "[a-z0-9]";
+    const alphaNumericDash = "[a-z0-9-]";
+    const component = `${alphaNumeric}(${alphaNumericDash}*${alphaNumeric})?`;
+    return new RegExp(`^${component}/${component}$`);
+})();
+// Exported for testing
+function parsePacks(packsByLanguage, languages, configFile) {
+    const packs = {};
+    if (Array.isArray(packsByLanguage)) {
+        if (languages.length === 1) {
+            // single language analysis, so language is implicit
+            packsByLanguage = {
+                [languages[0]]: packsByLanguage,
+            };
+        }
+        else {
+            // this is an error since multi-language analysis requires
+            // packs split by language
+            throw new Error(getPacksInvalidSplit(configFile));
+        }
+    }
+    for (const [lang, packsArr] of Object.entries(packsByLanguage)) {
+        if (!Array.isArray(packsArr)) {
+            throw new Error(getPacksInvalid(configFile));
+        }
+        if (!languages.includes(lang)) {
+            throw new Error(getPacksRequireLanguage(lang, configFile));
+        }
+        packs[lang] = [];
+        for (const packStr of packsArr) {
+            packs[lang].push(toPackWithVersion(packStr, configFile));
+        }
+    }
+    return packs;
+}
+exports.parsePacks = parsePacks;
+function toPackWithVersion(packStr, configFile) {
+    if (typeof packStr !== "string") {
+        throw new Error(getPacksStrInvalid(packStr, configFile));
+    }
+    const nameWithVersion = packStr.split("@");
+    let version;
+    if (nameWithVersion.length > 2 ||
+        !PACK_IDENTIFIER_PATTERN.test(nameWithVersion[0])) {
+        throw new Error(getPacksStrInvalid(packStr, configFile));
+    }
+    else if (nameWithVersion.length === 2) {
+        version = semver.clean(nameWithVersion[1]) || undefined;
+        if (!version) {
+            throw new Error(getPacksStrInvalid(packStr, configFile));
+        }
+    }
+    return {
+        packName: nameWithVersion[0],
+        version,
+    };
+}
 function dbLocationOrDefault(dbLocation, tempDir) {
     return dbLocation || path.resolve(tempDir, "codeql_databases");
 }
@@ -526,6 +611,7 @@ function dbLocationOrDefault(dbLocation, tempDir) {
  * a default config. The parsed config is then stored to a known location.
  */
 async function initConfig(languagesInput, queriesInput, configFile, dbLocation, repository, tempDir, toolCacheDir, codeQL, checkoutPath, gitHubVersion, apiDetails, logger) {
+    var _a, _b, _c;
     let config;
     // If no config file was provided create an empty one
     if (!configFile) {
@@ -538,9 +624,10 @@ async function initConfig(languagesInput, queriesInput, configFile, dbLocation,
     // The list of queries should not be empty for any language. If it is then
     // it is a user configuration error.
     for (const language of config.languages) {
-        if (config.queries[language] === undefined ||
-            (config.queries[language].builtin.length === 0 &&
-                config.queries[language].custom.length === 0)) {
+        const hasBuiltinQueries = ((_a = config.queries[language]) === null || _a === void 0 ? void 0 : _a.builtin.length) > 0;
+        const hasCustomQueries = ((_b = config.queries[language]) === null || _b === void 0 ? void 0 : _b.custom.length) > 0;
+        const hasPacks = (((_c = config.packs[language]) === null || _c === void 0 ? void 0 : _c.length) || 0) > 0;
+        if (!hasPacks && !hasBuiltinQueries && !hasCustomQueries) {
             throw new Error(`Did not detect any queries to run for ${language}. ` +
                 "Please make sure that the default queries are enabled, or you are specifying queries to run.");
         }

lib/config-utils.test.js

@@ -14,6 +14,7 @@ const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
 const github = __importStar(require("@actions/github"));
 const ava_1 = __importDefault(require("ava"));
+const semver_1 = require("semver");
 const sinon_1 = __importDefault(require("sinon"));
 const api = __importStar(require("./api-client"));
 const codeql_1 = require("./codeql");
@@ -200,6 +201,7 @@ ava_1.default("load non-empty input", async (t) => {
             codeQLCmd: codeQL.getPath(),
             gitHubVersion,
             dbLocation: path.resolve(tmpDir, "codeql_databases"),
+            packs: {},
         };
         const languages = "javascript";
         const configFilePath = createConfigFile(inputFileContents, tmpDir);
@@ -557,6 +559,101 @@ ava_1.default("Unknown languages", async (t) => {
         }
     });
 });
+ava_1.default("Config specifies packages", async (t) => {
+    return await util.withTmpDir(async (tmpDir) => {
+        const codeQL = codeql_1.setCodeQL({
+            async resolveQueries() {
+                return {
+                    byLanguage: {},
+                    noDeclaredLanguage: {},
+                    multipleDeclaredLanguages: {},
+                };
+            },
+        });
+        const inputFileContents = `
+      name: my config
+      disable-default-queries: true
+      packs:
+        - a/b@1.2.3
+      `;
+        const configFile = path.join(tmpDir, "codeql-config.yaml");
+        fs.writeFileSync(configFile, inputFileContents);
+        const languages = "javascript";
+        const { packs } = await configUtils.initConfig(languages, undefined, configFile, undefined, { owner: "github", repo: "example " }, tmpDir, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, logging_1.getRunnerLogger(true));
+        t.deepEqual(packs, {
+            [languages_1.Language.javascript]: [
+                {
+                    packName: "a/b",
+                    version: semver_1.clean("1.2.3"),
+                },
+            ],
+        });
+    });
+});
+ava_1.default("Config specifies packages for multiple languages", async (t) => {
+    return await util.withTmpDir(async (tmpDir) => {
+        const codeQL = codeql_1.setCodeQL({
+            async resolveQueries() {
+                return {
+                    byLanguage: {
+                        cpp: { "/foo/a.ql": {} },
+                    },
+                    noDeclaredLanguage: {},
+                    multipleDeclaredLanguages: {},
+                };
+            },
+        });
+        const inputFileContents = `
+      name: my config
+      disable-default-queries: true
+      queries:
+      - uses: ./foo
+      packs:
+        javascript:
+          - a/b@1.2.3
+        python:
+          - c/d@1.2.3
+      `;
+        const configFile = path.join(tmpDir, "codeql-config.yaml");
+        fs.writeFileSync(configFile, inputFileContents);
+        fs.mkdirSync(path.join(tmpDir, "foo"));
+        const languages = "javascript,python,cpp";
+        const { packs, queries } = await configUtils.initConfig(languages, undefined, configFile, undefined, { owner: "github", repo: "example" }, tmpDir, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, logging_1.getRunnerLogger(true));
+        t.deepEqual(packs, {
+            [languages_1.Language.javascript]: [
+                {
+                    packName: "a/b",
+                    version: semver_1.clean("1.2.3"),
+                },
+            ],
+            [languages_1.Language.python]: [
+                {
+                    packName: "c/d",
+                    version: semver_1.clean("1.2.3"),
+                },
+            ],
+        });
+        t.deepEqual(queries, {
+            cpp: {
+                builtin: [],
+                custom: [
+                    {
+                        queries: ["/foo/a.ql"],
+                        searchPath: tmpDir,
+                    },
+                ],
+            },
+            javascript: {
+                builtin: [],
+                custom: [],
+            },
+            python: {
+                builtin: [],
+                custom: [],
+            },
+        });
+    });
+});
 function doInvalidInputTest(testName, inputFileContents, expectedErrorMessageGenerator) {
     ava_1.default(`load invalid input - ${testName}`, async (t) => {
         return await util.withTmpDir(async (tmpDir) => {
@@ -644,4 +741,54 @@ ava_1.default("path sanitisation", (t) => {
     // Trailing stars are stripped
     t.deepEqual(configUtils.validateAndSanitisePath("foo/**", propertyName, configFile, logging_1.getRunnerLogger(true)), "foo/");
 });
+/**
+ * Test macro for ensuring the packs block is valid
+ */
+function parsePacksMacro(t, packsByLanguage, languages, expected) {
+    t.deepEqual(configUtils.parsePacks(packsByLanguage, languages, "/a/b"), expected);
+}
+parsePacksMacro.title = (providedTitle) => `Parse Packs: ${providedTitle}`;
+/**
+ * Test macro for testing when the packs block is invalid
+ */
+function parsePacksErrorMacro(t, packsByLanguage, languages, expected) {
+    t.throws(() => {
+        configUtils.parsePacks(packsByLanguage, languages, "/a/b");
+    }, {
+        message: expected,
+    });
+}
+parsePacksErrorMacro.title = (providedTitle) => `Parse Packs Error: ${providedTitle}`;
+function invalidPackNameMacro(t, name) {
+    parsePacksErrorMacro(t, { [languages_1.Language.cpp]: [name] }, [languages_1.Language.cpp], new RegExp(`The configuration file "/a/b" is invalid: property "packs" "${name}" is not a valid pack`));
+}
+invalidPackNameMacro.title = (_, arg) => `Invalid pack string: ${arg}`;
+ava_1.default("no packs", parsePacksMacro, {}, [], {});
+ava_1.default("two packs", parsePacksMacro, ["a/b", "c/d@1.2.3"], [languages_1.Language.cpp], {
+    [languages_1.Language.cpp]: [
+        { packName: "a/b", version: undefined },
+        { packName: "c/d", version: semver_1.clean("1.2.3") },
+    ],
+});
+ava_1.default("two packs with language", parsePacksMacro, {
+    [languages_1.Language.cpp]: ["a/b", "c/d@1.2.3"],
+    [languages_1.Language.java]: ["d/e", "f/g@1.2.3"],
+}, [languages_1.Language.cpp, languages_1.Language.java, languages_1.Language.csharp], {
+    [languages_1.Language.cpp]: [
+        { packName: "a/b", version: undefined },
+        { packName: "c/d", version: semver_1.clean("1.2.3") },
+    ],
+    [languages_1.Language.java]: [
+        { packName: "d/e", version: undefined },
+        { packName: "f/g", version: semver_1.clean("1.2.3") },
+    ],
+});
+ava_1.default("no language", parsePacksErrorMacro, ["a/b@1.2.3"], [languages_1.Language.java, languages_1.Language.python], /The configuration file "\/a\/b" is invalid: property "packs" must split packages by language/);
+ava_1.default("invalid language", parsePacksErrorMacro, { [languages_1.Language.java]: ["c/d"] }, [languages_1.Language.cpp], /The configuration file "\/a\/b" is invalid: property "packs" has "java", but it is not one of the languages to analyze/);
+ava_1.default("not an array", parsePacksErrorMacro, { [languages_1.Language.cpp]: "c/d" }, [languages_1.Language.cpp], /The configuration file "\/a\/b" is invalid: property "packs" must be an array of non-empty strings/);
+ava_1.default(invalidPackNameMacro, "c"); // all packs require at least a scope and a name
+ava_1.default(invalidPackNameMacro, "c-/d");
+ava_1.default(invalidPackNameMacro, "-c/d");
+ava_1.default(invalidPackNameMacro, "c/d_d");
+ava_1.default(invalidPackNameMacro, "c/d@x");
 //# sourceMappingURL=config-utils.test.js.map

lib/count-loc.test.js

@@ -21,13 +21,13 @@ ava_1.default("ensure lines of code works for cpp and js", async (t) => {
     const results = await count_loc_1.countLoc(path.join(__dirname, "../tests/multi-language-repo"), [], [], [languages_1.Language.cpp, languages_1.Language.javascript], logging_1.getRunnerLogger(true));
     t.deepEqual(results, {
         cpp: 6,
-        javascript: 3,
+        javascript: 9,
     });
 });
 ava_1.default("ensure lines of code can handle undefined language", async (t) => {
     const results = await count_loc_1.countLoc(path.join(__dirname, "../tests/multi-language-repo"), [], [], [languages_1.Language.javascript, languages_1.Language.python, "hucairz"], logging_1.getRunnerLogger(true));
     t.deepEqual(results, {
-        javascript: 3,
+        javascript: 9,
         python: 5,
     });
 });
@@ -54,7 +54,7 @@ ava_1.default("ensure lines of code can handle empty includes", async (t) => {
 ava_1.default("ensure lines of code can handle exclude", async (t) => {
     const results = await count_loc_1.countLoc(path.join(__dirname, "../tests/multi-language-repo"), [], ["**/*.py"], [languages_1.Language.javascript, languages_1.Language.python], logging_1.getRunnerLogger(true));
     t.deepEqual(results, {
-        javascript: 3,
+        javascript: 9,
     });
 });
 //# sourceMappingURL=count-loc.test.js.map

lib/fingerprints.js

@@ -16,6 +16,7 @@ const tab = "\t".charCodeAt(0);
 const space = " ".charCodeAt(0);
 const lf = "\n".charCodeAt(0);
 const cr = "\r".charCodeAt(0);
+const EOF = 65535;
 const BLOCK_SIZE = 100;
 const MOD = long_1.default.fromInt(37); // L
 // Compute the starting point for the hash mod
@@ -36,9 +37,9 @@ function computeFirstMod() {
  * the hashes of the lines near the end of the file.
  *
  * @param callback function that is called with the line number (1-based) and hash for every line
- * @param input The file's contents
+ * @param filepath The path to the file to hash
  */
-function hash(callback, input) {
+async function hash(callback, filepath) {
     // A rolling view in to the input
     const window = Array(BLOCK_SIZE).fill(0);
     // If the character in the window is the start of a new line
@@ -82,12 +83,11 @@ function hash(callback, input) {
     // as we go. Once we reach a point in the window again then we've processed
     // BLOCK_SIZE characters and if the last character at this point in the window
     // was the start of a line then we should output the hash for that line.
-    for (let i = 0, len = input.length; i <= len; i++) {
-        let current = i === len ? 65535 : input.charCodeAt(i);
+    const processCharacter = function (current) {
         // skip tabs, spaces, and line feeds that come directly after a carriage return
         if (current === space || current === tab || (prevCR && current === lf)) {
             prevCR = false;
-            continue;
+            return;
         }
         // replace CR with LF
         if (current === cr) {
@@ -109,7 +109,14 @@ function hash(callback, input) {
             lineStart = true;
         }
         updateHash(current);
+    };
+    const readStream = fs.createReadStream(filepath, "utf8");
+    for await (const data of readStream) {
+        for (let i = 0; i < data.length; ++i) {
+            processCharacter(data.charCodeAt(i));
+        }
     }
+    processCharacter(EOF);
     // Flush the remaining lines
     for (let i = 0; i < BLOCK_SIZE; i++) {
         if (lineNumbers[index] !== -1) {
@@ -206,8 +213,8 @@ function resolveUriToFile(location, artifacts, checkoutPath, logger) {
 exports.resolveUriToFile = resolveUriToFile;
 // Compute fingerprints for results in the given sarif file
 // and return an updated sarif file contents.
-function addFingerprints(sarifContents, checkoutPath, logger) {
-    var _a, _b;
+async function addFingerprints(sarifContents, checkoutPath, logger) {
+    var _a, _b, _c, _d, _e;
     const sarif = JSON.parse(sarifContents);
     // Gather together results for the same file and construct
     // callbacks to accept hashes for that file and update the location
@@ -222,6 +229,10 @@ function addFingerprints(sarifContents, checkoutPath, logger) {
                 logger.debug(`Unable to compute fingerprint for invalid location: ${JSON.stringify(primaryLocation)}`);
                 continue;
             }
+            if (((_e = (_d = (_c = primaryLocation) === null || _c === void 0 ? void 0 : _c.physicalLocation) === null || _d === void 0 ? void 0 : _d.region) === null || _e === void 0 ? void 0 : _e.startLine) === undefined) {
+                // Locations without a line number are unlikely to be source files
+                continue;
+            }
             const filepath = resolveUriToFile(primaryLocation.physicalLocation.artifactLocation, artifacts, checkoutPath, logger);
             if (!filepath) {
                 continue;
@@ -240,8 +251,7 @@ function addFingerprints(sarifContents, checkoutPath, logger) {
                 c(lineNumber, hashValue);
             }
         };
-        const fileContents = fs.readFileSync(filepath).toString();
-        hash(teeCallback, fileContents);
+        await hash(teeCallback, filepath);
     }
     return JSON.stringify(sarif);
 }

lib/fingerprints.test.js

@@ -16,28 +16,33 @@ const ava_1 = __importDefault(require("ava"));
 const fingerprints = __importStar(require("./fingerprints"));
 const logging_1 = require("./logging");
 const testing_utils_1 = require("./testing-utils");
+const util = __importStar(require("./util"));
 testing_utils_1.setupTests(ava_1.default);
-function testHash(t, input, expectedHashes) {
-    let index = 0;
-    const callback = function (lineNumber, hash) {
-        t.is(lineNumber, index + 1);
-        t.is(hash, expectedHashes[index]);
-        index++;
-    };
-    fingerprints.hash(callback, input);
-    t.is(index, input.split(/\r\n|\r|\n/).length);
+async function testHash(t, input, expectedHashes) {
+    await util.withTmpDir(async (tmpDir) => {
+        const tmpFile = path.resolve(tmpDir, "testfile");
+        fs.writeFileSync(tmpFile, input);
+        let index = 0;
+        const callback = function (lineNumber, hash) {
+            t.is(lineNumber, index + 1);
+            t.is(hash, expectedHashes[index]);
+            index++;
+        };
+        await fingerprints.hash(callback, tmpFile);
+        t.is(index, input.split(/\r\n|\r|\n/).length);
+    });
 }
-ava_1.default("hash", (t) => {
+ava_1.default("hash", async (t) => {
     // Try empty file
-    testHash(t, "", ["c129715d7a2bc9a3:1"]);
+    await testHash(t, "", ["c129715d7a2bc9a3:1"]);
     // Try various combinations of newline characters
-    testHash(t, " a\nb\n  \t\tc\n d", [
+    await testHash(t, " a\nb\n  \t\tc\n d", [
         "271789c17abda88f:1",
         "54703d4cd895b18:1",
         "180aee12dab6264:1",
         "a23a3dc5e078b07b:1",
     ]);
-    testHash(t, " hello; \t\nworld!!!\n\n\n  \t\tGreetings\n End", [
+    await testHash(t, " hello; \t\nworld!!!\n\n\n  \t\tGreetings\n End", [
         "8b7cf3e952e7aeb2:1",
         "b1ae1287ec4718d9:1",
         "bff680108adb0fcc:1",
@@ -45,7 +50,7 @@ ava_1.default("hash", (t) => {
         "b86d3392aea1be30:1",
         "e6ceba753e1a442:1",
     ]);
-    testHash(t, " hello; \t\nworld!!!\n\n\n  \t\tGreetings\n End\n", [
+    await testHash(t, " hello; \t\nworld!!!\n\n\n  \t\tGreetings\n End\n", [
         "e9496ae3ebfced30:1",
         "fb7c023a8b9ccb3f:1",
         "ce8ba1a563dcdaca:1",
@@ -54,7 +59,7 @@ ava_1.default("hash", (t) => {
         "c8e28b0b4002a3a0:1",
         "c129715d7a2bc9a3:1",
     ]);
-    testHash(t, " hello; \t\nworld!!!\r\r\r  \t\tGreetings\r End\r", [
+    await testHash(t, " hello; \t\nworld!!!\r\r\r  \t\tGreetings\r End\r", [
         "e9496ae3ebfced30:1",
         "fb7c023a8b9ccb3f:1",
         "ce8ba1a563dcdaca:1",
@@ -63,7 +68,7 @@ ava_1.default("hash", (t) => {
         "c8e28b0b4002a3a0:1",
         "c129715d7a2bc9a3:1",
     ]);
-    testHash(t, " hello; \t\r\nworld!!!\r\n\r\n\r\n  \t\tGreetings\r\n End\r\n", [
+    await testHash(t, " hello; \t\r\nworld!!!\r\n\r\n\r\n  \t\tGreetings\r\n End\r\n", [
         "e9496ae3ebfced30:1",
         "fb7c023a8b9ccb3f:1",
         "ce8ba1a563dcdaca:1",
@@ -72,7 +77,7 @@ ava_1.default("hash", (t) => {
         "c8e28b0b4002a3a0:1",
         "c129715d7a2bc9a3:1",
     ]);
-    testHash(t, " hello; \t\nworld!!!\r\n\n\r  \t\tGreetings\r End\r\n", [
+    await testHash(t, " hello; \t\nworld!!!\r\n\n\r  \t\tGreetings\r End\r\n", [
         "e9496ae3ebfced30:1",
         "fb7c023a8b9ccb3f:1",
         "ce8ba1a563dcdaca:1",
@@ -82,7 +87,7 @@ ava_1.default("hash", (t) => {
         "c129715d7a2bc9a3:1",
     ]);
     // Try repeating line that will generate identical hashes
-    testHash(t, "Lorem ipsum dolor sit amet.\n".repeat(10), [
+    await testHash(t, "Lorem ipsum dolor sit amet.\n".repeat(10), [
         "a7f2ff13bc495cf2:1",
         "a7f2ff13bc495cf2:2",
         "a7f2ff13bc495cf2:3",
@@ -95,7 +100,7 @@ ava_1.default("hash", (t) => {
         "cc97dc7b1d7d8f7b:1",
         "c129715d7a2bc9a3:1",
     ]);
-    testHash(t, "x = 2\nx = 1\nprint(x)\nx = 3\nprint(x)\nx = 4\nprint(x)\n", [
+    await testHash(t, "x = 2\nx = 1\nprint(x)\nx = 3\nprint(x)\nx = 4\nprint(x)\n", [
         "e54938cc54b302f1:1",
         "bb609acbe9138d60:1",
         "1131fd5871777f34:1",
@@ -150,7 +155,7 @@ ava_1.default("resolveUriToFile", (t) => {
     t.is(testResolveUriToFile(dirpath, undefined, []), undefined);
     t.is(testResolveUriToFile(`file://${dirpath}`, undefined, []), undefined);
 });
-ava_1.default("addFingerprints", (t) => {
+ava_1.default("addFingerprints", async (t) => {
     // Run an end-to-end test on a test file
     let input = fs
         .readFileSync(`${__dirname}/../src/testdata/fingerprinting.input.sarif`)
@@ -163,9 +168,9 @@ ava_1.default("addFingerprints", (t) => {
     expected = JSON.stringify(JSON.parse(expected));
     // The URIs in the SARIF files resolve to files in the testdata directory
     const checkoutPath = path.normalize(`${__dirname}/../src/testdata`);
-    t.deepEqual(fingerprints.addFingerprints(input, checkoutPath, logging_1.getRunnerLogger(true)), expected);
+    t.deepEqual(await fingerprints.addFingerprints(input, checkoutPath, logging_1.getRunnerLogger(true)), expected);
 });
-ava_1.default("missingRegions", (t) => {
+ava_1.default("missingRegions", async (t) => {
     // Run an end-to-end test on a test file
     let input = fs
         .readFileSync(`${__dirname}/../src/testdata/fingerprinting2.input.sarif`)
@@ -178,6 +183,6 @@ ava_1.default("missingRegions", (t) => {
     expected = JSON.stringify(JSON.parse(expected));
     // The URIs in the SARIF files resolve to files in the testdata directory
     const checkoutPath = path.normalize(`${__dirname}/../src/testdata`);
-    t.deepEqual(fingerprints.addFingerprints(input, checkoutPath, logging_1.getRunnerLogger(true)), expected);
+    t.deepEqual(await fingerprints.addFingerprints(input, checkoutPath, logging_1.getRunnerLogger(true)), expected);
 });
 //# sourceMappingURL=fingerprints.test.js.map

lib/tracer-config.test.js

@@ -31,6 +31,7 @@ function getTestConfig(tmpDir) {
         codeQLCmd: "",
         gitHubVersion: { type: util.GitHubVariant.DOTCOM },
         dbLocation: path.resolve(tmpDir, "codeql_databases"),
+        packs: {},
     };
 }
 // A very minimal setup

lib/upload-lib.js

@@ -243,7 +243,7 @@ async function uploadFiles(sarifFiles, repositoryNwo, commitOid, ref, analysisKe
         validateSarifFileSchema(file, logger);
     }
     let sarifPayload = combineSarifFiles(sarifFiles);
-    sarifPayload = fingerprints.addFingerprints(sarifPayload, checkoutPath, logger);
+    sarifPayload = await fingerprints.addFingerprints(sarifPayload, checkoutPath, logger);
     sarifPayload = populateRunAutomationDetails(sarifPayload, category, analysisKey, environment);
     const zippedSarif = zlib_1.default.gzipSync(sarifPayload).toString("base64");
     const checkoutURI = file_url_1.default(checkoutPath);

lib/util.js

@@ -408,4 +408,9 @@ function getRequiredEnvParam(paramName) {
     return value;
 }
 exports.getRequiredEnvParam = getRequiredEnvParam;
+function isHTTPError(arg) {
+    var _a;
+    return ((_a = arg) === null || _a === void 0 ? void 0 : _a.status) !== undefined && Number.isInteger(arg.status);
+}
+exports.isHTTPError = isHTTPError;
 //# sourceMappingURL=util.js.map

node_modules/.package-lock.json

@@ -405,6 +405,12 @@
         "@types/node": "*"
       }
     },
+    "node_modules/@types/js-yaml": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/@types/js-yaml/-/js-yaml-4.0.1.tgz",
+      "integrity": "sha512-xdOvNmXmrZqqPy3kuCQ+fz6wA0xU5pji9cd1nDrflWaAWtYLLGk5ykW0H6yg5TVyehHP1pfmuuSaZkhP+kspVA==",
+      "dev": true
+    },
     "node_modules/@types/json-schema": {
       "version": "7.0.6",
       "dev": true,
@@ -1253,6 +1259,7 @@
       "dependencies": {
         "anymatch": "~3.1.1",
         "braces": "~3.0.2",
+        "fsevents": "~2.1.2",
         "glob-parent": "~5.1.0",
         "is-binary-path": "~2.1.0",
         "is-glob": "~4.0.1",
@@ -3238,6 +3245,9 @@
     "node_modules/jsonfile": {
       "version": "4.0.0",
       "license": "MIT",
+      "dependencies": {
+        "graceful-fs": "^4.1.6"
+      },
       "optionalDependencies": {
         "graceful-fs": "^4.1.6"
       }

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "codeql",
-  "version": "1.0.1",
+  "version": "1.0.2",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "codeql",
-  "version": "1.0.1",
+  "version": "1.0.2",
   "private": true,
   "description": "CodeQL action",
   "scripts": {

runner/package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "codeql-runner",
-  "version": "1.0.1",
+  "version": "1.0.2",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {

runner/package.json

@@ -1,6 +1,6 @@
 {
   "name": "codeql-runner",
-  "version": "1.0.1",
+  "version": "1.0.2",
   "private": true,
   "description": "CodeQL runner",
   "scripts": {

src/actions-util.ts

@@ -8,7 +8,7 @@ import * as yaml from "js-yaml";
 
 import * as api from "./api-client";
 import * as sharedEnv from "./shared-environment";
-import { getRequiredEnvParam, GITHUB_DOTCOM_URL } from "./util";
+import { getRequiredEnvParam, GITHUB_DOTCOM_URL, isHTTPError } from "./util";
 
 /**
  * The utils in this module are meant to be run inside of the action only.
@@ -576,15 +576,6 @@ export async function createStatusReportBase(
   return statusReport;
 }
 
-interface HTTPError {
-  status: number;
-  message?: string;
-}
-
-function isHTTPError(arg: any): arg is HTTPError {
-  return arg?.status !== undefined && Number.isInteger(arg.status);
-}
-
 const GENERIC_403_MSG =
   "The repo on which this action is running is not opted-in to CodeQL code scanning.";
 const GENERIC_404_MSG =
@@ -691,3 +682,30 @@ export function getRelativeScriptPath(): string {
   const actionsDirectory = path.join(path.dirname(runnerTemp), "_actions");
   return path.relative(actionsDirectory, __filename);
 }
+
+// Reads the contents of GITHUB_EVENT_PATH as a JSON object
+function getWorkflowEvent(): any {
+  const eventJsonFile = getRequiredEnvParam("GITHUB_EVENT_PATH");
+  try {
+    return JSON.parse(fs.readFileSync(eventJsonFile, "utf-8"));
+  } catch (e) {
+    throw new Error(
+      `Unable to read workflow event JSON from ${eventJsonFile}: ${e}`
+    );
+  }
+}
+
+// Is the version of the repository we are currently analyzing from the default branch,
+// or alternatively from another branch or a pull request.
+export async function isAnalyzingDefaultBranch(): Promise<boolean> {
+  // Get the current ref and trim and refs/heads/ prefix
+  let currentRef = await getRef();
+  currentRef = currentRef.startsWith("refs/heads/")
+    ? currentRef.substr("refs/heads/".length)
+    : currentRef;
+
+  const event = getWorkflowEvent();
+  const defaultBranch = event?.repository?.default_branch;
+
+  return currentRef === defaultBranch;
+}

src/analysis-paths.test.ts

@@ -21,6 +21,7 @@ test("emptyPaths", async (t) => {
       codeQLCmd: "",
       gitHubVersion: { type: util.GitHubVariant.DOTCOM } as util.GitHubVersion,
       dbLocation: path.resolve(tmpDir, "codeql_databases"),
+      packs: {},
     };
     analysisPaths.includeAndExcludeAnalysisPaths(config);
     t.is(process.env["LGTM_INDEX_INCLUDE"], undefined);
@@ -42,6 +43,7 @@ test("nonEmptyPaths", async (t) => {
       codeQLCmd: "",
       gitHubVersion: { type: util.GitHubVariant.DOTCOM } as util.GitHubVersion,
       dbLocation: path.resolve(tmpDir, "codeql_databases"),
+      packs: {},
     };
     analysisPaths.includeAndExcludeAnalysisPaths(config);
     t.is(process.env["LGTM_INDEX_INCLUDE"], "path1\npath2");
@@ -67,6 +69,7 @@ test("exclude temp dir", async (t) => {
       codeQLCmd: "",
       gitHubVersion: { type: util.GitHubVariant.DOTCOM } as util.GitHubVersion,
       dbLocation: path.resolve(tempDir, "codeql_databases"),
+      packs: {},
     };
     analysisPaths.includeAndExcludeAnalysisPaths(config);
     t.is(process.env["LGTM_INDEX_INCLUDE"], undefined);

src/analyze-action.ts

@@ -10,8 +10,11 @@ import {
   QueriesStatusReport,
   runCleanup,
 } from "./analyze";
+import { getApiClient, GitHubApiDetails } from "./api-client";
+import { getCodeQL } from "./codeql";
 import { Config, getConfig } from "./config-utils";
-import { getActionsLogger } from "./logging";
+import { getActionsLogger, Logger } from "./logging";
+import { parseRepositoryNwo, RepositoryNwo } from "./repository";
 import * as upload_lib from "./upload-lib";
 import * as util from "./util";
 
@@ -49,6 +52,76 @@ async function sendStatusReport(
   await actionsUtil.sendStatusReport(statusReport);
 }
 
+async function uploadDatabases(
+  repositoryNwo: RepositoryNwo,
+  config: Config,
+  apiDetails: GitHubApiDetails,
+  logger: Logger
+): Promise<void> {
+  if (actionsUtil.getRequiredInput("upload-database") !== "true") {
+    logger.debug("Database upload disabled in workflow. Skipping upload.");
+    return;
+  }
+
+  // Do nothing when not running against github.com
+  if (config.gitHubVersion.type !== util.GitHubVariant.DOTCOM) {
+    logger.debug("Not running against github.com. Skipping upload.");
+    return;
+  }
+
+  if (!(await actionsUtil.isAnalyzingDefaultBranch())) {
+    // We only want to upload a database if we are analyzing the default branch.
+    logger.debug("Not analyzing default branch. Skipping upload.");
+    return;
+  }
+
+  const client = getApiClient(apiDetails);
+  try {
+    await client.request("GET /repos/:owner/:repo/code-scanning/databases", {
+      owner: repositoryNwo.owner,
+      repo: repositoryNwo.repo,
+    });
+  } catch (e) {
+    console.log(e);
+    if (util.isHTTPError(e)) {
+      if (e.status === 404) {
+        logger.debug(
+          "Repository is not opted in to database uploads. Skipping upload."
+        );
+      } else {
+        logger.debug(`Skipping database upload due to unknown error: ${e}`);
+      }
+      return;
+    }
+  }
+
+  const codeql = getCodeQL(config.codeQLCmd);
+  for (const language of config.languages) {
+    // Bundle the database up into a single zip file
+    const databasePath = util.getCodeQLDatabasePath(config, language);
+    const databaseBundlePath = `${databasePath}.zip`;
+    await codeql.databaseBundle(databasePath, databaseBundlePath);
+
+    // Upload the database bundle
+    const payload = fs.readFileSync(databaseBundlePath);
+    try {
+      await client.request(
+        `PUT /repos/:owner/:repo/code-scanning/databases/${language}`,
+        {
+          owner: repositoryNwo.owner,
+          repo: repositoryNwo.repo,
+          data: payload,
+        }
+      );
+      logger.debug(`Successfully uploaded database for ${language}`);
+    } catch (e) {
+      console.log(e);
+      // Log a warning but don't fail the workflow
+      logger.warning(`Failed to upload database for ${language}: ${e}`);
+    }
+  }
+}
+
 async function run() {
   const startedAt = new Date();
   let stats: AnalysisStatusReport | undefined = undefined;
@@ -116,6 +189,11 @@ async function run() {
       logger.info("Not uploading results");
       stats = { ...queriesStats };
     }
+
+    const repositoryNwo = parseRepositoryNwo(
+      util.getRequiredEnvParam("GITHUB_REPOSITORY")
+    );
+    await uploadDatabases(repositoryNwo, config, apiDetails, logger);
   } catch (error) {
     core.setFailed(error.message);
     console.log(error);

src/analyze.test.ts

@@ -2,6 +2,8 @@ import * as fs from "fs";
 import * as path from "path";
 
 import test from "ava";
+import * as yaml from "js-yaml";
+import { clean } from "semver";
 import sinon from "sinon";
 
 import { runQueries } from "./analyze";
@@ -26,20 +28,41 @@ test("status report fields and search path setting", async (t) => {
     return obj;
   }, {});
   sinon.stub(count, "countLoc").resolves(mockLinesOfCode);
-  let searchPathsUsed: string[] = [];
+  let searchPathsUsed: Array<string | undefined> = [];
   return await util.withTmpDir(async (tmpDir) => {
     setupActionsVars(tmpDir, tmpDir);
 
     const memoryFlag = "";
     const addSnippetsFlag = "";
     const threadsFlag = "";
+    const packs = {
+      [Language.cpp]: [
+        {
+          packName: "a/b",
+          version: clean("1.0.0")!,
+        },
+      ],
+      [Language.java]: [
+        {
+          packName: "c/d",
+          version: clean("2.0.0")!,
+        },
+      ],
+    };
 
     for (const language of Object.values(Language)) {
       setCodeQL({
-        databaseAnalyze: async (
-          _,
-          sarifFile: string,
+        packDownload: async () => ({ packs: [] }),
+        databaseRunQueries: async (
+          _db: string,
           searchPath: string | undefined
+        ) => {
+          searchPathsUsed.push(searchPath);
+        },
+        databaseInterpretResults: async (
+          _db: string,
+          _queriesRun: string[],
+          sarifFile: string
         ) => {
           fs.writeFileSync(
             sarifFile,
@@ -75,7 +98,6 @@ test("status report fields and search path setting", async (t) => {
               ],
             })
           );
-          searchPathsUsed.push(searchPath!);
           return "";
         },
       });
@@ -94,6 +116,7 @@ test("status report fields and search path setting", async (t) => {
           type: util.GitHubVariant.DOTCOM,
         } as util.GitHubVersion,
         dbLocation: path.resolve(tmpDir, "codeql_databases"),
+        packs,
       };
       fs.mkdirSync(util.getCodeQLDatabasePath(config, language), {
         recursive: true,
@@ -112,10 +135,32 @@ test("status report fields and search path setting", async (t) => {
         config,
         getRunnerLogger(true)
       );
-      t.deepEqual(Object.keys(builtinStatusReport).length, 1);
-      t.true(
-        `analyze_builtin_queries_${language}_duration_ms` in builtinStatusReport
-      );
+      const hasPacks = language in packs;
+      const statusReportKeys = Object.keys(builtinStatusReport).sort();
+      if (hasPacks) {
+        t.deepEqual(statusReportKeys.length, 3, statusReportKeys.toString());
+        t.deepEqual(
+          statusReportKeys[0],
+          `analyze_builtin_queries_${language}_duration_ms`
+        );
+        t.deepEqual(
+          statusReportKeys[1],
+          `analyze_custom_queries_${language}_duration_ms`
+        );
+        t.deepEqual(
+          statusReportKeys[2],
+          `interpret_results_${language}_duration_ms`
+        );
+      } else {
+        t.deepEqual(
+          statusReportKeys[0],
+          `analyze_builtin_queries_${language}_duration_ms`
+        );
+        t.deepEqual(
+          statusReportKeys[1],
+          `interpret_results_${language}_duration_ms`
+        );
+      }
 
       config.queries[language] = {
         builtin: [],
@@ -139,14 +184,19 @@ test("status report fields and search path setting", async (t) => {
         config,
         getRunnerLogger(true)
       );
-      t.deepEqual(Object.keys(customStatusReport).length, 1);
+      t.deepEqual(Object.keys(customStatusReport).length, 2);
       t.true(
         `analyze_custom_queries_${language}_duration_ms` in customStatusReport
       );
-      t.deepEqual(searchPathsUsed, [undefined, "/1", "/2"]);
+      const expectedSearchPathsUsed = hasPacks
+        ? [undefined, undefined, "/1", "/2", undefined]
+        : [undefined, "/1", "/2"];
+      t.deepEqual(searchPathsUsed, expectedSearchPathsUsed);
+      t.true(`interpret_results_${language}_duration_ms` in customStatusReport);
     }
 
     verifyLineCounts(tmpDir);
+    verifyQuerySuites(tmpDir);
   });
 
   function verifyLineCounts(tmpDir: string) {
@@ -154,12 +204,7 @@ test("status report fields and search path setting", async (t) => {
     Object.keys(Language).forEach((lang, i) => {
       verifyLineCountForFile(
         lang as Language,
-        path.join(tmpDir, `${lang}-builtin.sarif`),
-        i + 1
-      );
-      verifyLineCountForFile(
-        lang as Language,
-        path.join(tmpDir, `${lang}-custom.sarif`),
+        path.join(tmpDir, `${lang}.sarif`),
         i + 1
       );
     });
@@ -188,7 +233,59 @@ test("status report fields and search path setting", async (t) => {
         baseline: lineCount,
       },
     ]);
-    // when the rule doesn't exists, it should not be added
+    // when the rule doesn't exist, it should not be added
     t.deepEqual(sarif.runs[2].properties.metricResults, []);
   }
+
+  function verifyQuerySuites(tmpDir: string) {
+    const qlsContent = [
+      {
+        query: "foo.ql",
+      },
+    ];
+    const qlsContent2 = [
+      {
+        query: "bar.ql",
+      },
+    ];
+    const qlsPackContentCpp = [
+      {
+        qlpack: "a/b",
+        version: "1.0.0",
+      },
+    ];
+    const qlsPackContentJava = [
+      {
+        qlpack: "c/d",
+        version: "2.0.0",
+      },
+    ];
+    for (const lang of Object.values(Language)) {
+      t.deepEqual(readContents(`${lang}-queries-builtin.qls`), qlsContent);
+      t.deepEqual(readContents(`${lang}-queries-custom-0.qls`), qlsContent);
+      t.deepEqual(readContents(`${lang}-queries-custom-1.qls`), qlsContent2);
+      const packSuiteName = `${lang}-queries-packs.qls`;
+      if (lang === Language.cpp) {
+        t.deepEqual(readContents(packSuiteName), qlsPackContentCpp);
+      } else if (lang === Language.java) {
+        t.deepEqual(readContents(packSuiteName), qlsPackContentJava);
+      } else {
+        t.false(
+          fs.existsSync(path.join(tmpDir, "codeql_databases", packSuiteName))
+        );
+      }
+    }
+
+    function readContents(name: string) {
+      const x = fs.readFileSync(
+        path.join(tmpDir, "codeql_databases", name),
+        "utf8"
+      );
+      console.log(x);
+
+      return yaml.safeLoad(
+        fs.readFileSync(path.join(tmpDir, "codeql_databases", name), "utf8")
+      );
+    }
+  }
 });

src/analyze.ts

@@ -10,7 +10,6 @@ import { countLoc, getIdPrefix } from "./count-loc";
 import { isScannedLanguage, Language } from "./languages";
 import { Logger } from "./logging";
 import * as sharedEnv from "./shared-environment";
-import { combineSarifFiles } from "./upload-lib";
 import * as util from "./util";
 
 export class CodeQLAnalysisError extends Error {
@@ -25,34 +24,48 @@ export class CodeQLAnalysisError extends Error {
 }
 
 export interface QueriesStatusReport {
-  // Time taken in ms to analyze builtin queries for cpp (or undefined if this language was not analyzed)
+  // Time taken in ms to run builtin queries for cpp (or undefined if this language was not analyzed)
   analyze_builtin_queries_cpp_duration_ms?: number;
-  // Time taken in ms to analyze builtin queries for csharp (or undefined if this language was not analyzed)
+  // Time taken in ms to run builtin queries for csharp (or undefined if this language was not analyzed)
   analyze_builtin_queries_csharp_duration_ms?: number;
-  // Time taken in ms to analyze builtin queries for go (or undefined if this language was not analyzed)
+  // Time taken in ms to run builtin queries for go (or undefined if this language was not analyzed)
   analyze_builtin_queries_go_duration_ms?: number;
-  // Time taken in ms to analyze builtin queries for java (or undefined if this language was not analyzed)
+  // Time taken in ms to run builtin queries for java (or undefined if this language was not analyzed)
   analyze_builtin_queries_java_duration_ms?: number;
-  // Time taken in ms to analyze builtin queries for javascript (or undefined if this language was not analyzed)
+  // Time taken in ms to run builtin queries for javascript (or undefined if this language was not analyzed)
   analyze_builtin_queries_javascript_duration_ms?: number;
-  // Time taken in ms to analyze builtin queries for python (or undefined if this language was not analyzed)
+  // Time taken in ms to run builtin queries for python (or undefined if this language was not analyzed)
   analyze_builtin_queries_python_duration_ms?: number;
-  // Time taken in ms to analyze builtin queries for ruby (or undefined if this language was not analyzed)
+  // Time taken in ms to run builtin queries for ruby (or undefined if this language was not analyzed)
   analyze_builtin_queries_ruby_duration_ms?: number;
-  // Time taken in ms to analyze custom queries for cpp (or undefined if this language was not analyzed)
+  // Time taken in ms to run custom queries for cpp (or undefined if this language was not analyzed)
   analyze_custom_queries_cpp_duration_ms?: number;
-  // Time taken in ms to analyze custom queries for csharp (or undefined if this language was not analyzed)
+  // Time taken in ms to run custom queries for csharp (or undefined if this language was not analyzed)
   analyze_custom_queries_csharp_duration_ms?: number;
-  // Time taken in ms to analyze custom queries for go (or undefined if this language was not analyzed)
+  // Time taken in ms to run custom queries for go (or undefined if this language was not analyzed)
   analyze_custom_queries_go_duration_ms?: number;
-  // Time taken in ms to analyze custom queries for java (or undefined if this language was not analyzed)
+  // Time taken in ms to run custom queries for java (or undefined if this language was not analyzed)
   analyze_custom_queries_java_duration_ms?: number;
-  // Time taken in ms to analyze custom queries for javascript (or undefined if this language was not analyzed)
+  // Time taken in ms to run custom queries for javascript (or undefined if this language was not analyzed)
   analyze_custom_queries_javascript_duration_ms?: number;
-  // Time taken in ms to analyze custom queries for python (or undefined if this language was not analyzed)
+  // Time taken in ms to run custom queries for python (or undefined if this language was not analyzed)
   analyze_custom_queries_python_duration_ms?: number;
-  // Time taken in ms to analyze custom queries for ruby (or undefined if this language was not analyzed)
+  // Time taken in ms to run custom queries for ruby (or undefined if this language was not analyzed)
   analyze_custom_queries_ruby_duration_ms?: number;
+  // Time taken in ms to interpret results for cpp (or undefined if this language was not analyzed)
+  interpret_results_cpp_duration_ms?: number;
+  // Time taken in ms to interpret results for csharp (or undefined if this language was not analyzed)
+  interpret_results_csharp_duration_ms?: number;
+  // Time taken in ms to interpret results for go (or undefined if this language was not analyzed)
+  interpret_results_go_duration_ms?: number;
+  // Time taken in ms to interpret results for java (or undefined if this language was not analyzed)
+  interpret_results_java_duration_ms?: number;
+  // Time taken in ms to interpret results for javascript (or undefined if this language was not analyzed)
+  interpret_results_javascript_duration_ms?: number;
+  // Time taken in ms to interpret results for python (or undefined if this language was not analyzed)
+  interpret_results_python_duration_ms?: number;
+  // Time taken in ms to interpret results for ruby (or undefined if this language was not analyzed)
+  interpret_results_ruby_duration_ms?: number;
   // Name of language that errored during analysis (or undefined if no language failed)
   analyze_failure_language?: string;
 }
@@ -163,86 +176,103 @@ export async function runQueries(
   );
 
   for (const language of config.languages) {
-    logger.startGroup(`Analyzing ${language}`);
-
     const queries = config.queries[language];
-    if (
-      queries === undefined ||
-      (queries.builtin.length === 0 && queries.custom.length === 0)
-    ) {
+    const packsWithVersion = config.packs[language] || [];
+
+    const hasBuiltinQueries = queries?.builtin.length > 0;
+    const hasCustomQueries = queries?.custom.length > 0;
+    const hasPackWithCustomQueries = packsWithVersion.length > 0;
+
+    if (!hasBuiltinQueries && !hasCustomQueries && !hasPackWithCustomQueries) {
       throw new Error(
         `Unable to analyse ${language} as no queries were selected for this language`
       );
     }
 
     try {
-      let analysisSummaryBuiltIn = "";
-      const customAnalysisSummaries: string[] = [];
+      if (hasPackWithCustomQueries) {
+        logger.info("*************");
+        logger.info(
+          "Performing analysis with custom QL Packs. QL Packs are an experimental feature."
+        );
+        logger.info("And should not be used in production yet.");
+        logger.info("*************");
+        logger.startGroup(`Downloading custom packs for ${language}`);
+
+        const codeql = getCodeQL(config.codeQLCmd);
+        const results = await codeql.packDownload(packsWithVersion);
+        logger.info(
+          `Downloaded packs: ${results.packs
+            .map((r) => `${r.name}@${r.version || "latest"}`)
+            .join(", ")}`
+        );
+
+        logger.endGroup();
+      }
+
+      logger.startGroup(`Running queries for ${language}`);
+      const querySuitePaths: string[] = [];
       if (queries["builtin"].length > 0) {
         const startTimeBuiltIn = new Date().getTime();
-        const { sarifFile, stdout } = await runQueryGroup(
-          language,
-          "builtin",
-          queries["builtin"],
-          sarifFolder,
-          undefined
+        querySuitePaths.push(
+          await runQueryGroup(
+            language,
+            "builtin",
+            createQuerySuiteContents(queries["builtin"]),
+            undefined
+          )
         );
-        analysisSummaryBuiltIn = stdout;
-        await injectLinesOfCode(sarifFile, language, locPromise);
-
         statusReport[`analyze_builtin_queries_${language}_duration_ms`] =
           new Date().getTime() - startTimeBuiltIn;
       }
       const startTimeCustom = new Date().getTime();
-      const temporarySarifDir = config.tempDir;
-      const temporarySarifFiles: string[] = [];
+      let ranCustom = false;
       for (let i = 0; i < queries["custom"].length; ++i) {
         if (queries["custom"][i].queries.length > 0) {
-          const { sarifFile, stdout } = await runQueryGroup(
-            language,
-            `custom-${i}`,
-            queries["custom"][i].queries,
-            temporarySarifDir,
-            queries["custom"][i].searchPath
+          querySuitePaths.push(
+            await runQueryGroup(
+              language,
+              `custom-${i}`,
+              createQuerySuiteContents(queries["custom"][i].queries),
+              queries["custom"][i].searchPath
+            )
           );
-          customAnalysisSummaries.push(stdout);
-          temporarySarifFiles.push(sarifFile);
+          ranCustom = true;
         }
       }
-      if (temporarySarifFiles.length > 0) {
-        const sarifFile = path.join(sarifFolder, `${language}-custom.sarif`);
-        fs.writeFileSync(sarifFile, combineSarifFiles(temporarySarifFiles));
-        await injectLinesOfCode(sarifFile, language, locPromise);
-
+      if (packsWithVersion.length > 0) {
+        querySuitePaths.push(
+          await runQueryGroup(
+            language,
+            "packs",
+            createPackSuiteContents(packsWithVersion),
+            undefined
+          )
+        );
+        ranCustom = true;
+      }
+      if (ranCustom) {
         statusReport[`analyze_custom_queries_${language}_duration_ms`] =
           new Date().getTime() - startTimeCustom;
       }
       logger.endGroup();
-
-      // Print the LoC baseline and the summary results from database analyze for the standard
-      // query suite and (if appropriate) each custom query suite.
-      logger.startGroup(`Analysis summary for ${language}`);
-
-      printLinesOfCodeSummary(logger, language, await locPromise);
-      logger.info(analysisSummaryBuiltIn);
-
-      for (const [i, customSummary] of customAnalysisSummaries.entries()) {
-        if (customSummary.trim() === "") {
-          continue;
-        }
-        const description =
-          customAnalysisSummaries.length === 1
-            ? "custom queries"
-            : `custom query suite ${i + 1}/${customAnalysisSummaries.length}`;
-        logger.info(`Analysis summary for ${description}:`);
-        logger.info("");
-        logger.info(customSummary);
-        logger.info("");
-      }
-
+      logger.startGroup(`Interpreting results for ${language}`);
+      const startTimeInterpretResults = new Date().getTime();
+      const sarifFile = path.join(sarifFolder, `${language}.sarif`);
+      const analysisSummary = await runInterpretResults(
+        language,
+        querySuitePaths,
+        sarifFile
+      );
+      await injectLinesOfCode(sarifFile, language, locPromise);
+      statusReport[`interpret_results_${language}_duration_ms`] =
+        new Date().getTime() - startTimeInterpretResults;
       logger.endGroup();
+      logger.info(analysisSummary);
+      printLinesOfCodeSummary(logger, language, await locPromise);
     } catch (e) {
       logger.info(e);
+      logger.info(e.stack);
       statusReport.analyze_failure_language = language;
       throw new CodeQLAnalysisError(
         statusReport,
@@ -253,42 +283,70 @@ export async function runQueries(
 
   return statusReport;
 
-  async function runQueryGroup(
+  async function runInterpretResults(
     language: Language,
-    type: string,
     queries: string[],
-    destinationFolder: string,
-    searchPath: string | undefined
-  ): Promise<{ sarifFile: string; stdout: string }> {
+    sarifFile: string
+  ): Promise<string> {
     const databasePath = util.getCodeQLDatabasePath(config, language);
-    // Pass the queries to codeql using a file instead of using the command
-    // line to avoid command line length restrictions, particularly on windows.
-    const querySuitePath = `${databasePath}-queries-${type}.qls`;
-    const querySuiteContents = queries
-      .map((q: string) => `- query: ${q}`)
-      .join("\n");
-    fs.writeFileSync(querySuitePath, querySuiteContents);
-    logger.debug(`Query suite file for ${language}...\n${querySuiteContents}`);
-
-    const sarifFile = path.join(destinationFolder, `${language}-${type}.sarif`);
-
     const codeql = getCodeQL(config.codeQLCmd);
-    const databaseAnalyzeStdout = await codeql.databaseAnalyze(
+    return await codeql.databaseInterpretResults(
       databasePath,
+      queries,
       sarifFile,
-      searchPath,
-      querySuitePath,
-      memoryFlag,
       addSnippetsFlag,
       threadsFlag,
       automationDetailsId
     );
-
-    logger.debug(
-      `SARIF results for database ${language} created at "${sarifFile}"`
-    );
-    return { sarifFile, stdout: databaseAnalyzeStdout };
   }
+
+  async function runQueryGroup(
+    language: Language,
+    type: string,
+    querySuiteContents: string,
+    searchPath: string | undefined
+  ): Promise<string> {
+    const databasePath = util.getCodeQLDatabasePath(config, language);
+    // Pass the queries to codeql using a file instead of using the command
+    // line to avoid command line length restrictions, particularly on windows.
+    const querySuitePath = `${databasePath}-queries-${type}.qls`;
+    fs.writeFileSync(querySuitePath, querySuiteContents);
+    logger.debug(
+      `Query suite file for ${language}-${type}...\n${querySuiteContents}`
+    );
+
+    const codeql = getCodeQL(config.codeQLCmd);
+    await codeql.databaseRunQueries(
+      databasePath,
+      searchPath,
+      querySuitePath,
+      memoryFlag,
+      threadsFlag
+    );
+
+    logger.debug(`BQRS results produced for ${language} (queries: ${type})"`);
+    return querySuitePath;
+  }
+}
+
+function createQuerySuiteContents(queries: string[]) {
+  return queries.map((q: string) => `- query: ${q}`).join("\n");
+}
+
+function createPackSuiteContents(
+  packsWithVersion: configUtils.PackWithVersion[]
+) {
+  return packsWithVersion.map(packWithVersionToQuerySuiteEntry).join("\n");
+}
+
+function packWithVersionToQuerySuiteEntry(
+  pack: configUtils.PackWithVersion
+): string {
+  let text = `- qlpack: ${pack.packName}`;
+  if (pack.version) {
+    text += `\n  version: ${pack.version}`;
+  }
+  return text;
 }
 
 export async function runAnalyze(
@@ -305,10 +363,8 @@ export async function runAnalyze(
 
   fs.mkdirSync(outputDir, { recursive: true });
 
-  logger.info("Finalizing database creation");
   await finalizeDatabaseCreation(config, threadsFlag, logger);
 
-  logger.info("Analyzing database");
   const queriesStats = await runQueries(
     outputDir,
     memoryFlag,
@@ -327,12 +383,13 @@ export async function runCleanup(
   cleanupLevel: string,
   logger: Logger
 ): Promise<void> {
-  logger.info("Cleaning up databases...");
+  logger.startGroup("Cleaning up databases");
   for (const language of config.languages) {
     const codeql = getCodeQL(config.codeQLCmd);
     const databasePath = util.getCodeQLDatabasePath(config, language);
     await codeql.databaseCleanup(databasePath, cleanupLevel);
   }
+  logger.endGroup();
 }
 
 async function injectLinesOfCode(
@@ -370,7 +427,7 @@ function printLinesOfCodeSummary(
 ) {
   if (language in lineCounts) {
     logger.info(
-      `Counted ${lineCounts[language]} lines of code for ${language} as a baseline.`
+      `Counted a baseline of ${lineCounts[language]} lines of code for ${language}.`
     );
   }
 }

src/codeql.ts

@@ -13,6 +13,7 @@ import { v4 as uuidV4 } from "uuid";
 
 import { isRunningLocalAction, getRelativeScriptPath } from "./actions-util";
 import * as api from "./api-client";
+import { PackWithVersion } from "./config-utils";
 import * as defaults from "./defaults.json"; // Referenced from codeql-action-sync-tool!
 import { errorMatchers } from "./error-matcher";
 import { Language } from "./languages";
@@ -88,23 +89,41 @@ export interface CodeQL {
     queries: string[],
     extraSearchPath: string | undefined
   ): Promise<ResolveQueriesOutput>;
+
   /**
-   * Run 'codeql database analyze'.
+   * Run 'codeql pack download'.
    */
-  databaseAnalyze(
-    databasePath: string,
-    sarifFile: string,
-    extraSearchPath: string | undefined,
-    querySuite: string,
-    memoryFlag: string,
-    addSnippetsFlag: string,
-    threadsFlag: string,
-    automationDetailsId: string | undefined
-  ): Promise<string>;
+  packDownload(packs: PackWithVersion[]): Promise<PackDownloadOutput>;
+
   /**
    * Run 'codeql database cleanup'.
    */
   databaseCleanup(databasePath: string, cleanupLevel: string): Promise<void>;
+  /**
+   * Run 'codeql database bundle'.
+   */
+  databaseBundle(databasePath: string, outputFilePath: string): Promise<void>;
+  /**
+   * Run 'codeql database run-queries'.
+   */
+  databaseRunQueries(
+    databasePath: string,
+    extraSearchPath: string | undefined,
+    querySuitePath: string,
+    memoryFlag: string,
+    threadsFlag: string
+  ): Promise<void>;
+  /**
+   * Run 'codeql database interpret-results'.
+   */
+  databaseInterpretResults(
+    databasePath: string,
+    querySuitePaths: string[],
+    sarifFile: string,
+    addSnippetsFlag: string,
+    threadsFlag: string,
+    automationDetailsId: string | undefined
+  ): Promise<string>;
 }
 
 export interface ResolveLanguagesOutput {
@@ -125,6 +144,17 @@ export interface ResolveQueriesOutput {
   };
 }
 
+export interface PackDownloadOutput {
+  packs: PackDownloadItem[];
+}
+
+interface PackDownloadItem {
+  name: string;
+  version: string;
+  packDir: string;
+  installResult: string;
+}
+
 /**
  * Stores the CodeQL object, and is populated by `setupCodeQL` or `getCodeQL`.
  * Can be overridden in tests using `setCodeQL`.
@@ -484,8 +514,14 @@ export function setCodeQL(partialCodeql: Partial<CodeQL>): CodeQL {
     finalizeDatabase: resolveFunction(partialCodeql, "finalizeDatabase"),
     resolveLanguages: resolveFunction(partialCodeql, "resolveLanguages"),
     resolveQueries: resolveFunction(partialCodeql, "resolveQueries"),
-    databaseAnalyze: resolveFunction(partialCodeql, "databaseAnalyze"),
+    packDownload: resolveFunction(partialCodeql, "packDownload"),
     databaseCleanup: resolveFunction(partialCodeql, "databaseCleanup"),
+    databaseBundle: resolveFunction(partialCodeql, "databaseBundle"),
+    databaseRunQueries: resolveFunction(partialCodeql, "databaseRunQueries"),
+    databaseInterpretResults: resolveFunction(
+      partialCodeql,
+      "databaseInterpretResults"
+    ),
   };
   return cachedCodeQL;
 }
@@ -510,7 +546,7 @@ function getCodeQLForCmd(cmd: string): CodeQL {
       return cmd;
     },
     async printVersion() {
-      await new toolrunner.ToolRunner(cmd, ["version", "--format=json"]).exec();
+      await runTool(cmd, ["version", "--format=json"]);
     },
     async getTracerEnv(databasePath: string) {
       // Write tracer-env.js to a temp location.
@@ -551,7 +587,7 @@ function getCodeQLForCmd(cmd: string): CodeQL {
       // _and_ is present in the latest supported CLI release.)
       const envFile = path.resolve(databasePath, "working", "env.tmp");
 
-      await new toolrunner.ToolRunner(cmd, [
+      await runTool(cmd, [
         "database",
         "trace-command",
         databasePath,
@@ -559,7 +595,7 @@ function getCodeQLForCmd(cmd: string): CodeQL {
         process.execPath,
         tracerEnvJs,
         envFile,
-      ]).exec();
+      ]);
       return JSON.parse(fs.readFileSync(envFile, "utf-8"));
     },
     async databaseInit(
@@ -567,14 +603,14 @@ function getCodeQLForCmd(cmd: string): CodeQL {
       language: Language,
       sourceRoot: string
     ) {
-      await new toolrunner.ToolRunner(cmd, [
+      await runTool(cmd, [
         "database",
         "init",
         databasePath,
         `--language=${language}`,
         `--source-root=${sourceRoot}`,
         ...getExtraOptionsFromEnv(["database", "init"]),
-      ]).exec();
+      ]);
     },
     async runAutobuild(language: Language) {
       const cmdName =
@@ -598,7 +634,7 @@ function getCodeQLForCmd(cmd: string): CodeQL {
         "-Dmaven.wagon.http.pool=false",
       ].join(" ");
 
-      await new toolrunner.ToolRunner(autobuildCmd).exec();
+      await runTool(autobuildCmd);
     },
     async extractScannedLanguage(databasePath: string, language: Language) {
       // Get extractor location
@@ -653,6 +689,7 @@ function getCodeQLForCmd(cmd: string): CodeQL {
         [
           "database",
           "finalize",
+          "--finalize-dataset",
           threadsFlag,
           ...getExtraOptionsFromEnv(["database", "finalize"]),
           databasePath,
@@ -662,14 +699,7 @@ function getCodeQLForCmd(cmd: string): CodeQL {
     },
     async resolveLanguages() {
       const codeqlArgs = ["resolve", "languages", "--format=json"];
-      let output = "";
-      await new toolrunner.ToolRunner(cmd, codeqlArgs, {
-        listeners: {
-          stdout: (data: Buffer) => {
-            output += data.toString();
-          },
-        },
-      }).exec();
+      const output = await runTool(cmd, codeqlArgs);
 
       try {
         return JSON.parse(output);
@@ -693,14 +723,7 @@ function getCodeQLForCmd(cmd: string): CodeQL {
       if (extraSearchPath !== undefined) {
         codeqlArgs.push("--additional-packs", extraSearchPath);
       }
-      let output = "";
-      await new toolrunner.ToolRunner(cmd, codeqlArgs, {
-        listeners: {
-          stdout: (data: Buffer) => {
-            output += data.toString();
-          },
-        },
-      }).exec();
+      const output = await runTool(cmd, codeqlArgs);
 
       try {
         return JSON.parse(output);
@@ -708,66 +731,127 @@ function getCodeQLForCmd(cmd: string): CodeQL {
         throw new Error(`Unexpected output from codeql resolve queries: ${e}`);
       }
     },
-    async databaseAnalyze(
+    async databaseRunQueries(
       databasePath: string,
-      sarifFile: string,
       extraSearchPath: string | undefined,
-      querySuite: string,
+      querySuitePath: string,
       memoryFlag: string,
-      addSnippetsFlag: string,
-      threadsFlag: string,
-      automationDetailsId: string | undefined
-    ): Promise<string> {
-      const args = [
+      threadsFlag: string
+    ): Promise<void> {
+      const codeqlArgs = [
         "database",
-        "analyze",
+        "run-queries",
         memoryFlag,
         threadsFlag,
         databasePath,
         "--min-disk-free=1024", // Try to leave at least 1GB free
-        "--format=sarif-latest",
-        "--sarif-multicause-markdown",
-        "--sarif-group-rules-by-pack",
-        `--output=${sarifFile}`,
-        addSnippetsFlag,
-        // Enable progress verbosity so we log each query as it's interpreted. This aids debugging
-        // when interpretation takes a while for one of the queries being analyzed.
         "-v",
-        ...getExtraOptionsFromEnv(["database", "analyze"]),
+        ...getExtraOptionsFromEnv(["database", "run-queries"]),
       ];
       if (extraSearchPath !== undefined) {
-        args.push("--additional-packs", extraSearchPath);
+        codeqlArgs.push("--additional-packs", extraSearchPath);
       }
+      codeqlArgs.push(querySuitePath);
+      await runTool(cmd, codeqlArgs);
+    },
+    async databaseInterpretResults(
+      databasePath: string,
+      querySuitePaths: string[],
+      sarifFile: string,
+      addSnippetsFlag: string,
+      threadsFlag: string,
+      automationDetailsId: string | undefined
+    ): Promise<string> {
+      const codeqlArgs = [
+        "database",
+        "interpret-results",
+        threadsFlag,
+        "--format=sarif-latest",
+        "--print-metrics-summary",
+        "--sarif-group-rules-by-pack",
+        "-v",
+        `--output=${sarifFile}`,
+        addSnippetsFlag,
+        ...getExtraOptionsFromEnv(["database", "interpret-results"]),
+      ];
       if (automationDetailsId !== undefined) {
-        args.push("--sarif-category", automationDetailsId);
+        codeqlArgs.push("--sarif-category", automationDetailsId);
       }
-      args.push(querySuite);
+      codeqlArgs.push(databasePath, ...querySuitePaths);
       // capture stdout, which contains analysis summaries
-      let output = "";
-      await new toolrunner.ToolRunner(cmd, args, {
-        listeners: {
-          stdout: (data: Buffer) => {
-            output += data.toString("utf8");
-          },
-        },
-      }).exec();
-      return output;
+      return await runTool(cmd, codeqlArgs);
+    },
+
+    /**
+     * Download specified packs into the package cache. If the specified
+     * package and version already exists (e.g., from a previous analysis run),
+     * then it is not downloaded again (unless the extra option `--force` is
+     * specified).
+     *
+     * If no version is specified, then the latest version is
+     * downloaded. The check to determine what the latest version is is done
+     * each time this package is requested.
+     */
+    async packDownload(packs: PackWithVersion[]): Promise<PackDownloadOutput> {
+      const codeqlArgs = [
+        "pack",
+        "download",
+        "--format=json",
+        ...getExtraOptionsFromEnv(["pack", "download"]),
+        ...packs.map(packWithVersionToString),
+      ];
+
+      const output = await runTool(cmd, codeqlArgs);
+
+      try {
+        const parsedOutput: PackDownloadOutput = JSON.parse(output);
+        if (
+          Array.isArray(parsedOutput.packs) &&
+          // TODO PackDownloadOutput will not include the version if it is not specified
+          // in the input. The version is always the latest version available.
+          // It should be added to the output, but this requires a CLI change
+          parsedOutput.packs.every((p) => p.name /* && p.version */)
+        ) {
+          return parsedOutput;
+        } else {
+          throw new Error("Unexpected output from pack download");
+        }
+      } catch (e) {
+        throw new Error(
+          `Attempted to download specified packs but got an error:\n${output}\n${e}`
+        );
+      }
     },
     async databaseCleanup(
       databasePath: string,
       cleanupLevel: string
     ): Promise<void> {
-      const args = [
+      const codeqlArgs = [
         "database",
         "cleanup",
         databasePath,
         `--mode=${cleanupLevel}`,
       ];
+      await runTool(cmd, codeqlArgs);
+    },
+    async databaseBundle(
+      databasePath: string,
+      outputFilePath: string
+    ): Promise<void> {
+      const args = [
+        "database",
+        "bundle",
+        databasePath,
+        `--output=${outputFilePath}`,
+      ];
       await new toolrunner.ToolRunner(cmd, args).exec();
     },
   };
 }
 
+function packWithVersionToString(pack: PackWithVersion): string {
+  return pack.version ? `${pack.packName}@${pack.version}` : pack.packName;
+}
 /**
  * Gets the options for `path` of `options` as an array of extra option strings.
  */
@@ -827,3 +911,15 @@ export function getExtraOptions(
         );
   return all.concat(specific);
 }
+
+async function runTool(cmd: string, args: string[] = []) {
+  let output = "";
+  await new toolrunner.ToolRunner(cmd, args, {
+    listeners: {
+      stdout: (data: Buffer) => {
+        output += data.toString();
+      },
+    },
+  }).exec();
+  return output;
+}

src/config-utils.test.ts

@@ -2,7 +2,8 @@ import * as fs from "fs";
 import * as path from "path";
 
 import * as github from "@actions/github";
-import test from "ava";
+import test, { ExecutionContext } from "ava";
+import { clean } from "semver";
 import sinon from "sinon";
 
 import * as api from "./api-client";
@@ -318,6 +319,7 @@ test("load non-empty input", async (t) => {
       codeQLCmd: codeQL.getPath(),
       gitHubVersion,
       dbLocation: path.resolve(tmpDir, "codeql_databases"),
+      packs: {} as configUtils.Packs,
     };
 
     const languages = "javascript";
@@ -983,6 +985,137 @@ test("Unknown languages", async (t) => {
   });
 });
 
+test("Config specifies packages", async (t) => {
+  return await util.withTmpDir(async (tmpDir) => {
+    const codeQL = setCodeQL({
+      async resolveQueries() {
+        return {
+          byLanguage: {},
+          noDeclaredLanguage: {},
+          multipleDeclaredLanguages: {},
+        };
+      },
+    });
+
+    const inputFileContents = `
+      name: my config
+      disable-default-queries: true
+      packs:
+        - a/b@1.2.3
+      `;
+
+    const configFile = path.join(tmpDir, "codeql-config.yaml");
+    fs.writeFileSync(configFile, inputFileContents);
+
+    const languages = "javascript";
+
+    const { packs } = await configUtils.initConfig(
+      languages,
+      undefined,
+      configFile,
+      undefined,
+      { owner: "github", repo: "example " },
+      tmpDir,
+      tmpDir,
+      codeQL,
+      tmpDir,
+      gitHubVersion,
+      sampleApiDetails,
+      getRunnerLogger(true)
+    );
+    t.deepEqual(packs as unknown, {
+      [Language.javascript]: [
+        {
+          packName: "a/b",
+          version: clean("1.2.3"),
+        },
+      ],
+    });
+  });
+});
+
+test("Config specifies packages for multiple languages", async (t) => {
+  return await util.withTmpDir(async (tmpDir) => {
+    const codeQL = setCodeQL({
+      async resolveQueries() {
+        return {
+          byLanguage: {
+            cpp: { "/foo/a.ql": {} },
+          },
+          noDeclaredLanguage: {},
+          multipleDeclaredLanguages: {},
+        };
+      },
+    });
+
+    const inputFileContents = `
+      name: my config
+      disable-default-queries: true
+      queries:
+      - uses: ./foo
+      packs:
+        javascript:
+          - a/b@1.2.3
+        python:
+          - c/d@1.2.3
+      `;
+
+    const configFile = path.join(tmpDir, "codeql-config.yaml");
+    fs.writeFileSync(configFile, inputFileContents);
+    fs.mkdirSync(path.join(tmpDir, "foo"));
+
+    const languages = "javascript,python,cpp";
+
+    const { packs, queries } = await configUtils.initConfig(
+      languages,
+      undefined,
+      configFile,
+      undefined,
+      { owner: "github", repo: "example" },
+      tmpDir,
+      tmpDir,
+      codeQL,
+      tmpDir,
+      gitHubVersion,
+      sampleApiDetails,
+      getRunnerLogger(true)
+    );
+    t.deepEqual(packs as unknown, {
+      [Language.javascript]: [
+        {
+          packName: "a/b",
+          version: clean("1.2.3"),
+        },
+      ],
+      [Language.python]: [
+        {
+          packName: "c/d",
+          version: clean("1.2.3"),
+        },
+      ],
+    });
+    t.deepEqual(queries, {
+      cpp: {
+        builtin: [],
+        custom: [
+          {
+            queries: ["/foo/a.ql"],
+            searchPath: tmpDir,
+          },
+        ],
+      },
+      javascript: {
+        builtin: [],
+        custom: [],
+      },
+      python: {
+        builtin: [],
+        custom: [],
+      },
+    });
+  });
+});
+
 function doInvalidInputTest(
   testName: string,
   inputFileContents: string,
@@ -1177,3 +1310,109 @@ test("path sanitisation", (t) => {
     "foo/"
   );
 });
+
+/**
+ * Test macro for ensuring the packs block is valid
+ */
+function parsePacksMacro(
+  t: ExecutionContext<unknown>,
+  packsByLanguage: string[] | Record<string, string[]>,
+  languages: Language[],
+  expected
+) {
+  t.deepEqual(
+    configUtils.parsePacks(packsByLanguage, languages, "/a/b"),
+    expected
+  );
+}
+parsePacksMacro.title = (providedTitle: string) =>
+  `Parse Packs: ${providedTitle}`;
+
+/**
+ * Test macro for testing when the packs block is invalid
+ */
+function parsePacksErrorMacro(
+  t: ExecutionContext<unknown>,
+  packsByLanguage,
+  languages: Language[],
+  expected: RegExp
+) {
+  t.throws(
+    () => {
+      configUtils.parsePacks(packsByLanguage, languages, "/a/b");
+    },
+    {
+      message: expected,
+    }
+  );
+}
+parsePacksErrorMacro.title = (providedTitle: string) =>
+  `Parse Packs Error: ${providedTitle}`;
+
+function invalidPackNameMacro(t: ExecutionContext<unknown>, name: string) {
+  parsePacksErrorMacro(
+    t,
+    { [Language.cpp]: [name] },
+    [Language.cpp],
+    new RegExp(
+      `The configuration file "/a/b" is invalid: property "packs" "${name}" is not a valid pack`
+    )
+  );
+}
+invalidPackNameMacro.title = (_: string, arg: string) =>
+  `Invalid pack string: ${arg}`;
+
+test("no packs", parsePacksMacro, {}, [], {});
+test("two packs", parsePacksMacro, ["a/b", "c/d@1.2.3"], [Language.cpp], {
+  [Language.cpp]: [
+    { packName: "a/b", version: undefined },
+    { packName: "c/d", version: clean("1.2.3") },
+  ],
+});
+test(
+  "two packs with language",
+  parsePacksMacro,
+  {
+    [Language.cpp]: ["a/b", "c/d@1.2.3"],
+    [Language.java]: ["d/e", "f/g@1.2.3"],
+  },
+  [Language.cpp, Language.java, Language.csharp],
+  {
+    [Language.cpp]: [
+      { packName: "a/b", version: undefined },
+      { packName: "c/d", version: clean("1.2.3") },
+    ],
+    [Language.java]: [
+      { packName: "d/e", version: undefined },
+      { packName: "f/g", version: clean("1.2.3") },
+    ],
+  }
+);
+
+test(
+  "no language",
+  parsePacksErrorMacro,
+  ["a/b@1.2.3"],
+  [Language.java, Language.python],
+  /The configuration file "\/a\/b" is invalid: property "packs" must split packages by language/
+);
+test(
+  "invalid language",
+  parsePacksErrorMacro,
+  { [Language.java]: ["c/d"] },
+  [Language.cpp],
+  /The configuration file "\/a\/b" is invalid: property "packs" has "java", but it is not one of the languages to analyze/
+);
+test(
+  "not an array",
+  parsePacksErrorMacro,
+  { [Language.cpp]: "c/d" },
+  [Language.cpp],
+  /The configuration file "\/a\/b" is invalid: property "packs" must be an array of non-empty strings/
+);
+
+test(invalidPackNameMacro, "c"); // all packs require at least a scope and a name
+test(invalidPackNameMacro, "c-/d");
+test(invalidPackNameMacro, "-c/d");
+test(invalidPackNameMacro, "c/d_d");
+test(invalidPackNameMacro, "c/d@x");

src/config-utils.ts

@@ -2,6 +2,7 @@ import * as fs from "fs";
 import * as path from "path";
 
 import * as yaml from "js-yaml";
+import * as semver from "semver";
 
 import * as api from "./api-client";
 import { CodeQL, ResolveQueriesOutput } from "./codeql";
@@ -18,6 +19,7 @@ const QUERIES_PROPERTY = "queries";
 const QUERIES_USES_PROPERTY = "uses";
 const PATHS_IGNORE_PROPERTY = "paths-ignore";
 const PATHS_PROPERTY = "paths";
+const PACKS_PROPERTY = "packs";
 
 /**
  * Format of the config file supplied by the user.
@@ -31,6 +33,11 @@ export interface UserConfig {
   }>;
   "paths-ignore"?: string[];
   paths?: string[];
+
+  // If this is a multi-language analysis, then the packages must be split by
+  // language. If this is a single language analysis, then no split by
+  // language is necessary.
+  packs?: Record<string, string[]> | string[];
 }
 
 /**
@@ -114,6 +121,19 @@ export interface Config {
    * The location where CodeQL databases should be stored.
    */
   dbLocation: string;
+  /**
+   * List of packages, separated by language to download before any analysis.
+   */
+  packs: Packs;
+}
+
+export type Packs = Partial<Record<Language, PackWithVersion[]>>;
+
+export interface PackWithVersion {
+  /** qualified name of a package reference */
+  packName: string;
+  /** version of the package, or undefined, which means latest version */
+  version?: string;
 }
 
 /**
@@ -536,6 +556,44 @@ export function getPathsInvalid(configFile: string): string {
   );
 }
 
+export function getPacksRequireLanguage(
+  lang: string,
+  configFile: string
+): string {
+  return getConfigFilePropertyError(
+    configFile,
+    PACKS_PROPERTY,
+    `has "${lang}", but it is not one of the languages to analyze`
+  );
+}
+
+export function getPacksInvalidSplit(configFile: string): string {
+  return getConfigFilePropertyError(
+    configFile,
+    PACKS_PROPERTY,
+    "must split packages by language"
+  );
+}
+
+export function getPacksInvalid(configFile: string): string {
+  return getConfigFilePropertyError(
+    configFile,
+    PACKS_PROPERTY,
+    "must be an array of non-empty strings"
+  );
+}
+
+export function getPacksStrInvalid(
+  packStr: string,
+  configFile: string
+): string {
+  return getConfigFilePropertyError(
+    configFile,
+    PACKS_PROPERTY,
+    `"${packStr}" is not a valid pack`
+  );
+}
+
 export function getLocalPathOutsideOfRepository(
   configFile: string | undefined,
   localPath: string
@@ -787,6 +845,7 @@ export async function getDefaultConfig(
     queries,
     pathsIgnore: [],
     paths: [],
+    packs: {},
     originalUserInput: {},
     tempDir,
     toolCacheDir,
@@ -883,10 +942,11 @@ async function loadConfig(
     shouldAddConfigFileQueries(queriesInput) &&
     QUERIES_PROPERTY in parsedYAML
   ) {
-    if (!(parsedYAML[QUERIES_PROPERTY] instanceof Array)) {
+    const queriesArr = parsedYAML[QUERIES_PROPERTY];
+    if (!Array.isArray(queriesArr)) {
       throw new Error(getQueriesInvalid(configFile));
     }
-    for (const query of parsedYAML[QUERIES_PROPERTY]!) {
+    for (const query of queriesArr) {
       if (
         !(QUERIES_USES_PROPERTY in query) ||
         typeof query[QUERIES_USES_PROPERTY] !== "string"
@@ -908,7 +968,7 @@ async function loadConfig(
   }
 
   if (PATHS_IGNORE_PROPERTY in parsedYAML) {
-    if (!(parsedYAML[PATHS_IGNORE_PROPERTY] instanceof Array)) {
+    if (!Array.isArray(parsedYAML[PATHS_IGNORE_PROPERTY])) {
       throw new Error(getPathsIgnoreInvalid(configFile));
     }
     for (const ignorePath of parsedYAML[PATHS_IGNORE_PROPERTY]!) {
@@ -927,7 +987,7 @@ async function loadConfig(
   }
 
   if (PATHS_PROPERTY in parsedYAML) {
-    if (!(parsedYAML[PATHS_PROPERTY] instanceof Array)) {
+    if (!Array.isArray(parsedYAML[PATHS_PROPERTY])) {
       throw new Error(getPathsInvalid(configFile));
     }
     for (const includePath of parsedYAML[PATHS_PROPERTY]!) {
@@ -940,11 +1000,18 @@ async function loadConfig(
     }
   }
 
+  const packs = parsePacks(
+    parsedYAML[PACKS_PROPERTY] ?? {},
+    languages,
+    configFile
+  );
+
   return {
     languages,
     queries,
     pathsIgnore,
     paths,
+    packs,
     originalUserInput: parsedYAML,
     tempDir,
     toolCacheDir,
@@ -954,6 +1021,78 @@ async function loadConfig(
   };
 }
 
+/**
+ * Pack names must be in the form of `scope/name`, with only alpha-numeric characters,
+ * and `-` allowed as long as not the first or last char.
+ **/
+const PACK_IDENTIFIER_PATTERN = (function () {
+  const alphaNumeric = "[a-z0-9]";
+  const alphaNumericDash = "[a-z0-9-]";
+  const component = `${alphaNumeric}(${alphaNumericDash}*${alphaNumeric})?`;
+  return new RegExp(`^${component}/${component}$`);
+})();
+
+// Exported for testing
+export function parsePacks(
+  packsByLanguage: string[] | Record<string, string[]>,
+  languages: Language[],
+  configFile: string
+): Packs {
+  const packs = {};
+
+  if (Array.isArray(packsByLanguage)) {
+    if (languages.length === 1) {
+      // single language analysis, so language is implicit
+      packsByLanguage = {
+        [languages[0]]: packsByLanguage,
+      };
+    } else {
+      // this is an error since multi-language analysis requires
+      // packs split by language
+      throw new Error(getPacksInvalidSplit(configFile));
+    }
+  }
+
+  for (const [lang, packsArr] of Object.entries(packsByLanguage)) {
+    if (!Array.isArray(packsArr)) {
+      throw new Error(getPacksInvalid(configFile));
+    }
+    if (!languages.includes(lang as Language)) {
+      throw new Error(getPacksRequireLanguage(lang, configFile));
+    }
+    packs[lang] = [];
+    for (const packStr of packsArr) {
+      packs[lang].push(toPackWithVersion(packStr, configFile));
+    }
+  }
+  return packs;
+}
+
+function toPackWithVersion(packStr, configFile: string): PackWithVersion {
+  if (typeof packStr !== "string") {
+    throw new Error(getPacksStrInvalid(packStr, configFile));
+  }
+
+  const nameWithVersion = packStr.split("@");
+  let version: string | undefined;
+  if (
+    nameWithVersion.length > 2 ||
+    !PACK_IDENTIFIER_PATTERN.test(nameWithVersion[0])
+  ) {
+    throw new Error(getPacksStrInvalid(packStr, configFile));
+  } else if (nameWithVersion.length === 2) {
+    version = semver.clean(nameWithVersion[1]) || undefined;
+    if (!version) {
+      throw new Error(getPacksStrInvalid(packStr, configFile));
+    }
+  }
+
+  return {
+    packName: nameWithVersion[0],
+    version,
+  };
+}
+
 function dbLocationOrDefault(
   dbLocation: string | undefined,
   tempDir: string
@@ -1019,11 +1158,10 @@ export async function initConfig(
   // The list of queries should not be empty for any language. If it is then
   // it is a user configuration error.
   for (const language of config.languages) {
-    if (
-      config.queries[language] === undefined ||
-      (config.queries[language].builtin.length === 0 &&
-        config.queries[language].custom.length === 0)
-    ) {
+    const hasBuiltinQueries = config.queries[language]?.builtin.length > 0;
+    const hasCustomQueries = config.queries[language]?.custom.length > 0;
+    const hasPacks = (config.packs[language]?.length || 0) > 0;
+    if (!hasPacks && !hasBuiltinQueries && !hasCustomQueries) {
       throw new Error(
         `Did not detect any queries to run for ${language}. ` +
           "Please make sure that the default queries are enabled, or you are specifying queries to run."

src/count-loc.test.ts

@@ -20,7 +20,7 @@ test("ensure lines of code works for cpp and js", async (t) => {
 
   t.deepEqual(results, {
     cpp: 6,
-    javascript: 3,
+    javascript: 9,
   });
 });
 
@@ -34,7 +34,7 @@ test("ensure lines of code can handle undefined language", async (t) => {
   );
 
   t.deepEqual(results, {
-    javascript: 3,
+    javascript: 9,
     python: 5,
   });
 });
@@ -93,6 +93,6 @@ test("ensure lines of code can handle exclude", async (t) => {
   );
 
   t.deepEqual(results, {
-    javascript: 3,
+    javascript: 9,
   });
 });

src/fingerprints.test.ts

@@ -7,32 +7,41 @@ import test from "ava";
 import * as fingerprints from "./fingerprints";
 import { getRunnerLogger } from "./logging";
 import { setupTests } from "./testing-utils";
+import * as util from "./util";
 
 setupTests(test);
 
-function testHash(t: ava.Assertions, input: string, expectedHashes: string[]) {
-  let index = 0;
-  const callback = function (lineNumber: number, hash: string) {
-    t.is(lineNumber, index + 1);
-    t.is(hash, expectedHashes[index]);
-    index++;
-  };
-  fingerprints.hash(callback, input);
-  t.is(index, input.split(/\r\n|\r|\n/).length);
+async function testHash(
+  t: ava.Assertions,
+  input: string,
+  expectedHashes: string[]
+) {
+  await util.withTmpDir(async (tmpDir) => {
+    const tmpFile = path.resolve(tmpDir, "testfile");
+    fs.writeFileSync(tmpFile, input);
+    let index = 0;
+    const callback = function (lineNumber: number, hash: string) {
+      t.is(lineNumber, index + 1);
+      t.is(hash, expectedHashes[index]);
+      index++;
+    };
+    await fingerprints.hash(callback, tmpFile);
+    t.is(index, input.split(/\r\n|\r|\n/).length);
+  });
 }
 
-test("hash", (t: ava.Assertions) => {
+test("hash", async (t: ava.Assertions) => {
   // Try empty file
-  testHash(t, "", ["c129715d7a2bc9a3:1"]);
+  await testHash(t, "", ["c129715d7a2bc9a3:1"]);
 
   // Try various combinations of newline characters
-  testHash(t, " a\nb\n  \t\tc\n d", [
+  await testHash(t, " a\nb\n  \t\tc\n d", [
     "271789c17abda88f:1",
     "54703d4cd895b18:1",
     "180aee12dab6264:1",
     "a23a3dc5e078b07b:1",
   ]);
-  testHash(t, " hello; \t\nworld!!!\n\n\n  \t\tGreetings\n End", [
+  await testHash(t, " hello; \t\nworld!!!\n\n\n  \t\tGreetings\n End", [
     "8b7cf3e952e7aeb2:1",
     "b1ae1287ec4718d9:1",
     "bff680108adb0fcc:1",
@@ -40,7 +49,7 @@ test("hash", (t: ava.Assertions) => {
     "b86d3392aea1be30:1",
     "e6ceba753e1a442:1",
   ]);
-  testHash(t, " hello; \t\nworld!!!\n\n\n  \t\tGreetings\n End\n", [
+  await testHash(t, " hello; \t\nworld!!!\n\n\n  \t\tGreetings\n End\n", [
     "e9496ae3ebfced30:1",
     "fb7c023a8b9ccb3f:1",
     "ce8ba1a563dcdaca:1",
@@ -49,7 +58,7 @@ test("hash", (t: ava.Assertions) => {
     "c8e28b0b4002a3a0:1",
     "c129715d7a2bc9a3:1",
   ]);
-  testHash(t, " hello; \t\nworld!!!\r\r\r  \t\tGreetings\r End\r", [
+  await testHash(t, " hello; \t\nworld!!!\r\r\r  \t\tGreetings\r End\r", [
     "e9496ae3ebfced30:1",
     "fb7c023a8b9ccb3f:1",
     "ce8ba1a563dcdaca:1",
@@ -58,16 +67,20 @@ test("hash", (t: ava.Assertions) => {
     "c8e28b0b4002a3a0:1",
     "c129715d7a2bc9a3:1",
   ]);
-  testHash(t, " hello; \t\r\nworld!!!\r\n\r\n\r\n  \t\tGreetings\r\n End\r\n", [
-    "e9496ae3ebfced30:1",
-    "fb7c023a8b9ccb3f:1",
-    "ce8ba1a563dcdaca:1",
-    "e20e36e16fcb0cc8:1",
-    "b3edc88f2938467e:1",
-    "c8e28b0b4002a3a0:1",
-    "c129715d7a2bc9a3:1",
-  ]);
-  testHash(t, " hello; \t\nworld!!!\r\n\n\r  \t\tGreetings\r End\r\n", [
+  await testHash(
+    t,
+    " hello; \t\r\nworld!!!\r\n\r\n\r\n  \t\tGreetings\r\n End\r\n",
+    [
+      "e9496ae3ebfced30:1",
+      "fb7c023a8b9ccb3f:1",
+      "ce8ba1a563dcdaca:1",
+      "e20e36e16fcb0cc8:1",
+      "b3edc88f2938467e:1",
+      "c8e28b0b4002a3a0:1",
+      "c129715d7a2bc9a3:1",
+    ]
+  );
+  await testHash(t, " hello; \t\nworld!!!\r\n\n\r  \t\tGreetings\r End\r\n", [
     "e9496ae3ebfced30:1",
     "fb7c023a8b9ccb3f:1",
     "ce8ba1a563dcdaca:1",
@@ -78,7 +91,7 @@ test("hash", (t: ava.Assertions) => {
   ]);
 
   // Try repeating line that will generate identical hashes
-  testHash(t, "Lorem ipsum dolor sit amet.\n".repeat(10), [
+  await testHash(t, "Lorem ipsum dolor sit amet.\n".repeat(10), [
     "a7f2ff13bc495cf2:1",
     "a7f2ff13bc495cf2:2",
     "a7f2ff13bc495cf2:3",
@@ -92,16 +105,20 @@ test("hash", (t: ava.Assertions) => {
     "c129715d7a2bc9a3:1",
   ]);
 
-  testHash(t, "x = 2\nx = 1\nprint(x)\nx = 3\nprint(x)\nx = 4\nprint(x)\n", [
-    "e54938cc54b302f1:1",
-    "bb609acbe9138d60:1",
-    "1131fd5871777f34:1",
-    "5c482a0f8b35ea28:1",
-    "54517377da7028d2:1",
-    "2c644846cb18d53e:1",
-    "f1b89f20de0d133:1",
-    "c129715d7a2bc9a3:1",
-  ]);
+  await testHash(
+    t,
+    "x = 2\nx = 1\nprint(x)\nx = 3\nprint(x)\nx = 4\nprint(x)\n",
+    [
+      "e54938cc54b302f1:1",
+      "bb609acbe9138d60:1",
+      "1131fd5871777f34:1",
+      "5c482a0f8b35ea28:1",
+      "54517377da7028d2:1",
+      "2c644846cb18d53e:1",
+      "f1b89f20de0d133:1",
+      "c129715d7a2bc9a3:1",
+    ]
+  );
 });
 
 function testResolveUriToFile(uri: any, index: any, artifactsURIs: any[]) {
@@ -170,7 +187,7 @@ test("resolveUriToFile", (t) => {
   t.is(testResolveUriToFile(`file://${dirpath}`, undefined, []), undefined);
 });
 
-test("addFingerprints", (t) => {
+test("addFingerprints", async (t) => {
   // Run an end-to-end test on a test file
   let input = fs
     .readFileSync(`${__dirname}/../src/testdata/fingerprinting.input.sarif`)
@@ -187,12 +204,16 @@ test("addFingerprints", (t) => {
   const checkoutPath = path.normalize(`${__dirname}/../src/testdata`);
 
   t.deepEqual(
-    fingerprints.addFingerprints(input, checkoutPath, getRunnerLogger(true)),
+    await fingerprints.addFingerprints(
+      input,
+      checkoutPath,
+      getRunnerLogger(true)
+    ),
     expected
   );
 });
 
-test("missingRegions", (t) => {
+test("missingRegions", async (t) => {
   // Run an end-to-end test on a test file
   let input = fs
     .readFileSync(`${__dirname}/../src/testdata/fingerprinting2.input.sarif`)
@@ -209,7 +230,11 @@ test("missingRegions", (t) => {
   const checkoutPath = path.normalize(`${__dirname}/../src/testdata`);
 
   t.deepEqual(
-    fingerprints.addFingerprints(input, checkoutPath, getRunnerLogger(true)),
+    await fingerprints.addFingerprints(
+      input,
+      checkoutPath,
+      getRunnerLogger(true)
+    ),
     expected
   );
 });

src/fingerprints.ts

@@ -8,6 +8,7 @@ const tab = "\t".charCodeAt(0);
 const space = " ".charCodeAt(0);
 const lf = "\n".charCodeAt(0);
 const cr = "\r".charCodeAt(0);
+const EOF = 65535;
 const BLOCK_SIZE = 100;
 const MOD = Long.fromInt(37); // L
 
@@ -34,9 +35,9 @@ type hashCallback = (lineNumber: number, hash: string) => void;
  * the hashes of the lines near the end of the file.
  *
  * @param callback function that is called with the line number (1-based) and hash for every line
- * @param input The file's contents
+ * @param filepath The path to the file to hash
  */
-export function hash(callback: hashCallback, input: string) {
+export async function hash(callback: hashCallback, filepath: string) {
   // A rolling view in to the input
   const window = Array(BLOCK_SIZE).fill(0);
 
@@ -87,12 +88,11 @@ export function hash(callback: hashCallback, input: string) {
   // as we go. Once we reach a point in the window again then we've processed
   // BLOCK_SIZE characters and if the last character at this point in the window
   // was the start of a line then we should output the hash for that line.
-  for (let i = 0, len = input.length; i <= len; i++) {
-    let current = i === len ? 65535 : input.charCodeAt(i);
+  const processCharacter = function (current: number) {
     // skip tabs, spaces, and line feeds that come directly after a carriage return
     if (current === space || current === tab || (prevCR && current === lf)) {
       prevCR = false;
-      continue;
+      return;
     }
     // replace CR with LF
     if (current === cr) {
@@ -113,7 +113,15 @@ export function hash(callback: hashCallback, input: string) {
       lineStart = true;
     }
     updateHash(current);
+  };
+
+  const readStream = fs.createReadStream(filepath, "utf8");
+  for await (const data of readStream) {
+    for (let i = 0; i < data.length; ++i) {
+      processCharacter(data.charCodeAt(i));
+    }
   }
+  processCharacter(EOF);
 
   // Flush the remaining lines
   for (let i = 0; i < BLOCK_SIZE; i++) {
@@ -237,11 +245,11 @@ export function resolveUriToFile(
 
 // Compute fingerprints for results in the given sarif file
 // and return an updated sarif file contents.
-export function addFingerprints(
+export async function addFingerprints(
   sarifContents: string,
   checkoutPath: string,
   logger: Logger
-): string {
+): Promise<string> {
   const sarif = JSON.parse(sarifContents);
 
   // Gather together results for the same file and construct
@@ -263,6 +271,11 @@ export function addFingerprints(
         continue;
       }
 
+      if (primaryLocation?.physicalLocation?.region?.startLine === undefined) {
+        // Locations without a line number are unlikely to be source files
+        continue;
+      }
+
       const filepath = resolveUriToFile(
         primaryLocation.physicalLocation.artifactLocation,
         artifacts,
@@ -289,8 +302,7 @@ export function addFingerprints(
         c(lineNumber, hashValue);
       }
     };
-    const fileContents = fs.readFileSync(filepath).toString();
-    hash(teeCallback, fileContents);
+    await hash(teeCallback, filepath);
   }
 
   return JSON.stringify(sarif);

src/testdata/fingerprinting2.expected.sarif

@@ -30,9 +30,7 @@
                     "message": {
                         "text": "This header file should contain a header guard to prevent multiple inclusion."
                     },
-                    "partialFingerprints": {
-                        "primaryLocationLineHash": "599c824c91d0f75e:1"
-                    },
+                    "partialFingerprints": {},
                     "ruleId": "cpp/missing-header-guard",
                     "ruleIndex": 0
                 }

src/tracer-config.test.ts

@@ -28,6 +28,7 @@ function getTestConfig(tmpDir: string): configUtils.Config {
     codeQLCmd: "",
     gitHubVersion: { type: util.GitHubVariant.DOTCOM } as util.GitHubVersion,
     dbLocation: path.resolve(tmpDir, "codeql_databases"),
+    packs: {},
   };
 }
 

src/upload-lib.ts

@@ -356,7 +356,7 @@ async function uploadFiles(
   }
 
   let sarifPayload = combineSarifFiles(sarifFiles);
-  sarifPayload = fingerprints.addFingerprints(
+  sarifPayload = await fingerprints.addFingerprints(
     sarifPayload,
     checkoutPath,
     logger

src/util.ts

@@ -478,3 +478,12 @@ export function getRequiredEnvParam(paramName: string): string {
   }
   return value;
 }
+
+export interface HTTPError {
+  status: number;
+  message?: string;
+}
+
+export function isHTTPError(arg: any): arg is HTTPError {
+  return arg?.status !== undefined && Number.isInteger(arg.status);
+}

tests/multi-language-repo/.github/codeql/codeql-config-packaging.yml

@@ -0,0 +1,10 @@
+name: Pack testing in the CodeQL Action
+
+disable-default-queries: true
+packs:
+  javascript:
+    - dsp-testing/codeql-pack1@0.0.4
+    - dsp-testing/codeql-pack2 # latest
+paths-ignore:
+  - tests
+  - lib

tests/multi-language-repo/main.js

@@ -1,3 +1,12 @@
+if (true) {
+    console.log("Hello, World!");
+    console.log("Good-bye, World!");
+}
+
 if (true) {
     console.log("Hello, World!");
 }
+
+if (true) {
+    // empty
+}