Merge branch 'main' into henrymercer/semver-bundles

lib/codeql.js

@@ -66,7 +66,6 @@ const CODEQL_MINIMUM_VERSION = "2.8.5";
  * For convenience, please keep these in descending order. Once a version
  * flag is older than the oldest supported version above, it may be removed.
  */
-const CODEQL_VERSION_CUSTOM_QUERY_HELP = "2.7.1";
 const CODEQL_VERSION_LUA_TRACER_CONFIG = "2.10.0";
 const CODEQL_VERSION_LUA_TRACING_GO_WINDOWS_FIXED = "2.10.4";
 exports.CODEQL_VERSION_GHES_PACK_DOWNLOAD = "2.10.4";
@@ -174,6 +173,7 @@ function setCodeQL(partialCodeql) {
         databasePrintBaseline: resolveFunction(partialCodeql, "databasePrintBaseline"),
         databaseExportDiagnostics: resolveFunction(partialCodeql, "databaseExportDiagnostics"),
         diagnosticsExport: resolveFunction(partialCodeql, "diagnosticsExport"),
+        resolveExtractor: resolveFunction(partialCodeql, "resolveExtractor"),
     };
     return cachedCodeQL;
 }
@@ -269,10 +269,7 @@ async function getCodeQLForCmd(cmd, checkVersion) {
             ], { stdin: externalRepositoryToken });
         },
         async runAutobuild(language) {
-            const cmdName = process.platform === "win32" ? "autobuild.cmd" : "autobuild.sh";
+            const autobuildCmd = path.join(await this.resolveExtractor(language), "tools", process.platform === "win32" ? "autobuild.cmd" : "autobuild.sh");
-            // The autobuilder for Swift is located in the experimental/ directory.
-            const possibleExperimentalDir = language === languages_1.Language.swift ? "experimental" : "";
-            const autobuildCmd = path.join(path.dirname(cmd), possibleExperimentalDir, language, "tools", cmdName);
             // Update JAVA_TOOL_OPTIONS to contain '-Dhttp.keepAlive=false'
             // This is because of an issue with Azure pipelines timing out connections after 4 minutes
             // and Maven not properly handling closed connections
@@ -301,31 +298,9 @@ async function getCodeQLForCmd(cmd, checkVersion) {
         },
         async extractScannedLanguage(config, language) {
             const databasePath = util.getCodeQLDatabasePath(config, language);
-            // Get extractor location
-            //
-            // Request it using `format=json` so we don't need to strip the trailing new line generated by
-            // the CLI.
-            let extractorPath = "";
-            await new toolrunner.ToolRunner(cmd, [
-                "resolve",
-                "extractor",
-                "--format=json",
-                `--language=${language}`,
-                ...getExtraOptionsFromEnv(["resolve", "extractor"]),
-            ], {
-                silent: true,
-                listeners: {
-                    stdout: (data) => {
-                        extractorPath += data.toString();
-                    },
-                    stderr: (data) => {
-                        process.stderr.write(data);
-                    },
-                },
-            }).exec();
             // Set trace command
             const ext = process.platform === "win32" ? ".cmd" : ".sh";
-            const traceCommand = path.resolve(JSON.parse(extractorPath), "tools", `autobuild${ext}`);
+            const traceCommand = path.resolve(await this.resolveExtractor(language), "tools", `autobuild${ext}`);
             // Run trace command
             await (0, toolrunner_error_catcher_1.toolrunnerErrorCatcher)(cmd, [
                 "database",
@@ -438,12 +413,11 @@ async function getCodeQLForCmd(cmd, checkVersion) {
                 addSnippetsFlag,
                 "--print-diagnostics-summary",
                 "--print-metrics-summary",
+                "--sarif-add-query-help",
                 "--sarif-group-rules-by-pack",
                 ...(await getCodeScanningConfigExportArguments(config, this, features)),
                 ...getExtraOptionsFromEnv(["database", "interpret-results"]),
             ];
-            if (await util.codeQlVersionAbove(this, CODEQL_VERSION_CUSTOM_QUERY_HELP))
-                codeqlArgs.push("--sarif-add-query-help");
             if (automationDetailsId !== undefined) {
                 codeqlArgs.push("--sarif-category", automationDetailsId);
             }
@@ -581,6 +555,29 @@ async function getCodeQLForCmd(cmd, checkVersion) {
             }
             await new toolrunner.ToolRunner(cmd, args).exec();
         },
+        async resolveExtractor(language) {
+            // Request it using `format=json` so we don't need to strip the trailing new line generated by
+            // the CLI.
+            let extractorPath = "";
+            await new toolrunner.ToolRunner(cmd, [
+                "resolve",
+                "extractor",
+                "--format=json",
+                `--language=${language}`,
+                ...getExtraOptionsFromEnv(["resolve", "extractor"]),
+            ], {
+                silent: true,
+                listeners: {
+                    stdout: (data) => {
+                        extractorPath += data.toString();
+                    },
+                    stderr: (data) => {
+                        process.stderr.write(data);
+                    },
+                },
+            }).exec();
+            return JSON.parse(extractorPath);
+        },
     };
     // To ensure that status reports include the CodeQL CLI version wherever
     // possible, we want to call getVersion(), which populates the version value

lib/codeql.test.js

@@ -388,24 +388,6 @@ for (const isBundleVersionInUrl of [true, false]) {
     t.throws(() => codeql.getExtraOptions({ foo: 87 }, ["foo"], []));
     t.throws(() => codeql.getExtraOptions({ "*": [42], foo: { "*": 87, bar: [99] } }, ["foo", "bar"], []));
 });
-(0, ava_1.default)("databaseInterpretResults() does not set --sarif-add-query-help for 2.7.0", async (t) => {
-    const runnerConstructorStub = stubToolRunnerConstructor();
-    const codeqlObject = await codeql.getCodeQLForTesting();
-    sinon.stub(codeqlObject, "getVersion").resolves("2.7.0");
-    // safeWhich throws because of the test CodeQL object.
-    sinon.stub(safeWhich, "safeWhich").resolves("");
-    await codeqlObject.databaseInterpretResults("", [], "", "", "", "-v", "", stubConfig, (0, testing_utils_1.createFeatures)([]), (0, logging_1.getRunnerLogger)(true));
-    t.false(runnerConstructorStub.firstCall.args[1].includes("--sarif-add-query-help"), "--sarif-add-query-help should be absent, but it is present");
-});
-(0, ava_1.default)("databaseInterpretResults() sets --sarif-add-query-help for 2.7.1", async (t) => {
-    const runnerConstructorStub = stubToolRunnerConstructor();
-    const codeqlObject = await codeql.getCodeQLForTesting();
-    sinon.stub(codeqlObject, "getVersion").resolves("2.7.1");
-    // safeWhich throws because of the test CodeQL object.
-    sinon.stub(safeWhich, "safeWhich").resolves("");
-    await codeqlObject.databaseInterpretResults("", [], "", "", "", "-v", "", stubConfig, (0, testing_utils_1.createFeatures)([]), (0, logging_1.getRunnerLogger)(true));
-    t.true(runnerConstructorStub.firstCall.args[1].includes("--sarif-add-query-help"), "--sarif-add-query-help should be present, but it is absent");
-});
 (0, ava_1.default)("databaseInitCluster() without injected codescanning config", async (t) => {
     await util.withTmpDir(async (tempDir) => {
         const runnerConstructorStub = stubToolRunnerConstructor();

src/codeql.test.ts

@@ -568,54 +568,6 @@ test("getExtraOptions throws for bad content", (t) => {
   );
 });
 
-test("databaseInterpretResults() does not set --sarif-add-query-help for 2.7.0", async (t) => {
-  const runnerConstructorStub = stubToolRunnerConstructor();
-  const codeqlObject = await codeql.getCodeQLForTesting();
-  sinon.stub(codeqlObject, "getVersion").resolves("2.7.0");
-  // safeWhich throws because of the test CodeQL object.
-  sinon.stub(safeWhich, "safeWhich").resolves("");
-  await codeqlObject.databaseInterpretResults(
-    "",
-    [],
-    "",
-    "",
-    "",
-    "-v",
-    "",
-    stubConfig,
-    createFeatures([]),
-    getRunnerLogger(true)
-  );
-  t.false(
-    runnerConstructorStub.firstCall.args[1].includes("--sarif-add-query-help"),
-    "--sarif-add-query-help should be absent, but it is present"
-  );
-});
-
-test("databaseInterpretResults() sets --sarif-add-query-help for 2.7.1", async (t) => {
-  const runnerConstructorStub = stubToolRunnerConstructor();
-  const codeqlObject = await codeql.getCodeQLForTesting();
-  sinon.stub(codeqlObject, "getVersion").resolves("2.7.1");
-  // safeWhich throws because of the test CodeQL object.
-  sinon.stub(safeWhich, "safeWhich").resolves("");
-  await codeqlObject.databaseInterpretResults(
-    "",
-    [],
-    "",
-    "",
-    "",
-    "-v",
-    "",
-    stubConfig,
-    createFeatures([]),
-    getRunnerLogger(true)
-  );
-  t.true(
-    runnerConstructorStub.firstCall.args[1].includes("--sarif-add-query-help"),
-    "--sarif-add-query-help should be present, but it is absent"
-  );
-});
-
 test("databaseInitCluster() without injected codescanning config", async (t) => {
   await util.withTmpDir(async (tempDir) => {
     const runnerConstructorStub = stubToolRunnerConstructor();

src/codeql.ts

@@ -196,6 +196,8 @@ export interface CodeQL {
     config: Config,
     features: FeatureEnablement
   ): Promise<void>;
+  /** Get the location of an extractor for the specified language. */
+  resolveExtractor(language: Language): Promise<string>;
 }
 
 export interface ResolveLanguagesOutput {
@@ -259,7 +261,6 @@ const CODEQL_MINIMUM_VERSION = "2.8.5";
  * For convenience, please keep these in descending order. Once a version
  * flag is older than the oldest supported version above, it may be removed.
  */
-const CODEQL_VERSION_CUSTOM_QUERY_HELP = "2.7.1";
 const CODEQL_VERSION_LUA_TRACER_CONFIG = "2.10.0";
 const CODEQL_VERSION_LUA_TRACING_GO_WINDOWS_FIXED = "2.10.4";
 export const CODEQL_VERSION_GHES_PACK_DOWNLOAD = "2.10.4";
@@ -418,6 +419,7 @@ export function setCodeQL(partialCodeql: Partial<CodeQL>): CodeQL {
       "databaseExportDiagnostics"
     ),
     diagnosticsExport: resolveFunction(partialCodeql, "diagnosticsExport"),
+    resolveExtractor: resolveFunction(partialCodeql, "resolveExtractor"),
   };
   return cachedCodeQL;
 }
@@ -547,17 +549,10 @@ export async function getCodeQLForCmd(
       );
     },
     async runAutobuild(language: Language) {
-      const cmdName =
-        process.platform === "win32" ? "autobuild.cmd" : "autobuild.sh";
-      // The autobuilder for Swift is located in the experimental/ directory.
-      const possibleExperimentalDir =
-        language === Language.swift ? "experimental" : "";
       const autobuildCmd = path.join(
-        path.dirname(cmd),
+        await this.resolveExtractor(language),
-        possibleExperimentalDir,
-        language,
         "tools",
-        cmdName
+        process.platform === "win32" ? "autobuild.cmd" : "autobuild.sh"
       );
 
       // Update JAVA_TOOL_OPTIONS to contain '-Dhttp.keepAlive=false'
@@ -590,37 +585,11 @@ export async function getCodeQLForCmd(
     },
     async extractScannedLanguage(config: Config, language: Language) {
       const databasePath = util.getCodeQLDatabasePath(config, language);
-      // Get extractor location
-      //
-      // Request it using `format=json` so we don't need to strip the trailing new line generated by
-      // the CLI.
-      let extractorPath = "";
-      await new toolrunner.ToolRunner(
-        cmd,
-        [
-          "resolve",
-          "extractor",
-          "--format=json",
-          `--language=${language}`,
-          ...getExtraOptionsFromEnv(["resolve", "extractor"]),
-        ],
-        {
-          silent: true,
-          listeners: {
-            stdout: (data) => {
-              extractorPath += data.toString();
-            },
-            stderr: (data) => {
-              process.stderr.write(data);
-            },
-          },
-        }
-      ).exec();
 
       // Set trace command
       const ext = process.platform === "win32" ? ".cmd" : ".sh";
       const traceCommand = path.resolve(
-        JSON.parse(extractorPath) as string,
+        await this.resolveExtractor(language),
         "tools",
         `autobuild${ext}`
       );
@@ -773,12 +742,11 @@ export async function getCodeQLForCmd(
         addSnippetsFlag,
         "--print-diagnostics-summary",
         "--print-metrics-summary",
+        "--sarif-add-query-help",
         "--sarif-group-rules-by-pack",
         ...(await getCodeScanningConfigExportArguments(config, this, features)),
         ...getExtraOptionsFromEnv(["database", "interpret-results"]),
       ];
-      if (await util.codeQlVersionAbove(this, CODEQL_VERSION_CUSTOM_QUERY_HELP))
-        codeqlArgs.push("--sarif-add-query-help");
       if (automationDetailsId !== undefined) {
         codeqlArgs.push("--sarif-category", automationDetailsId);
       }
@@ -954,6 +922,33 @@ export async function getCodeQLForCmd(
       }
       await new toolrunner.ToolRunner(cmd, args).exec();
     },
+    async resolveExtractor(language: Language): Promise<string> {
+      // Request it using `format=json` so we don't need to strip the trailing new line generated by
+      // the CLI.
+      let extractorPath = "";
+      await new toolrunner.ToolRunner(
+        cmd,
+        [
+          "resolve",
+          "extractor",
+          "--format=json",
+          `--language=${language}`,
+          ...getExtraOptionsFromEnv(["resolve", "extractor"]),
+        ],
+        {
+          silent: true,
+          listeners: {
+            stdout: (data) => {
+              extractorPath += data.toString();
+            },
+            stderr: (data) => {
+              process.stderr.write(data);
+            },
+          },
+        }
+      ).exec();
+      return JSON.parse(extractorPath);
+    },
   };
   // To ensure that status reports include the CodeQL CLI version wherever
   // possible, we want to call getVersion(), which populates the version value