Merge branch 'aeisenberg/unrevert-query-filters' into aeisenberg/fix-config-files

src/analyze.ts

@@ -224,6 +224,9 @@ export async function runQueries(
   const codeql = await getCodeQL(config.codeQLCmd);
   for (const language of config.languages) {
     const queries = config.queries[language];
+    const queryFilters = validateQueryFilters(
+      config.originalUserInput["query-filters"]
+    );
     const packsWithVersion = config.packs[language] || [];
 
     const hasBuiltinQueries = queries?.builtin.length > 0;
@@ -288,7 +291,7 @@ export async function runQueries(
             (await runQueryGroup(
               language,
               "builtin",
-              createQuerySuiteContents(queries["builtin"]),
+              createQuerySuiteContents(queries["builtin"], queryFilters),
               undefined
             )) as string
           );
@@ -303,7 +306,10 @@ export async function runQueries(
               (await runQueryGroup(
                 language,
                 `custom-${i}`,
-                createQuerySuiteContents(queries["custom"][i].queries),
+                createQuerySuiteContents(
+                  queries["custom"][i].queries,
+                  queryFilters
+                ),
                 queries["custom"][i].searchPath
               )) as string
             );
@@ -312,12 +318,12 @@ export async function runQueries(
         }
         if (packsWithVersion.length > 0) {
           querySuitePaths.push(
-            ...(await runQueryPacks(
+            await runQueryPacks(
               language,
               "packs",
               packsWithVersion,
-              undefined
-            ))
+              queryFilters
+            )
           );
           ranCustom = true;
         }
@@ -427,31 +433,60 @@ export async function runQueries(
     language: Language,
     type: string,
     packs: string[],
-    searchPath: string | undefined
-  ): Promise<string[]> {
+    queryFilters: configUtils.QueryFilter[]
+  ): Promise<string> {
     const databasePath = util.getCodeQLDatabasePath(config, language);
-    // Run the queries individually instead of all at once to avoid command
-    // line length restrictions, particularly on windows.
 
     for (const pack of packs) {
       logger.debug(`Running query pack for ${language}-${type}: ${pack}`);
-
-      await codeql.databaseRunQueries(
-        databasePath,
-        searchPath,
-        pack,
-        memoryFlag,
-        threadsFlag
-      );
-
-      logger.debug(`BQRS results produced for ${language} (queries: ${type})"`);
     }
-    return packs;
+
+    // combine the list of packs into a query suite in order to run them all simultaneously.
+    const querySuite = (
+      packs.map(convertPackToQuerySuiteEntry) as configUtils.QuerySuiteEntry[]
+    ).concat(queryFilters);
+
+    const querySuitePath = `${databasePath}-queries-${type}.qls`;
+    fs.writeFileSync(querySuitePath, yaml.dump(querySuite));
+
+    logger.debug(`BQRS results produced for ${language} (queries: ${type})"`);
+
+    await codeql.databaseRunQueries(
+      databasePath,
+      undefined,
+      querySuitePath,
+      memoryFlag,
+      threadsFlag
+    );
+
+    return querySuitePath;
   }
 }
 
-function createQuerySuiteContents(queries: string[]) {
-  return queries.map((q: string) => `- query: ${q}`).join("\n");
+export function convertPackToQuerySuiteEntry(
+  packStr: string
+): configUtils.QuerySuitePackEntry {
+  const pack = configUtils.parsePacksSpecification(packStr);
+  return {
+    qlpack: !pack.path ? pack.name : undefined,
+    from: pack.path ? pack.name : undefined,
+    version: pack.version,
+    query: pack.path?.endsWith(".ql") ? pack.path : undefined,
+    queries:
+      !pack.path?.endsWith(".ql") && !pack.path?.endsWith(".qls")
+        ? pack.path
+        : undefined,
+    apply: pack.path?.endsWith(".qls") ? pack.path : undefined,
+  };
+}
+
+export function createQuerySuiteContents(
+  queries: string[],
+  queryFilters: configUtils.QueryFilter[]
+) {
+  return yaml.dump(
+    queries.map((q: string) => ({ query: q })).concat(queryFilters as any)
+  );
 }
 
 export async function runFinalize(
@@ -551,3 +586,33 @@ function printLinesOfCodeSummary(
     );
   }
 }
+
+// exported for testing
+export function validateQueryFilters(queryFilters?: configUtils.QueryFilter[]) {
+  if (!queryFilters) {
+    return [];
+  }
+
+  const errors: string[] = [];
+  for (const qf of queryFilters) {
+    const keys = Object.keys(qf);
+    if (keys.length !== 1) {
+      errors.push(
+        `Query filter must have exactly one key: ${JSON.stringify(qf)}`
+      );
+    }
+    if (!["exclude", "include"].includes(keys[0])) {
+      errors.push(
+        `Only "include" or "exclude" filters are allowed:\n${JSON.stringify(
+          qf
+        )}`
+      );
+    }
+  }
+
+  if (errors.length) {
+    throw new Error(`Invalid query filter.\n${errors.join("\n")}`);
+  }
+
+  return queryFilters;
+}