Add the feature to bypass the toolcache for kotlin and swift

This works by moving the logic to check for toolcache bypass out of
creating the codeql instance. The logic now _may_ perform an API request
in order to check what languages are in the repository. This check is
redundant because the same call is being made later in the action when
the actual list of languages is calculated.

src/codeql.test.ts

@@ -14,7 +14,7 @@ import { GitHubApiDetails } from "./api-client";
 import * as codeql from "./codeql";
 import { AugmentationProperties, Config } from "./config-utils";
 import * as defaults from "./defaults.json";
-import { Feature, FeatureEnablement } from "./feature-flags";
+import { Feature } from "./feature-flags";
 import { Language } from "./languages";
 import { getRunnerLogger } from "./logging";
 import { setupTests, setupActionsVars, createFeatures } from "./testing-utils";
@@ -70,14 +70,14 @@ test.beforeEach(() => {
 
 async function mockApiAndSetupCodeQL({
   apiDetails,
-  featureEnablement,
+  bypassToolcache,
   isPinned,
   tmpDir,
   toolsInput,
   version,
 }: {
   apiDetails?: GitHubApiDetails;
-  featureEnablement?: FeatureEnablement;
+  bypassToolcache?: boolean;
   isPinned?: boolean;
   tmpDir: string;
   toolsInput?: { input?: string };
@@ -110,7 +110,7 @@ async function mockApiAndSetupCodeQL({
     apiDetails ?? sampleApiDetails,
     tmpDir,
     util.GitHubVariant.DOTCOM,
-    featureEnablement ?? createFeatures([]),
+    !!bypassToolcache,
     getRunnerLogger(true),
     false
   );
@@ -173,7 +173,7 @@ test("don't download codeql bundle cache with pinned different version cached",
       sampleApiDetails,
       tmpDir,
       util.GitHubVariant.DOTCOM,
-      createFeatures([]),
+      false,
       getRunnerLogger(true),
       false
     );
@@ -281,9 +281,7 @@ for (const [
       await mockApiAndSetupCodeQL({
         version: defaults.bundleVersion,
         apiDetails: sampleApiDetails,
-        featureEnablement: createFeatures(
-          isFeatureEnabled ? [Feature.BypassToolcacheEnabled] : []
-        ),
+        bypassToolcache: isFeatureEnabled,
         toolsInput: { input: toolsInput },
         tmpDir,
       });
@@ -339,7 +337,7 @@ test("download codeql bundle from github ae endpoint", async (t) => {
       sampleGHAEApiDetails,
       tmpDir,
       util.GitHubVariant.GHAE,
-      createFeatures([]),
+      false,
       getRunnerLogger(true),
       false
     );

src/codeql.ts

@@ -415,7 +415,7 @@ export async function setupCodeQL(
   apiDetails: api.GitHubApiDetails,
   tempDir: string,
   variant: util.GitHubVariant,
-  features: FeatureEnablement,
+  bypassToolcache: boolean,
   logger: Logger,
   checkVersion: boolean
 ): Promise<{ codeql: CodeQL; toolsVersion: string }> {
@@ -429,8 +429,7 @@ export async function setupCodeQL(
         // the toolcache when the appropriate feature is enabled. This
         // allows us to quickly rollback a broken bundle that has made its way
         // into the toolcache.
-        codeqlURL === undefined &&
-          (await features.getValue(Feature.BypassToolcacheEnabled))
+        codeqlURL === undefined && bypassToolcache
         ? "a specific version of CodeQL was not requested and the bypass toolcache feature is enabled"
         : undefined;
     const forceLatest = forceLatestReason !== undefined;

src/config-utils.ts

@@ -15,7 +15,12 @@ import {
 } from "./codeql";
 import * as externalQueries from "./external-queries";
 import { Feature, FeatureEnablement } from "./feature-flags";
-import { Language, parseLanguage } from "./languages";
+import {
+  Language,
+  LanguageOrAlias,
+  LANGUAGE_ALIASES,
+  parseLanguage,
+} from "./languages";
 import { Logger } from "./logging";
 import { RepositoryNwo } from "./repository";
 import { downloadTrapCaches } from "./trap-caching";
@@ -857,7 +862,7 @@ export function getUnknownLanguagesError(languages: string[]): string {
 async function getLanguagesInRepo(
   repository: RepositoryNwo,
   logger: Logger
-): Promise<Language[]> {
+): Promise<LanguageOrAlias[]> {
   logger.debug(`GitHub repo ${repository.owner} ${repository.repo}`);
   const response = await api.getApiClient().repos.listLanguages({
     owner: repository.owner,
@@ -870,7 +875,7 @@ async function getLanguagesInRepo(
   // When we pick a language to autobuild we want to pick the most popular traced language
   // Since sets in javascript maintain insertion order, using a set here and then splatting it
   // into an array gives us an array of languages ordered by popularity
-  const languages: Set<Language> = new Set();
+  const languages: Set<LanguageOrAlias> = new Set();
   for (const lang of Object.keys(response.data)) {
     const parsedLang = parseLanguage(lang);
     if (parsedLang !== undefined) {
@@ -896,21 +901,21 @@ async function getLanguages(
   repository: RepositoryNwo,
   logger: Logger
 ): Promise<Language[]> {
-  // Obtain from action input 'languages' if set
-  let languages = (languagesInput || "")
-    .split(",")
-    .map((x) => x.trim())
-    .filter((x) => x.length > 0);
-  logger.info(`Languages from configuration: ${JSON.stringify(languages)}`);
+  // Obtain languages without filtering them.
+  const { rawLanguages, autodetected } = await getRawLanguages(
+    languagesInput,
+    repository,
+    logger
+  );
 
-  if (languages.length === 0) {
-    // Obtain languages as all languages in the repo that can be analysed
-    languages = await getLanguagesInRepo(repository, logger);
+  let languages: string[];
+  if (autodetected) {
     const availableLanguages = await codeQL.resolveLanguages();
-    languages = languages.filter((value) => value in availableLanguages);
-    logger.info(
-      `Automatically detected languages: ${JSON.stringify(languages)}`
-    );
+    languages = rawLanguages.filter((value) => value in availableLanguages);
+    logger.info(`Automatically detected languages: ${languages.join(", ")}`);
+  } else {
+    languages = rawLanguages;
+    logger.info(`Languages from configuration: ${languages.join(", ")}`);
   }
 
   // If the languages parameter was not given and no languages were
@@ -924,10 +929,14 @@ async function getLanguages(
   const unknownLanguages: string[] = [];
   for (const language of languages) {
     const parsedLanguage = parseLanguage(language);
+    const dealiasedLanguage =
+      parsedLanguage && parsedLanguage in LANGUAGE_ALIASES
+        ? LANGUAGE_ALIASES[parsedLanguage]
+        : (parsedLanguage as Language);
     if (parsedLanguage === undefined) {
       unknownLanguages.push(language);
-    } else if (parsedLanguages.indexOf(parsedLanguage) === -1) {
-      parsedLanguages.push(parsedLanguage);
+    } else if (!parsedLanguages.includes(dealiasedLanguage)) {
+      parsedLanguages.push(dealiasedLanguage);
     }
   }
   if (unknownLanguages.length > 0) {
@@ -937,6 +946,38 @@ async function getLanguages(
   return parsedLanguages;
 }
 
+/**
+ * Gets the set of languages in the current repository without checking to
+ * see if these languages are actually supported by CodeQL.
+ *
+ * @param languagesInput The languages from the workflow input
+ * @param repository the owner/name of the repository
+ * @param logger a logger
+ * @returns A tuple containing a list of languages in this repository that might be
+ * analyzable and whether or not this list was determined automatically.
+ */
+export async function getRawLanguages(
+  languagesInput: string | undefined,
+  repository: RepositoryNwo,
+  logger: Logger
+) {
+  // Obtain from action input 'languages' if set
+  let rawLanguages = (languagesInput || "")
+    .split(",")
+    .map((x) => x.trim().toLowerCase())
+    .filter((x) => x.length > 0);
+  let autodetected: boolean;
+  if (rawLanguages.length) {
+    autodetected = false;
+  } else {
+    autodetected = true;
+
+    // Obtain languages as all languages in the repo that can be analysed
+    rawLanguages = (await getLanguagesInRepo(repository, logger)) as string[];
+  }
+  return { rawLanguages, autodetected };
+}
+
 async function addQueriesAndPacksFromWorkflow(
   codeQL: CodeQL,
   queriesInput: string,

src/init-action.ts

@@ -41,6 +41,7 @@ import {
   getThreadsFlagValue,
   initializeEnvironment,
   isHostedRunner,
+  shouldBypassToolcache,
 } from "./util";
 
 // eslint-disable-next-line import/no-commonjs
@@ -186,7 +187,13 @@ async function run() {
       apiDetails,
       getTemporaryDirectory(),
       gitHubVersion.type,
-      features,
+      await shouldBypassToolcache(
+        features,
+        getOptionalInput("tools"),
+        getOptionalInput("languages"),
+        repositoryNwo,
+        logger
+      ),
       logger
     );
     codeql = initCodeQLResult.codeql;

src/init.ts

@@ -20,7 +20,7 @@ export async function initCodeQL(
   apiDetails: GitHubApiDetails,
   tempDir: string,
   variant: util.GitHubVariant,
-  featureEnablement: FeatureEnablement,
+  bypassToolcache: boolean,
   logger: Logger
 ): Promise<{ codeql: CodeQL; toolsVersion: string }> {
   logger.startGroup("Setup CodeQL tools");
@@ -29,7 +29,7 @@ export async function initCodeQL(
     apiDetails,
     tempDir,
     variant,
-    featureEnablement,
+    bypassToolcache,
     logger,
     true
   );

src/languages.ts

@@ -11,7 +11,7 @@ export enum Language {
 }
 
 // Additional names for languages
-const LANGUAGE_ALIASES: { [lang: string]: Language } = {
+export const LANGUAGE_ALIASES: { [lang: string]: Language } = {
   c: Language.cpp,
   "c++": Language.cpp,
   "c#": Language.csharp,
@@ -19,8 +19,12 @@ const LANGUAGE_ALIASES: { [lang: string]: Language } = {
   typescript: Language.javascript,
 };
 
+export type LanguageOrAlias = Language | keyof typeof LANGUAGE_ALIASES;
+
+export const KOTLIN_SWIFT_BYPASS = ["kotlin", "swift"];
+
 // Translate from user input or GitHub's API names for languages to CodeQL's names for languages
-export function parseLanguage(language: string): Language | undefined {
+export function parseLanguage(language: string): LanguageOrAlias | undefined {
   // Normalise to lower case
   language = language.toLowerCase();
 
@@ -31,7 +35,7 @@ export function parseLanguage(language: string): Language | undefined {
 
   // Check language aliases
   if (language in LANGUAGE_ALIASES) {
-    return LANGUAGE_ALIASES[language];
+    return language;
   }
 
   return undefined;

src/testing-utils.ts

@@ -162,6 +162,26 @@ export function mockFeatureFlagApiEndpoint(
   sinon.stub(apiClient, "getApiClient").value(() => client);
 }
 
+export function mockLangaugesInRepo(languages: string[]) {
+  const mockClient = sinon.stub(apiClient, "getApiClient");
+  const listLanguages = sinon.stub().resolves({
+    status: 200,
+    data: languages.reduce((acc, lang) => {
+      acc[lang] = 1;
+      return acc;
+    }, {}),
+    headers: {},
+    url: "GET /repos/:owner/:repo/languages",
+  });
+
+  mockClient.returns({
+    repos: {
+      listLanguages,
+    },
+  } as any);
+  return listLanguages;
+}
+
 export function mockCodeQLVersion(version) {
   return {
     async getVersion() {

src/util.test.ts

@@ -9,8 +9,15 @@ import * as sinon from "sinon";
 
 import * as api from "./api-client";
 import { Config } from "./config-utils";
+import { Feature } from "./feature-flags";
 import { getRunnerLogger } from "./logging";
-import { setupTests } from "./testing-utils";
+import { parseRepositoryNwo } from "./repository";
+import {
+  createFeatures,
+  getRecordingLogger,
+  mockLangaugesInRepo,
+  setupTests,
+} from "./testing-utils";
 import * as util from "./util";
 
 setupTests(test);
@@ -449,3 +456,105 @@ test("withTimeout doesn't call callback if promise resolves", async (t) => {
   t.deepEqual(shortTaskTimedOut, false);
   t.deepEqual(result, 99);
 });
+
+const mockRepositoryNwo = parseRepositoryNwo("owner/repo");
+// eslint-disable-next-line github/array-foreach
+[
+  {
+    name: "disabled",
+    features: [],
+    hasCustomCodeQL: false,
+    languagesInput: undefined,
+    languagesInRepository: [],
+    expected: false,
+    expectedApiCall: false,
+  },
+  {
+    name: "disabled even though swift kotlin bypassed",
+    features: [Feature.BypassToolcacheKotlinSwiftEnabled],
+    hasCustomCodeQL: false,
+    languagesInput: undefined,
+    languagesInRepository: [],
+    expected: false,
+    expectedApiCall: true,
+  },
+  {
+    name: "disabled even though swift kotlin analyzed",
+    features: [],
+    hasCustomCodeQL: false,
+    languagesInput: " sWiFt , KoTlIn ",
+    languagesInRepository: [],
+    expected: false,
+    expectedApiCall: false,
+  },
+  {
+    name: "toolcache bypass all",
+    features: [Feature.BypassToolcacheEnabled],
+    hasCustomCodeQL: false,
+    languagesInput: undefined,
+    languagesInRepository: [],
+    expected: true,
+    expectedApiCall: false,
+  },
+  {
+    name: "custom CodeQL",
+    features: [],
+    hasCustomCodeQL: true,
+    languagesInput: undefined,
+    languagesInRepository: [],
+    expected: true,
+    expectedApiCall: false,
+  },
+  {
+    name: "bypass swift",
+    features: [Feature.BypassToolcacheKotlinSwiftEnabled],
+    hasCustomCodeQL: false,
+    languagesInput: " sWiFt ,other",
+    languagesInRepository: [],
+    expected: true,
+    expectedApiCall: false,
+  },
+  {
+    name: "bypass kotlin",
+    features: [Feature.BypassToolcacheKotlinSwiftEnabled],
+    hasCustomCodeQL: false,
+    languagesInput: "other, KoTlIn ",
+    languagesInRepository: [],
+    expected: true,
+    expectedApiCall: false,
+  },
+  {
+    name: "bypass kotlin language from repository",
+    features: [Feature.BypassToolcacheKotlinSwiftEnabled],
+    hasCustomCodeQL: false,
+    languagesInput: "",
+    languagesInRepository: ["KoTlIn", "other"],
+    expected: true,
+    expectedApiCall: true,
+  },
+  {
+    name: "bypass swift language from repository",
+    features: [Feature.BypassToolcacheKotlinSwiftEnabled],
+    hasCustomCodeQL: false,
+    languagesInput: "",
+    languagesInRepository: ["SwiFt", "other"],
+    expected: true,
+    expectedApiCall: true,
+  },
+].forEach((args) => {
+  test(`shouldBypassToolcache: ${args.name}`, async (t) => {
+    const mockRequest = mockLangaugesInRepo(args.languagesInRepository);
+    const mockLogger = getRecordingLogger([]);
+    const featureEnablement = createFeatures(args.features);
+    const codeqlUrl = args.hasCustomCodeQL ? "custom-codeql-url" : undefined;
+    const actual = await util.shouldBypassToolcache(
+      featureEnablement,
+      codeqlUrl,
+      args.languagesInput,
+      mockRepositoryNwo,
+      mockLogger
+    );
+    t.deepEqual(actual, args.expected);
+    t.deepEqual(mockRequest.called, args.expectedApiCall);
+  });
+});

src/util.ts

@@ -14,12 +14,14 @@ import * as apiCompatibility from "./api-compatibility.json";
 import { CodeQL, CODEQL_VERSION_NEW_TRACING } from "./codeql";
 import {
   Config,
+  getRawLanguages,
   parsePacksSpecification,
   prettyPrintPack,
 } from "./config-utils";
 import { Feature, FeatureEnablement } from "./feature-flags";
-import { Language } from "./languages";
+import { KOTLIN_SWIFT_BYPASS, Language } from "./languages";
 import { Logger } from "./logging";
+import { RepositoryNwo } from "./repository";
 import { CODEQL_ACTION_TEST_MODE } from "./shared-environment";
 
 /**
@@ -832,3 +834,43 @@ export function isHostedRunner() {
     process.env["RUNNER_TOOL_CACHE"]?.includes("hostedtoolcache")
   );
 }
+
+/**
+ *
+ * @param featuresEnablement The features enabled for the current run
+ * @param languagesInput Languages input from the workflow
+ * @param repository The owner/name of the repository
+ * @param logger A logger
+ * @returns A boolean indicating whether or not the toolcache should be bypassed and the latest codeql should be downloaded.
+ */
+export async function shouldBypassToolcache(
+  featuresEnablement: FeatureEnablement,
+  codeqlUrl: string | undefined,
+  languagesInput: string | undefined,
+  repository: RepositoryNwo,
+  logger: Logger
+): Promise<boolean> {
+  // An explicit codeql url is specified, that means the toolcache will not be used.
+  if (codeqlUrl) {
+    return true;
+  }
+
+  // Check if the toolcache is disabled for all languages
+  if (await featuresEnablement.getValue(Feature.BypassToolcacheEnabled)) {
+    return true;
+  }
+
+  // Check if the toolcache is disabled for kotlin and swift.
+  if (
+    await featuresEnablement.getValue(Feature.BypassToolcacheKotlinSwiftEnabled)
+  ) {
+    // Now check to see if kotlin or swift is one of the languages being analyzed.
+    const { rawLanguages } = await getRawLanguages(
+      languagesInput,
+      repository,
+      logger
+    );
+    return rawLanguages.some((lang) => KOTLIN_SWIFT_BYPASS.includes(lang));
+  }
+  return false;
+}