github/codeql-action
1
0
Fork 0
mirror of https://github.com/github/codeql-action.git synced 2026-01-02 04:30:14 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into henrymercer/upgrade-typescript

Browse Source
This commit is contained in:
Henry Mercer
2023-01-19 16:15:51 +00:00
parent 4306b97e96 60e5868d6e
commit ba93815e25
31 changed files with 2179 additions and 880 deletions
Download Patch File Download Diff File Expand all files Collapse all files

430
src/codeql.test.ts
View File

@@ -1,5 +1,5 @@
import * as fs from "fs";
import * as path from "path";
import path from "path";
import * as toolrunner from "@actions/exec/lib/toolrunner";
import * as toolcache from "@actions/tool-cache";
@@ -11,14 +11,19 @@ import nock from "nock";
import * as sinon from "sinon";
import * as actionsUtil from "./actions-util";
import * as api from "./api-client";
import { GitHubApiDetails } from "./api-client";
import * as codeql from "./codeql";
import { AugmentationProperties, Config } from "./config-utils";
import * as defaults from "./defaults.json";
import { Feature, featureConfig } from "./feature-flags";
import {
CodeQLDefaultVersionInfo,
Feature,
featureConfig,
} from "./feature-flags";
import { Language } from "./languages";
import { getRunnerLogger } from "./logging";
import { setupTests, setupActionsVars, createFeatures } from "./testing-utils";
import { setupTests, createFeatures, setupActionsVars } from "./testing-utils";
import * as util from "./util";
import { initializeEnvironment } from "./util";
@@ -36,6 +41,11 @@ const sampleGHAEApiDetails = {
apiURL: "https://example.githubenterprise.com/api/v3",
};
const SAMPLE_DEFAULT_CLI_VERSION: CodeQLDefaultVersionInfo = {
cliVersion: "2.0.0",
variant: util.GitHubVariant.DOTCOM,
};
let stubConfig: Config;
test.beforeEach(() => {
@@ -73,7 +83,7 @@ test.beforeEach(() => {
* @returns the download URL for the bundle. This can be passed to the tools parameter of
* `codeql.setupCodeQL`.
*/
async function mockDownloadApi({
function mockDownloadApi({
apiDetails = sampleApiDetails,
isPinned,
tagName,
@@ -81,7 +91,7 @@ async function mockDownloadApi({
apiDetails?: GitHubApiDetails;
isPinned?: boolean;
tagName: string;
}): Promise<string> {
}): string {
const platform =
process.platform === "win32"
? "win64"
@@ -109,27 +119,67 @@ async function mockDownloadApi({
async function installIntoToolcache({
apiDetails = sampleApiDetails,
cliVersion,
isPinned,
tagName,
tmpDir,
}: {
apiDetails?: GitHubApiDetails;
cliVersion?: string;
isPinned: boolean;
tagName: string;
tmpDir: string;
}) {
const url = await mockDownloadApi({ apiDetails, isPinned, tagName });
const url = mockDownloadApi({ apiDetails, isPinned, tagName });
await codeql.setupCodeQL(
url,
cliVersion !== undefined ? undefined : url,
apiDetails,
tmpDir,
util.GitHubVariant.DOTCOM,
util.GitHubVariant.GHES,
false,
cliVersion !== undefined
? { cliVersion, tagName, variant: util.GitHubVariant.GHES }
: SAMPLE_DEFAULT_CLI_VERSION,
getRunnerLogger(true),
false
);
}
function mockReleaseApi({
apiDetails = sampleApiDetails,
assetNames,
tagName,
}: {
apiDetails?: GitHubApiDetails;
assetNames: string[];
tagName: string;
}): nock.Scope {
return nock(apiDetails.apiURL!)
.get(`/repos/github/codeql-action/releases/tags/${tagName}`)
.reply(200, {
assets: assetNames.map((name) => ({
name,
})),
tag_name: tagName,
});
}
function mockApiDetails(apiDetails: GitHubApiDetails) {
// This is a workaround to mock `api.getApiDetails()` since it doesn't seem to be possible to
// mock this directly. The difficulty is that `getApiDetails()` is called locally in
// `api-client.ts`, but `sinon.stub(api, "getApiDetails")` only affects calls to
// `getApiDetails()` via an imported `api` module.
sinon
.stub(actionsUtil, "getRequiredInput")
.withArgs("token")
.returns(apiDetails.auth);
const requiredEnvParamStub = sinon.stub(util, "getRequiredEnvParam");
requiredEnvParamStub.withArgs("GITHUB_SERVER_URL").returns(apiDetails.url);
requiredEnvParamStub
.withArgs("GITHUB_API_URL")
.returns(apiDetails.apiURL || "");
}
test("downloads and caches explicitly requested bundles that aren't in the toolcache", async (t) => {
await util.withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);
@@ -139,7 +189,7 @@ test("downloads and caches explicitly requested bundles that aren't in the toolc
for (let i = 0; i < versions.length; i++) {
const version = versions[i];
const url = await mockDownloadApi({
const url = mockDownloadApi({
tagName: `codeql-bundle-${version}`,
isPinned: false,
});
@@ -149,11 +199,12 @@ test("downloads and caches explicitly requested bundles that aren't in the toolc
tmpDir,
util.GitHubVariant.DOTCOM,
false,
SAMPLE_DEFAULT_CLI_VERSION,
getRunnerLogger(true),
false
);
t.assert(toolcache.find("CodeQL", `0.0.0-${version}`));
t.is(result.toolsVersion, version);
t.is(result.toolsVersion, `0.0.0-${version}`);
}
t.is(toolcache.findAllVersions("CodeQL").length, 2);
@@ -170,7 +221,7 @@ test("downloads an explicitly requested bundle even if a different version is ca
tmpDir,
});
const url = await mockDownloadApi({
const url = mockDownloadApi({
tagName: "codeql-bundle-20200610",
});
const result = await codeql.setupCodeQL(
@@ -179,71 +230,196 @@ test("downloads an explicitly requested bundle even if a different version is ca
tmpDir,
util.GitHubVariant.DOTCOM,
false,
SAMPLE_DEFAULT_CLI_VERSION,
getRunnerLogger(true),
false
);
t.assert(toolcache.find("CodeQL", "0.0.0-20200610"));
t.deepEqual(result.toolsVersion, "20200610");
t.deepEqual(result.toolsVersion, "0.0.0-20200610");
});
});
test("uses a cached bundle when no tools input is given", async (t) => {
await util.withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);
const EXPLICITLY_REQUESTED_BUNDLE_TEST_CASES = [
{
cliVersion: "2.10.0",
expectedToolcacheVersion: "2.10.0-20200610",
},
{
cliVersion: "2.10.0-pre",
expectedToolcacheVersion: "0.0.0-20200610",
},
];
await installIntoToolcache({
tagName: "codeql-bundle-20200601",
isPinned: true,
tmpDir,
for (const {
cliVersion,
expectedToolcacheVersion,
} of EXPLICITLY_REQUESTED_BUNDLE_TEST_CASES) {
test(`caches an explicitly requested bundle containing CLI ${cliVersion} as ${expectedToolcacheVersion}`, async (t) => {
await util.withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);
mockApiDetails(sampleApiDetails);
sinon.stub(actionsUtil, "isRunningLocalAction").returns(true);
const releaseApiMock = mockReleaseApi({
assetNames: [`cli-version-${cliVersion}.txt`],
tagName: "codeql-bundle-20200610",
});
const url = mockDownloadApi({
tagName: "codeql-bundle-20200610",
});
const result = await codeql.setupCodeQL(
url,
sampleApiDetails,
tmpDir,
util.GitHubVariant.DOTCOM,
false,
SAMPLE_DEFAULT_CLI_VERSION,
getRunnerLogger(true),
false
);
t.assert(releaseApiMock.isDone(), "Releases API should have been called");
t.assert(toolcache.find("CodeQL", expectedToolcacheVersion));
t.deepEqual(result.toolsVersion, cliVersion);
});
const result = await codeql.setupCodeQL(
undefined,
sampleApiDetails,
tmpDir,
util.GitHubVariant.DOTCOM,
false,
getRunnerLogger(true),
false
);
t.deepEqual(result.toolsVersion, "0.0.0-20200601");
const cachedVersions = toolcache.findAllVersions("CodeQL");
t.is(cachedVersions.length, 1);
});
});
}
test("downloads bundle if only an unpinned version is cached", async (t) => {
await util.withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);
for (const { isCached, tagName, toolcacheCliVersion } of [
{
isCached: true,
tagName: "codeql-bundle-20230101",
toolcacheCliVersion: SAMPLE_DEFAULT_CLI_VERSION.cliVersion,
},
{
isCached: true,
// By leaving toolcacheCliVersion undefined, the bundle will be installed
// into the toolcache as `${SAMPLE_DEFAULT_CLI_VERSION.cliVersion}-20230101`.
// This lets us test that `x.y.z-yyyymmdd` toolcache versions are used if an
// `x.y.z` version isn't in the toolcache.
tagName: `codeql-bundle-${SAMPLE_DEFAULT_CLI_VERSION.cliVersion}-20230101`,
},
{
isCached: false,
tagName: "codeql-bundle-20230101",
},
]) {
test(`uses default version on Dotcom when default version bundle ${tagName} is ${
isCached ? "" : "not "
}cached`, async (t) => {
await util.withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);
await installIntoToolcache({
tagName: "codeql-bundle-20200601",
isPinned: false,
tmpDir,
if (isCached) {
await installIntoToolcache({
cliVersion: toolcacheCliVersion,
tagName,
isPinned: true,
tmpDir,
});
} else {
mockDownloadApi({
tagName,
});
sinon.stub(api, "getApiClient").value(() => ({
repos: {
listReleases: sinon.stub().resolves(undefined),
},
paginate: sinon.stub().resolves([
{
assets: [
{
name: "cli-version-2.0.0.txt",
},
],
tag_name: tagName,
},
]),
}));
}
const result = await codeql.setupCodeQL(
undefined,
sampleApiDetails,
tmpDir,
util.GitHubVariant.DOTCOM,
false,
SAMPLE_DEFAULT_CLI_VERSION,
getRunnerLogger(true),
false
);
t.is(result.toolsVersion, SAMPLE_DEFAULT_CLI_VERSION.cliVersion);
});
await mockDownloadApi({
tagName: defaults.bundleVersion,
});
const result = await codeql.setupCodeQL(
undefined,
sampleApiDetails,
tmpDir,
util.GitHubVariant.DOTCOM,
false,
getRunnerLogger(true),
false
);
t.deepEqual(
result.toolsVersion,
defaults.bundleVersion.replace("codeql-bundle-", "")
);
const cachedVersions = toolcache.findAllVersions("CodeQL");
t.is(cachedVersions.length, 2);
});
});
}
for (const variant of [util.GitHubVariant.GHAE, util.GitHubVariant.GHES]) {
test(`uses a cached bundle when no tools input is given on ${util.GitHubVariant[variant]}`, async (t) => {
await util.withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);
await installIntoToolcache({
tagName: "codeql-bundle-20200601",
isPinned: true,
tmpDir,
});
const result = await codeql.setupCodeQL(
undefined,
sampleApiDetails,
tmpDir,
variant,
false,
{
cliVersion: defaults.cliVersion,
tagName: defaults.bundleVersion,
variant,
},
getRunnerLogger(true),
false
);
t.deepEqual(result.toolsVersion, "0.0.0-20200601");
const cachedVersions = toolcache.findAllVersions("CodeQL");
t.is(cachedVersions.length, 1);
});
});
test(`downloads bundle if only an unpinned version is cached on ${util.GitHubVariant[variant]}`, async (t) => {
await util.withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);
await installIntoToolcache({
tagName: "codeql-bundle-20200601",
isPinned: false,
tmpDir,
});
mockDownloadApi({
tagName: defaults.bundleVersion,
});
const result = await codeql.setupCodeQL(
undefined,
sampleApiDetails,
tmpDir,
variant,
false,
{
cliVersion: defaults.cliVersion,
tagName: defaults.bundleVersion,
variant,
},
getRunnerLogger(true),
false
);
t.deepEqual(result.toolsVersion, defaults.cliVersion);
const cachedVersions = toolcache.findAllVersions("CodeQL");
t.is(cachedVersions.length, 2);
});
});
}
test('downloads bundle if "latest" tools specified but not cached', async (t) => {
await util.withTmpDir(async (tmpDir) => {
@@ -255,7 +431,7 @@ test('downloads bundle if "latest" tools specified but not cached', async (t) =>
tmpDir,
});
await mockDownloadApi({
mockDownloadApi({
tagName: defaults.bundleVersion,
});
const result = await codeql.setupCodeQL(
@@ -264,69 +440,17 @@ test('downloads bundle if "latest" tools specified but not cached', async (t) =>
tmpDir,
util.GitHubVariant.DOTCOM,
false,
SAMPLE_DEFAULT_CLI_VERSION,
getRunnerLogger(true),
false
);
t.deepEqual(
result.toolsVersion,
defaults.bundleVersion.replace("codeql-bundle-", "")
);
t.deepEqual(result.toolsVersion, defaults.cliVersion);
const cachedVersions = toolcache.findAllVersions("CodeQL");
t.is(cachedVersions.length, 2);
});
});
const TOOLCACHE_BYPASS_TEST_CASES: Array<
[boolean, string | undefined, boolean]
> = [
[true, undefined, true],
[false, undefined, false],
[
true,
"https://github.com/github/codeql-action/releases/download/codeql-bundle-20200601/codeql-bundle.tar.gz",
false,
],
];
for (const [
isFeatureEnabled,
toolsInput,
shouldToolcacheBeBypassed,
] of TOOLCACHE_BYPASS_TEST_CASES) {
test(`download codeql bundle ${
shouldToolcacheBeBypassed ? "bypasses" : "does not bypass"
} toolcache when feature ${
isFeatureEnabled ? "enabled" : "disabled"
} and tools: ${toolsInput} passed`, async (t) => {
await util.withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);
await installIntoToolcache({
tagName: "codeql-bundle-20200601",
isPinned: true,
tmpDir,
});
await mockDownloadApi({
tagName: defaults.bundleVersion,
});
await codeql.setupCodeQL(
toolsInput,
sampleApiDetails,
tmpDir,
util.GitHubVariant.DOTCOM,
isFeatureEnabled,
getRunnerLogger(true),
false
);
const cachedVersions = toolcache.findAllVersions("CodeQL");
t.is(cachedVersions.length, shouldToolcacheBeBypassed ? 2 : 1);
});
});
}
test("download codeql bundle from github ae endpoint", async (t) => {
await util.withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);
@@ -366,22 +490,7 @@ test("download codeql bundle from github ae endpoint", async (t) => {
path.join(__dirname, `/../src/testdata/codeql-bundle-pinned.tar.gz`)
);
// This is a workaround to mock `api.getApiDetails()` since it doesn't seem to be possible to
// mock this directly. The difficulty is that `getApiDetails()` is called locally in
// `api-client.ts`, but `sinon.stub(api, "getApiDetails")` only affects calls to
// `getApiDetails()` via an imported `api` module.
sinon
.stub(actionsUtil, "getRequiredInput")
.withArgs("token")
.returns(sampleGHAEApiDetails.auth);
const requiredEnvParamStub = sinon.stub(util, "getRequiredEnvParam");
requiredEnvParamStub
.withArgs("GITHUB_SERVER_URL")
.returns(sampleGHAEApiDetails.url);
requiredEnvParamStub
.withArgs("GITHUB_API_URL")
.returns(sampleGHAEApiDetails.apiURL);
mockApiDetails(sampleGHAEApiDetails);
sinon.stub(actionsUtil, "isRunningLocalAction").returns(false);
process.env["GITHUB_ACTION_REPOSITORY"] = "github/codeql-action";
@@ -391,6 +500,11 @@ test("download codeql bundle from github ae endpoint", async (t) => {
tmpDir,
util.GitHubVariant.GHAE,
false,
{
cliVersion: defaults.cliVersion,
tagName: defaults.bundleVersion,
variant: util.GitHubVariant.GHAE,
},
getRunnerLogger(true),
false
);
@@ -400,38 +514,6 @@ test("download codeql bundle from github ae endpoint", async (t) => {
});
});
test("parse codeql bundle url version", (t) => {
t.deepEqual(
codeql.getCodeQLURLVersion(
"https://github.com/.../codeql-bundle-20200601/..."
),
"20200601"
);
});
test("convert to semver", (t) => {
const tests = {
"20200601": "0.0.0-20200601",
"20200601.0": "0.0.0-20200601.0",
"20200601.0.0": "20200601.0.0",
"1.2.3": "1.2.3",
"1.2.3-alpha": "1.2.3-alpha",
"1.2.3-beta.1": "1.2.3-beta.1",
};
for (const [version, expectedVersion] of Object.entries(tests)) {
try {
const parsedVersion = codeql.convertToSemVer(
version,
getRunnerLogger(true)
);
t.deepEqual(parsedVersion, expectedVersion);
} catch (e) {
t.fail(e instanceof Error ? e.message : String(e));
}
}
});
test("getExtraOptions works for explicit paths", (t) => {
t.deepEqual(codeql.getExtraOptions({}, ["foo"], []), []);
@@ -473,24 +555,6 @@ test("getExtraOptions throws for bad content", (t) => {
);
});
test("getCodeQLActionRepository", (t) => {
const logger = getRunnerLogger(true);
initializeEnvironment("1.2.3");
// isRunningLocalAction() === true
delete process.env["GITHUB_ACTION_REPOSITORY"];
process.env["RUNNER_TEMP"] = path.dirname(__dirname);
const repoLocalRunner = codeql.getCodeQLActionRepository(logger);
t.deepEqual(repoLocalRunner, "github/codeql-action");
// isRunningLocalAction() === false
sinon.stub(actionsUtil, "isRunningLocalAction").returns(false);
process.env["GITHUB_ACTION_REPOSITORY"] = "xxx/yyy";
const repoEnv = codeql.getCodeQLActionRepository(logger);
t.deepEqual(repoEnv, "xxx/yyy");
});
test("databaseInterpretResults() does not set --sarif-add-query-help for 2.7.0", async (t) => {
const runnerConstructorStub = stubToolRunnerConstructor();
const codeqlObject = await codeql.getCodeQLForTesting();

333
src/codeql.ts
View File

@@ -1,29 +1,23 @@
import * as fs from "fs";
import { OutgoingHttpHeaders } from "http";
import * as path from "path";
import * as toolrunner from "@actions/exec/lib/toolrunner";
import * as toolcache from "@actions/tool-cache";
import { default as deepEqual } from "fast-deep-equal";
import * as yaml from "js-yaml";
import * as semver from "semver";
import { v4 as uuidV4 } from "uuid";
import { getOptionalInput, isRunningLocalAction } from "./actions-util";
import { getOptionalInput } from "./actions-util";
import * as api from "./api-client";
import { Config } from "./config-utils";
import * as defaults from "./defaults.json"; // Referenced from codeql-action-sync-tool!
import { errorMatchers } from "./error-matcher";
import { FeatureEnablement } from "./feature-flags";
import { CodeQLDefaultVersionInfo, FeatureEnablement } from "./feature-flags";
import { isTracedLanguage, Language } from "./languages";
import { Logger } from "./logging";
import * as setupCodeql from "./setup-codeql";
import { toolrunnerErrorCatcher } from "./toolrunner-error-catcher";
import {
getTrapCachingExtractorConfigArgs,
getTrapCachingExtractorConfigArgsForLang,
} from "./trap-caching";
import * as util from "./util";
import { assertNever, isGoodVersion } from "./util";
type Options = Array<string | number | boolean>;
@@ -232,9 +226,6 @@ interface PackDownloadItem {
*/
let cachedCodeQL: CodeQL | undefined = undefined;
const CODEQL_BUNDLE_VERSION = defaults.bundleVersion;
export const CODEQL_DEFAULT_ACTION_REPOSITORY = "github/codeql-action";
/**
* The oldest version of CodeQL that the Action will run with. This should be
* at least three minor versions behind the current version and must include the
@@ -286,262 +277,6 @@ export const CODEQL_VERSION_ML_POWERED_QUERIES_WINDOWS = "2.9.0";
*/
export const CODEQL_VERSION_BETTER_RESOLVE_LANGUAGES = "2.10.3";
function getCodeQLBundleName(): string {
let platform: string;
if (process.platform === "win32") {
platform = "win64";
} else if (process.platform === "linux") {
platform = "linux64";
} else if (process.platform === "darwin") {
platform = "osx64";
} else {
return "codeql-bundle.tar.gz";
}
return `codeql-bundle-${platform}.tar.gz`;
}
export function getCodeQLActionRepository(logger: Logger): string {
if (isRunningLocalAction()) {
// This handles the case where the Action does not come from an Action repository,
// e.g. our integration tests which use the Action code from the current checkout.
// In these cases, the GITHUB_ACTION_REPOSITORY environment variable is not set.
logger.info(
"The CodeQL Action is checked out locally. Using the default CodeQL Action repository."
);
return CODEQL_DEFAULT_ACTION_REPOSITORY;
}
return util.getRequiredEnvParam("GITHUB_ACTION_REPOSITORY");
}
async function getCodeQLBundleDownloadURL(
apiDetails: api.GitHubApiDetails,
variant: util.GitHubVariant,
logger: Logger
): Promise<string> {
const codeQLActionRepository = getCodeQLActionRepository(logger);
const potentialDownloadSources = [
// This GitHub instance, and this Action.
[apiDetails.url, codeQLActionRepository],
// This GitHub instance, and the canonical Action.
[apiDetails.url, CODEQL_DEFAULT_ACTION_REPOSITORY],
// GitHub.com, and the canonical Action.
[util.GITHUB_DOTCOM_URL, CODEQL_DEFAULT_ACTION_REPOSITORY],
];
// We now filter out any duplicates.
// Duplicates will happen either because the GitHub instance is GitHub.com, or because the Action is not a fork.
const uniqueDownloadSources = potentialDownloadSources.filter(
(source, index, self) => {
return !self.slice(0, index).some((other) => deepEqual(source, other));
}
);
const codeQLBundleName = getCodeQLBundleName();
if (variant === util.GitHubVariant.GHAE) {
try {
const release = await api
.getApiClient()
.request("GET /enterprise/code-scanning/codeql-bundle/find/{tag}", {
tag: CODEQL_BUNDLE_VERSION,
});
const assetID = release.data.assets[codeQLBundleName];
if (assetID !== undefined) {
const download = await api
.getApiClient()
.request(
"GET /enterprise/code-scanning/codeql-bundle/download/{asset_id}",
{ asset_id: assetID }
);
const downloadURL = download.data.url;
logger.info(
`Found CodeQL bundle at GitHub AE endpoint with URL ${downloadURL}.`
);
return downloadURL;
} else {
logger.info(
`Attempted to fetch bundle from GitHub AE endpoint but the bundle ${codeQLBundleName} was not found in the assets ${JSON.stringify(
release.data.assets
)}.`
);
}
} catch (e) {
logger.info(
`Attempted to fetch bundle from GitHub AE endpoint but got error ${e}.`
);
}
}
for (const downloadSource of uniqueDownloadSources) {
const [apiURL, repository] = downloadSource;
// If we've reached the final case, short-circuit the API check since we know the bundle exists and is public.
if (
apiURL === util.GITHUB_DOTCOM_URL &&
repository === CODEQL_DEFAULT_ACTION_REPOSITORY
) {
break;
}
const [repositoryOwner, repositoryName] = repository.split("/");
try {
const release = await api.getApiClient().repos.getReleaseByTag({
owner: repositoryOwner,
repo: repositoryName,
tag: CODEQL_BUNDLE_VERSION,
});
for (const asset of release.data.assets) {
if (asset.name === codeQLBundleName) {
logger.info(
`Found CodeQL bundle in ${downloadSource[1]} on ${downloadSource[0]} with URL ${asset.url}.`
);
return asset.url;
}
}
} catch (e) {
logger.info(
`Looked for CodeQL bundle in ${downloadSource[1]} on ${downloadSource[0]} but got error ${e}.`
);
}
}
return `https://github.com/${CODEQL_DEFAULT_ACTION_REPOSITORY}/releases/download/${CODEQL_BUNDLE_VERSION}/${codeQLBundleName}`;
}
type CodeQLToolsSource =
| { codeqlTarPath: string; sourceType: "local"; toolsVersion: "local" }
| {
codeqlFolder: string;
sourceType: "toolcache";
toolsVersion: string;
}
| {
codeqlURL: string;
semanticVersion: string;
sourceType: "download";
toolsVersion: string;
};
async function getCodeQLSource(
toolsInput: string | undefined,
bypassToolcache: boolean,
apiDetails: api.GitHubApiDetails,
variant: util.GitHubVariant,
logger: Logger
): Promise<CodeQLToolsSource> {
if (toolsInput && toolsInput !== "latest" && !toolsInput.startsWith("http")) {
return {
codeqlTarPath: toolsInput,
sourceType: "local",
toolsVersion: "local",
};
}
const forceLatestReason =
// We use the special value of 'latest' to prioritize the version in the
// defaults over any pinned cached version.
toolsInput === "latest"
? '"tools: latest" was requested'
: // If the user hasn't requested a particular CodeQL version, then bypass
// the toolcache when the appropriate feature is enabled. This
// allows us to quickly rollback a broken bundle that has made its way
// into the toolcache.
toolsInput === undefined && bypassToolcache
? "a specific version of CodeQL was not requested and the bypass toolcache feature is enabled"
: undefined;
const forceLatest = forceLatestReason !== undefined;
if (forceLatest) {
logger.debug(
`Forcing the latest version of the CodeQL tools since ${forceLatestReason}.`
);
}
const codeqlURL = forceLatest ? undefined : toolsInput;
const requestedSemVer = convertToSemVer(
getCodeQLURLVersion(codeqlURL || `/${CODEQL_BUNDLE_VERSION}/`),
logger
);
// If we find the specified version, we always use that.
const codeqlFolder = toolcache.find("CodeQL", requestedSemVer);
if (codeqlFolder) {
return {
codeqlFolder,
sourceType: "toolcache",
toolsVersion: requestedSemVer,
};
}
// If we don't find the requested version, in some cases we may allow a
// different version to save download time if the version hasn't been
// specified explicitly (in which case we always honor it).
if (!codeqlURL && !forceLatest) {
const codeqlVersions = toolcache.findAllVersions("CodeQL");
if (codeqlVersions.length === 1 && isGoodVersion(codeqlVersions[0])) {
const tmpCodeqlFolder = toolcache.find("CodeQL", codeqlVersions[0]);
if (fs.existsSync(path.join(tmpCodeqlFolder, "pinned-version"))) {
logger.debug(
`CodeQL in cache overriding the default ${CODEQL_BUNDLE_VERSION}`
);
return {
codeqlFolder: tmpCodeqlFolder,
sourceType: "toolcache",
toolsVersion: codeqlVersions[0],
};
}
}
}
return {
codeqlURL:
codeqlURL ||
(await getCodeQLBundleDownloadURL(apiDetails, variant, logger)),
semanticVersion: requestedSemVer,
sourceType: "download",
toolsVersion:
semver.prerelease(requestedSemVer)?.join(".") || requestedSemVer,
};
}
async function downloadCodeQL(
codeqlURL: string,
semanticVersion: string,
apiDetails: api.GitHubApiDetails,
tempDir: string,
logger: Logger
): Promise<string> {
const parsedCodeQLURL = new URL(codeqlURL);
const searchParams = new URLSearchParams(parsedCodeQLURL.search);
const headers: OutgoingHttpHeaders = {
accept: "application/octet-stream",
};
// We only want to provide an authorization header if we are downloading
// from the same GitHub instance the Action is running on.
// This avoids leaking Enterprise tokens to dotcom.
// We also don't want to send an authorization header if there's already a token provided in the URL.
if (searchParams.has("token")) {
logger.debug("CodeQL tools URL contains an authorization token.");
} else if (codeqlURL.startsWith(`${apiDetails.url}/`)) {
logger.debug("Providing an authorization token to download CodeQL tools.");
headers.authorization = `token ${apiDetails.auth}`;
} else {
logger.debug("Downloading CodeQL tools without an authorization token.");
}
logger.info(
`Downloading CodeQL tools from ${codeqlURL}. This may take a while.`
);
const dest = path.join(tempDir, uuidV4() as string);
const finalHeaders = Object.assign(
{ "User-Agent": "CodeQL Action" },
headers
);
const codeqlPath = await toolcache.downloadTool(
codeqlURL,
dest,
undefined,
finalHeaders
);
logger.debug(`CodeQL bundle download to ${codeqlPath} complete.`);
const codeqlExtracted = await toolcache.extractTar(codeqlPath);
return await toolcache.cacheDir(codeqlExtracted, "CodeQL", semanticVersion);
}
/**
* Set up CodeQL CLI access.
*
@@ -550,6 +285,7 @@ async function downloadCodeQL(
* @param tempDir
* @param variant
* @param bypassToolcache
* @param defaultCliVersion
* @param logger
* @param checkVersion Whether to check that CodeQL CLI meets the minimum
* version requirement. Must be set to true outside tests.
@@ -561,41 +297,20 @@ export async function setupCodeQL(
tempDir: string,
variant: util.GitHubVariant,
bypassToolcache: boolean,
defaultCliVersion: CodeQLDefaultVersionInfo,
logger: Logger,
checkVersion: boolean
): Promise<{ codeql: CodeQL; toolsVersion: string }> {
try {
const source = await getCodeQLSource(
const { codeqlFolder, toolsVersion } = await setupCodeql.setupCodeQLBundle(
toolsInput,
bypassToolcache,
apiDetails,
tempDir,
variant,
bypassToolcache,
defaultCliVersion,
logger
);
let codeqlFolder: string;
switch (source.sourceType) {
case "local":
codeqlFolder = await toolcache.extractTar(source.codeqlTarPath);
break;
case "toolcache":
codeqlFolder = source.codeqlFolder;
logger.debug(`CodeQL found in cache ${codeqlFolder}`);
break;
case "download":
codeqlFolder = await downloadCodeQL(
source.codeqlURL,
source.semanticVersion,
apiDetails,
tempDir,
logger
);
break;
default:
assertNever(source);
}
let codeqlCmd = path.join(codeqlFolder, "codeql", "codeql");
if (process.platform === "win32") {
codeqlCmd += ".exe";
@@ -604,39 +319,13 @@ export async function setupCodeQL(
}
cachedCodeQL = await getCodeQLForCmd(codeqlCmd, checkVersion);
return { codeql: cachedCodeQL, toolsVersion: source.toolsVersion };
return { codeql: cachedCodeQL, toolsVersion };
} catch (e) {
logger.error(e instanceof Error ? e : new Error(String(e)));
throw new Error("Unable to download and extract CodeQL CLI");
}
}
export function getCodeQLURLVersion(url: string): string {
const match = url.match(/\/codeql-bundle-(.*)\//);
if (match === null || match.length < 2) {
throw new Error(
`Malformed tools url: ${url}. Version could not be inferred`
);
}
return match[1];
}
export function convertToSemVer(version: string, logger: Logger): string {
if (!semver.valid(version)) {
logger.debug(
`Bundle version ${version} is not in SemVer format. Will treat it as pre-release 0.0.0-${version}.`
);
version = `0.0.0-${version}`;
}
const s = semver.clean(version);
if (!s) {
throw new Error(`Bundle version ${version} is not in SemVer format.`);
}
return s;
}
/**
* Use the CodeQL executable located at the given path.
*/
@@ -744,7 +433,7 @@ export async function getCodeQLForTesting(
* version requirement. Must be set to true outside tests.
* @returns A new CodeQL object
*/
async function getCodeQLForCmd(
export async function getCodeQLForCmd(
cmd: string,
checkVersion: boolean
): Promise<CodeQL> {

91
src/feature-flags.test.ts
View File

@@ -1,8 +1,9 @@
import * as fs from "fs";
import * as path from "path";
import test from "ava";
import test, { ExecutionContext } from "ava";
import * as defaults from "./defaults.json";
import {
Feature,
featureConfig,
@@ -371,7 +372,93 @@ test("Environment variable can override feature flag cache", async (t) => {
});
});
function assertAllFeaturesUndefinedInApi(t, loggedMessages: LoggedMessage[]) {
for (const variant of [GitHubVariant.GHAE, GitHubVariant.GHES]) {
test(`selects CLI from defaults.json on ${GitHubVariant[variant]}`, async (t) => {
await withTmpDir(async (tmpDir) => {
const features = setUpFeatureFlagTests(tmpDir);
t.deepEqual(await features.getDefaultCliVersion(variant), {
cliVersion: defaults.cliVersion,
tagName: defaults.bundleVersion,
variant,
});
});
});
}
test("selects CLI v2.12.1 on Dotcom when feature flags enable v2.12.0 and v2.12.1", async (t) => {
await withTmpDir(async (tmpDir) => {
const featureEnablement = setUpFeatureFlagTests(tmpDir);
const expectedFeatureEnablement = initializeFeatures(true);
expectedFeatureEnablement["default_codeql_version_2_12_0_enabled"] = true;
expectedFeatureEnablement["default_codeql_version_2_12_1_enabled"] = true;
expectedFeatureEnablement["default_codeql_version_2_12_2_enabled"] = false;
expectedFeatureEnablement["default_codeql_version_2_12_3_enabled"] = false;
expectedFeatureEnablement["default_codeql_version_2_12_4_enabled"] = false;
expectedFeatureEnablement["default_codeql_version_2_12_5_enabled"] = false;
mockFeatureFlagApiEndpoint(200, expectedFeatureEnablement);
t.deepEqual(
await featureEnablement.getDefaultCliVersion(GitHubVariant.DOTCOM),
{
cliVersion: "2.12.1",
variant: GitHubVariant.DOTCOM,
}
);
});
});
test(`selects CLI v2.11.6 on Dotcom when no default version feature flags are enabled`, async (t) => {
await withTmpDir(async (tmpDir) => {
const featureEnablement = setUpFeatureFlagTests(tmpDir);
const expectedFeatureEnablement = initializeFeatures(true);
mockFeatureFlagApiEndpoint(200, expectedFeatureEnablement);
t.deepEqual(
await featureEnablement.getDefaultCliVersion(GitHubVariant.DOTCOM),
{
cliVersion: "2.11.6",
variant: GitHubVariant.DOTCOM,
}
);
});
});
test("ignores invalid version numbers in default version feature flags", async (t) => {
await withTmpDir(async (tmpDir) => {
const loggedMessages = [];
const featureEnablement = setUpFeatureFlagTests(
tmpDir,
getRecordingLogger(loggedMessages)
);
const expectedFeatureEnablement = initializeFeatures(true);
expectedFeatureEnablement["default_codeql_version_2_12_0_enabled"] = true;
expectedFeatureEnablement["default_codeql_version_2_12_1_enabled"] = true;
expectedFeatureEnablement["default_codeql_version_2_12_invalid_enabled"] =
true;
mockFeatureFlagApiEndpoint(200, expectedFeatureEnablement);
t.deepEqual(
await featureEnablement.getDefaultCliVersion(GitHubVariant.DOTCOM),
{
cliVersion: "2.12.1",
variant: GitHubVariant.DOTCOM,
}
);
t.assert(
loggedMessages.find(
(v: LoggedMessage) =>
v.type === "warning" &&
v.message ===
"Ignoring feature flag default_codeql_version_2_12_invalid_enabled as it does not specify a valid CodeQL version."
) !== undefined
);
});
});
function assertAllFeaturesUndefinedInApi(
t: ExecutionContext<unknown>,
loggedMessages: LoggedMessage[]
) {
for (const feature of Object.keys(featureConfig)) {
t.assert(
loggedMessages.find(

104
src/feature-flags.ts
View File

@@ -1,13 +1,35 @@
import * as fs from "fs";
import * as path from "path";
import * as semver from "semver";
import { getApiClient } from "./api-client";
import { CodeQL } from "./codeql";
import * as defaults from "./defaults.json";
import { Logger } from "./logging";
import { RepositoryNwo } from "./repository";
import * as util from "./util";
const DEFAULT_VERSION_FEATURE_FLAG_PREFIX = "default_codeql_version_";
const DEFAULT_VERSION_FEATURE_FLAG_SUFFIX = "_enabled";
const MINIMUM_ENABLED_CODEQL_VERSION = "2.11.6";
export type CodeQLDefaultVersionInfo =
| {
cliVersion: string;
variant: util.GitHubVariant.DOTCOM;
}
| {
cliVersion: string;
tagName: string;
variant: util.GitHubVariant.GHAE | util.GitHubVariant.GHES;
};
export interface FeatureEnablement {
/** Gets the default version of the CodeQL tools. */
getDefaultCliVersion(
variant: util.GitHubVariant
): Promise<CodeQLDefaultVersionInfo>;
getValue(feature: Feature, codeql?: CodeQL): Promise<boolean>;
}
@@ -91,6 +113,12 @@ export class Features implements FeatureEnablement {
);
}
async getDefaultCliVersion(
variant: util.GitHubVariant
): Promise<CodeQLDefaultVersionInfo> {
return await this.gitHubFeatureFlags.getDefaultCliVersion(variant);
}
/**
*
* @param feature The feature to check.
@@ -153,6 +181,74 @@ class GitHubFeatureFlags implements FeatureEnablement {
/**/
}
private getCliVersionFromFeatureFlag(f: string): string | undefined {
if (
!f.startsWith(DEFAULT_VERSION_FEATURE_FLAG_PREFIX) ||
!f.endsWith(DEFAULT_VERSION_FEATURE_FLAG_SUFFIX)
) {
return undefined;
}
const version = f
.substring(
DEFAULT_VERSION_FEATURE_FLAG_PREFIX.length,
f.length - DEFAULT_VERSION_FEATURE_FLAG_SUFFIX.length
)
.replace(/_/g, ".");
if (!semver.valid(version)) {
this.logger.warning(
`Ignoring feature flag ${f} as it does not specify a valid CodeQL version.`
);
return undefined;
}
return version;
}
async getDefaultCliVersion(
variant: util.GitHubVariant
): Promise<CodeQLDefaultVersionInfo> {
if (variant === util.GitHubVariant.DOTCOM) {
return {
cliVersion: await this.getDefaultDotcomCliVersion(),
variant,
};
}
return {
cliVersion: defaults.cliVersion,
tagName: defaults.bundleVersion,
variant,
};
}
async getDefaultDotcomCliVersion(): Promise<string> {
const response = await this.getAllFeatures();
const enabledFeatureFlagCliVersions = Object.entries(response)
.map(([f, isEnabled]) =>
isEnabled ? this.getCliVersionFromFeatureFlag(f) : undefined
)
.filter((f) => f !== undefined)
.map((f) => f as string);
if (enabledFeatureFlagCliVersions.length === 0) {
this.logger.debug(
"Feature flags do not specify a default CLI version. Falling back to CLI version " +
`${MINIMUM_ENABLED_CODEQL_VERSION}.`
);
return MINIMUM_ENABLED_CODEQL_VERSION;
}
const maxCliVersion = enabledFeatureFlagCliVersions.reduce(
(maxVersion, currentVersion) =>
currentVersion > maxVersion ? currentVersion : maxVersion,
enabledFeatureFlagCliVersions[0]
);
this.logger.debug(
`Derived default CLI version of ${maxCliVersion} from feature flags.`
);
return maxCliVersion;
}
async getValue(feature: Feature): Promise<boolean> {
const response = await this.getAllFeatures();
if (response === undefined) {
@@ -246,7 +342,12 @@ class GitHubFeatureFlags implements FeatureEnablement {
repo: this.repositoryNwo.repo,
}
);
return response.data;
const remoteFlags = response.data;
this.logger.debug(
"Loaded the following default values for the feature flags from the Code Scanning API: " +
`${JSON.stringify(remoteFlags)}`
);
return remoteFlags;
} catch (e) {
if (util.isHTTPError(e) && e.status === 403) {
this.logger.warning(
@@ -255,6 +356,7 @@ class GitHubFeatureFlags implements FeatureEnablement {
"This could be because the Action is running on a pull request from a fork. If not, " +
`please ensure the Action has the 'security-events: write' permission. Details: ${e}`
);
return {};
} else {
// Some features, such as `ml_powered_queries_enabled` affect the produced alerts.
// Considering these features disabled in the event of a transient error could

4
src/init-action.ts
View File

@@ -182,6 +182,9 @@ async function run() {
return;
}
const defaultCliVersion = await features.getDefaultCliVersion(
gitHubVersion.type
);
const initCodeQLResult = await initCodeQL(
getOptionalInput("tools"),
apiDetails,
@@ -194,6 +197,7 @@ async function run() {
repositoryNwo,
logger
),
defaultCliVersion,
logger
);
codeql = initCodeQLResult.codeql;

4
src/init.ts
View File

@@ -8,7 +8,7 @@ import * as analysisPaths from "./analysis-paths";
import { GitHubApiCombinedDetails, GitHubApiDetails } from "./api-client";
import { CodeQL, CODEQL_VERSION_NEW_TRACING, setupCodeQL } from "./codeql";
import * as configUtils from "./config-utils";
import { FeatureEnablement } from "./feature-flags";
import { CodeQLDefaultVersionInfo, FeatureEnablement } from "./feature-flags";
import { Logger } from "./logging";
import { RepositoryNwo } from "./repository";
import { TracerConfig, getCombinedTracerConfig } from "./tracer-config";
@@ -21,6 +21,7 @@ export async function initCodeQL(
tempDir: string,
variant: util.GitHubVariant,
bypassToolcache: boolean,
defaultCliVersion: CodeQLDefaultVersionInfo,
logger: Logger
): Promise<{ codeql: CodeQL; toolsVersion: string }> {
logger.startGroup("Setup CodeQL tools");
@@ -30,6 +31,7 @@ export async function initCodeQL(
tempDir,
variant,
bypassToolcache,
defaultCliVersion,
logger,
true
);

125
src/setup-codeql.test.ts Normal file
View File

@@ -0,0 +1,125 @@
import * as path from "path";
import test from "ava";
import * as sinon from "sinon";
import * as actionsUtil from "./actions-util";
import * as api from "./api-client";
import { getRunnerLogger } from "./logging";
import * as setupCodeql from "./setup-codeql";
import { setupTests } from "./testing-utils";
import { initializeEnvironment } from "./util";
setupTests(test);
test.beforeEach(() => {
initializeEnvironment("1.2.3");
});
test("parse codeql bundle url version", (t) => {
t.deepEqual(
setupCodeql.getCodeQLURLVersion(
"https://github.com/.../codeql-bundle-20200601/..."
),
"20200601"
);
});
test("convert to semver", (t) => {
const tests = {
"20200601": "0.0.0-20200601",
"20200601.0": "0.0.0-20200601.0",
"20200601.0.0": "20200601.0.0",
"1.2.3": "1.2.3",
"1.2.3-alpha": "1.2.3-alpha",
"1.2.3-beta.1": "1.2.3-beta.1",
};
for (const [version, expectedVersion] of Object.entries(tests)) {
try {
const parsedVersion = setupCodeql.convertToSemVer(
version,
getRunnerLogger(true)
);
t.deepEqual(parsedVersion, expectedVersion);
} catch (e) {
t.fail(e instanceof Error ? e.message : String(e));
}
}
});
test("getCodeQLActionRepository", (t) => {
const logger = getRunnerLogger(true);
initializeEnvironment("1.2.3");
// isRunningLocalAction() === true
delete process.env["GITHUB_ACTION_REPOSITORY"];
process.env["RUNNER_TEMP"] = path.dirname(__dirname);
const repoLocalRunner = setupCodeql.getCodeQLActionRepository(logger);
t.deepEqual(repoLocalRunner, "github/codeql-action");
// isRunningLocalAction() === false
sinon.stub(actionsUtil, "isRunningLocalAction").returns(false);
process.env["GITHUB_ACTION_REPOSITORY"] = "xxx/yyy";
const repoEnv = setupCodeql.getCodeQLActionRepository(logger);
t.deepEqual(repoEnv, "xxx/yyy");
});
test("findCodeQLBundleTagDotcomOnly() matches GitHub Release with marker file", async (t) => {
// Look for GitHub Releases in github/codeql-action
sinon.stub(actionsUtil, "isRunningLocalAction").resolves(true);
sinon.stub(api, "getApiClient").value(() => ({
repos: {
listReleases: sinon.stub().resolves(undefined),
},
paginate: sinon.stub().resolves([
{
assets: [
{
name: "cli-version-2.12.0.txt",
},
],
tag_name: "codeql-bundle-20230106",
},
]),
}));
t.is(
await setupCodeql.findCodeQLBundleTagDotcomOnly(
"2.12.0",
getRunnerLogger(true)
),
"codeql-bundle-20230106"
);
});
test("findCodeQLBundleTagDotcomOnly() errors if no GitHub Release matches marker file", async (t) => {
// Look for GitHub Releases in github/codeql-action
sinon.stub(actionsUtil, "isRunningLocalAction").resolves(true);
sinon.stub(api, "getApiClient").value(() => ({
repos: {
listReleases: sinon.stub().resolves(undefined),
},
paginate: sinon.stub().resolves([
{
assets: [
{
name: "cli-version-2.12.0.txt",
},
],
tag_name: "codeql-bundle-20230106",
},
]),
}));
await t.throwsAsync(
async () =>
await setupCodeql.findCodeQLBundleTagDotcomOnly(
"2.12.1",
getRunnerLogger(true)
),
{
message:
"Failed to find a release of the CodeQL tools that contains CodeQL CLI 2.12.1.",
}
);
});

654
src/setup-codeql.ts Normal file
View File

@@ -0,0 +1,654 @@
import * as fs from "fs";
import { OutgoingHttpHeaders } from "http";
import * as path from "path";
import * as toolcache from "@actions/tool-cache";
import { default as deepEqual } from "fast-deep-equal";
import * as semver from "semver";
import { v4 as uuidV4 } from "uuid";
import { isRunningLocalAction } from "./actions-util";
import * as api from "./api-client";
// Note: defaults.json is referenced from the CodeQL Action sync tool and the Actions runner image
// creation scripts. Ensure that any changes to the format of this file are compatible with both of
// these dependents.
import * as defaults from "./defaults.json";
import { CodeQLDefaultVersionInfo } from "./feature-flags";
import { Logger } from "./logging";
import * as util from "./util";
import { isGoodVersion } from "./util";
export const CODEQL_DEFAULT_ACTION_REPOSITORY = "github/codeql-action";
function getCodeQLBundleName(): string {
let platform: string;
if (process.platform === "win32") {
platform = "win64";
} else if (process.platform === "linux") {
platform = "linux64";
} else if (process.platform === "darwin") {
platform = "osx64";
} else {
return "codeql-bundle.tar.gz";
}
return `codeql-bundle-${platform}.tar.gz`;
}
export function getCodeQLActionRepository(logger: Logger): string {
if (isRunningLocalAction()) {
// This handles the case where the Action does not come from an Action repository,
// e.g. our integration tests which use the Action code from the current checkout.
// In these cases, the GITHUB_ACTION_REPOSITORY environment variable is not set.
logger.info(
"The CodeQL Action is checked out locally. Using the default CodeQL Action repository."
);
return CODEQL_DEFAULT_ACTION_REPOSITORY;
}
return util.getRequiredEnvParam("GITHUB_ACTION_REPOSITORY");
}
/**
* Gets the tag name and, if known, the CodeQL CLI version for each CodeQL bundle release.
*
* CodeQL bundles are currently tagged in the form `codeql-bundle-yyyymmdd`, so it is not possible
* to directly find the CodeQL bundle release for a particular CLI version or find the CodeQL CLI
* version for a particular CodeQL bundle.
*
* To get around this, we add a `cli-version-x.y.z.txt` asset to each bundle release that specifies
* the CLI version for that bundle release. We can then use the GitHub Releases for the CodeQL
* Action as a source of truth.
*
* In the medium term, we should migrate to a tagging scheme that allows us to directly find the
* CodeQL bundle release for a particular CLI version, for example `codeql-bundle-vx.y.z`.
*/
async function getCodeQLBundleReleasesDotcomOnly(
logger: Logger
): Promise<Array<{ cliVersion?: string; tagName: string }>> {
logger.debug(
`Fetching CodeQL CLI version and CodeQL bundle tag name information for releases of the CodeQL tools.`
);
const apiClient = api.getApiClient();
const codeQLActionRepository = getCodeQLActionRepository(logger);
const releases = await apiClient.paginate(apiClient.repos.listReleases, {
owner: codeQLActionRepository.split("/")[0],
repo: codeQLActionRepository.split("/")[1],
});
logger.debug(`Found ${releases.length} releases.`);
return releases.map((release) => ({
cliVersion: tryGetCodeQLCliVersionForRelease(release, logger),
tagName: release.tag_name,
}));
}
function tryGetCodeQLCliVersionForRelease(
release,
logger: Logger
): string | undefined {
const cliVersionsFromMarkerFiles = release.assets
.map((asset) => asset.name.match(/cli-version-(.*)\.txt/)?.[1])
.filter((v) => v)
.map((v) => v as string);
if (cliVersionsFromMarkerFiles.length > 1) {
logger.warning(
`Ignoring release ${release.tag_name} with multiple CLI version marker files.`
);
return undefined;
} else if (cliVersionsFromMarkerFiles.length === 0) {
logger.debug(
`Failed to find the CodeQL CLI version for release ${release.tag_name}.`
);
return undefined;
}
return cliVersionsFromMarkerFiles[0];
}
export async function findCodeQLBundleTagDotcomOnly(
cliVersion: string,
logger: Logger
): Promise<string> {
const filtered = (await getCodeQLBundleReleasesDotcomOnly(logger)).filter(
(release) => release.cliVersion === cliVersion
);
if (filtered.length === 0) {
throw new Error(
`Failed to find a release of the CodeQL tools that contains CodeQL CLI ${cliVersion}.`
);
} else if (filtered.length > 1) {
throw new Error(
`Found multiple releases of the CodeQL tools that contain CodeQL CLI ${cliVersion}. ` +
`Only one such release should exist.`
);
}
return filtered[0].tagName;
}
export async function tryFindCliVersionDotcomOnly(
tagName: string,
logger: Logger
): Promise<string | undefined> {
try {
logger.debug(
`Fetching the GitHub Release for the CodeQL bundle tagged ${tagName}.`
);
const apiClient = api.getApiClient();
const codeQLActionRepository = getCodeQLActionRepository(logger);
const release = await apiClient.repos.getReleaseByTag({
owner: codeQLActionRepository.split("/")[0],
repo: codeQLActionRepository.split("/")[1],
tag: tagName,
});
return tryGetCodeQLCliVersionForRelease(release.data, logger);
} catch (e) {
logger.debug(
`Failed to find the CLI version for the CodeQL bundle tagged ${tagName}. ${
e instanceof Error ? e.message : e
}`
);
return undefined;
}
}
async function getCodeQLBundleDownloadURL(
tagName: string,
apiDetails: api.GitHubApiDetails,
variant: util.GitHubVariant,
logger: Logger
): Promise<string> {
const codeQLActionRepository = getCodeQLActionRepository(logger);
const potentialDownloadSources = [
// This GitHub instance, and this Action.
[apiDetails.url, codeQLActionRepository],
// This GitHub instance, and the canonical Action.
[apiDetails.url, CODEQL_DEFAULT_ACTION_REPOSITORY],
// GitHub.com, and the canonical Action.
[util.GITHUB_DOTCOM_URL, CODEQL_DEFAULT_ACTION_REPOSITORY],
];
// We now filter out any duplicates.
// Duplicates will happen either because the GitHub instance is GitHub.com, or because the Action is not a fork.
const uniqueDownloadSources = potentialDownloadSources.filter(
(source, index, self) => {
return !self.slice(0, index).some((other) => deepEqual(source, other));
}
);
const codeQLBundleName = getCodeQLBundleName();
if (variant === util.GitHubVariant.GHAE) {
try {
const release = await api
.getApiClient()
.request("GET /enterprise/code-scanning/codeql-bundle/find/{tag}", {
tag: tagName,
});
const assetID = release.data.assets[codeQLBundleName];
if (assetID !== undefined) {
const download = await api
.getApiClient()
.request(
"GET /enterprise/code-scanning/codeql-bundle/download/{asset_id}",
{ asset_id: assetID }
);
const downloadURL = download.data.url;
logger.info(
`Found CodeQL bundle at GitHub AE endpoint with URL ${downloadURL}.`
);
return downloadURL;
} else {
logger.info(
`Attempted to fetch bundle from GitHub AE endpoint but the bundle ${codeQLBundleName} was not found in the assets ${JSON.stringify(
release.data.assets
)}.`
);
}
} catch (e) {
logger.info(
`Attempted to fetch bundle from GitHub AE endpoint but got error ${e}.`
);
}
}
for (const downloadSource of uniqueDownloadSources) {
const [apiURL, repository] = downloadSource;
// If we've reached the final case, short-circuit the API check since we know the bundle exists and is public.
if (
apiURL === util.GITHUB_DOTCOM_URL &&
repository === CODEQL_DEFAULT_ACTION_REPOSITORY
) {
break;
}
const [repositoryOwner, repositoryName] = repository.split("/");
try {
const release = await api.getApiClient().repos.getReleaseByTag({
owner: repositoryOwner,
repo: repositoryName,
tag: tagName,
});
for (const asset of release.data.assets) {
if (asset.name === codeQLBundleName) {
logger.info(
`Found CodeQL bundle in ${downloadSource[1]} on ${downloadSource[0]} with URL ${asset.url}.`
);
return asset.url;
}
}
} catch (e) {
logger.info(
`Looked for CodeQL bundle in ${downloadSource[1]} on ${downloadSource[0]} but got error ${e}.`
);
}
}
return `https://github.com/${CODEQL_DEFAULT_ACTION_REPOSITORY}/releases/download/${tagName}/${codeQLBundleName}`;
}
export function getBundleVersionFromUrl(url: string): string {
const match = url.match(/\/codeql-bundle-(.*)\//);
if (match === null || match.length < 2) {
throw new Error(
`Malformed tools url: ${url}. Bundle version could not be inferred`
);
}
return match[1];
}
export function convertToSemVer(version: string, logger: Logger): string {
if (!semver.valid(version)) {
logger.debug(
`Bundle version ${version} is not in SemVer format. Will treat it as pre-release 0.0.0-${version}.`
);
version = `0.0.0-${version}`;
}
const s = semver.clean(version);
if (!s) {
throw new Error(`Bundle version ${version} is not in SemVer format.`);
}
return s;
}
type CodeQLToolsSource =
| {
codeqlTarPath: string;
sourceType: "local";
/** Human-readable description of the source of the tools for telemetry purposes. */
toolsVersion: "local";
}
| {
codeqlFolder: string;
sourceType: "toolcache";
/** Human-readable description of the source of the tools for telemetry purposes. */
toolsVersion: string;
}
| {
/** CLI version of the tools, if known. */
cliVersion?: string;
codeqlURL: string;
sourceType: "download";
/** Human-readable description of the source of the tools for telemetry purposes. */
toolsVersion: string;
};
async function getOrFindBundleTagName(
version: CodeQLDefaultVersionInfo,
logger: Logger
): Promise<string> {
if (version.variant === util.GitHubVariant.DOTCOM) {
return await findCodeQLBundleTagDotcomOnly(version.cliVersion, logger);
} else {
return version.tagName;
}
}
/**
* Look for a version of the CodeQL tools in the cache which could override the requested CLI version.
*/
async function findOverridingToolsInCache(
requestedCliVersion: string,
logger: Logger
): Promise<CodeQLToolsSource | undefined> {
const candidates = toolcache
.findAllVersions("CodeQL")
.filter(isGoodVersion)
.map((version) => ({
folder: toolcache.find("CodeQL", version),
version,
}))
.filter(({ folder }) => fs.existsSync(path.join(folder, "pinned-version")));
if (candidates.length === 1) {
const candidate = candidates[0];
logger.debug(
`CodeQL tools version ${candidate.version} in toolcache overriding version ${requestedCliVersion}.`
);
return {
codeqlFolder: candidate.folder,
sourceType: "toolcache",
toolsVersion: candidate.version,
};
} else if (candidates.length === 0) {
logger.debug(
"Did not find any candidate pinned versions of the CodeQL tools in the toolcache."
);
} else {
logger.debug(
"Could not use CodeQL tools from the toolcache since more than one candidate pinned " +
"version was found in the toolcache."
);
}
return undefined;
}
export async function getCodeQLSource(
toolsInput: string | undefined,
bypassToolcache: boolean,
defaultCliVersion: CodeQLDefaultVersionInfo,
apiDetails: api.GitHubApiDetails,
variant: util.GitHubVariant,
logger: Logger
): Promise<CodeQLToolsSource> {
if (toolsInput && toolsInput !== "latest" && !toolsInput.startsWith("http")) {
return {
codeqlTarPath: toolsInput,
sourceType: "local",
toolsVersion: "local",
};
}
const forceLatestReason =
// We use the special value of 'latest' to prioritize the version in the
// defaults over any pinned cached version.
toolsInput === "latest"
? '"tools: latest" was requested'
: // If the user hasn't requested a particular CodeQL version, then bypass
// the toolcache when the appropriate feature is enabled. This
// allows us to quickly rollback a broken bundle that has made its way
// into the toolcache.
toolsInput === undefined && bypassToolcache
? "a specific version of the CodeQL tools was not requested and the bypass toolcache feature is enabled"
: undefined;
const forceLatest = forceLatestReason !== undefined;
if (forceLatest) {
logger.debug(
`Forcing the latest version of the CodeQL tools since ${forceLatestReason}.`
);
}
/**
* The requested version is:
*
* 1. The one in `defaults.json`, if forceLatest is true.
* 2. The version specified by the tools input URL, if one was provided.
* 3. The default CLI version, otherwise.
* We include a `variant` property to let us verify using the type system that
* `tagName` is only undefined when the variant is Dotcom. This lets us ensure
* that we can always compute `tagName`, either by using the existing tag name
* on enterprise instances, or calling `findCodeQLBundleTagDotcomOnly` on
* Dotcom.
*/
const requestedVersion = forceLatest
? // case 1
{
cliVersion: defaults.cliVersion,
syntheticCliVersion: defaults.cliVersion,
tagName: defaults.bundleVersion,
variant,
}
: toolsInput !== undefined
? // case 2
{
syntheticCliVersion: convertToSemVer(
getBundleVersionFromUrl(toolsInput),
logger
),
tagName: `codeql-bundle-${getBundleVersionFromUrl(toolsInput)}`,
url: toolsInput,
variant,
}
: // case 3
{
...defaultCliVersion,
syntheticCliVersion: defaultCliVersion.cliVersion,
};
// If we find the specified version, we always use that.
let codeqlFolder = toolcache.find(
"CodeQL",
requestedVersion.syntheticCliVersion
);
let tagName: string | undefined = requestedVersion["tagName"];
if (!codeqlFolder) {
logger.debug(
"Didn't find a version of the CodeQL tools in the toolcache with a version number " +
`exactly matching ${requestedVersion.syntheticCliVersion}.`
);
if (requestedVersion.cliVersion) {
const allVersions = toolcache.findAllVersions("CodeQL");
logger.debug(
`Found the following versions of the CodeQL tools in the toolcache: ${JSON.stringify(
allVersions
)}.`
);
// If there is exactly one version of the CodeQL tools in the toolcache, and that version is
// the form `x.y.z-<tagName>`, then use it.
const candidateVersions = allVersions.filter((version) =>
version.startsWith(`${requestedVersion.cliVersion}-`)
);
if (candidateVersions.length === 1) {
logger.debug("Exactly one candidate version found, using that.");
codeqlFolder = toolcache.find("CodeQL", candidateVersions[0]);
} else {
logger.debug(
"Did not find exactly one version of the CodeQL tools starting with the requested version."
);
}
}
}
if (!codeqlFolder && requestedVersion.cliVersion) {
// Fall back to accepting a `0.0.0-<tagName>` version if we didn't find the
// `x.y.z` version. This is to support old versions of the toolcache.
//
// If we are on Dotcom, we will make an HTTP request to the Releases API here
// to find the tag name for the requested version.
tagName =
tagName || (await getOrFindBundleTagName(requestedVersion, logger));
const fallbackVersion = convertToSemVer(tagName, logger);
logger.debug(
`Computed a fallback toolcache version number of ${fallbackVersion} for CodeQL tools version ` +
`${requestedVersion.cliVersion}.`
);
codeqlFolder = toolcache.find("CodeQL", fallbackVersion);
}
if (codeqlFolder) {
return {
codeqlFolder,
sourceType: "toolcache",
toolsVersion: requestedVersion.syntheticCliVersion,
};
}
logger.debug(
`Did not find CodeQL tools version ${requestedVersion.syntheticCliVersion} in the toolcache.`
);
// If we don't find the requested version on Enterprise, we may allow a
// different version to save download time if the version hasn't been
// specified explicitly (in which case we always honor it).
if (variant !== util.GitHubVariant.DOTCOM && !forceLatest && !toolsInput) {
const result = await findOverridingToolsInCache(
requestedVersion.syntheticCliVersion,
logger
);
if (result !== undefined) {
return result;
}
}
return {
cliVersion: requestedVersion.cliVersion || undefined,
codeqlURL:
requestedVersion["url"] ||
(await getCodeQLBundleDownloadURL(
tagName ||
// The check on `requestedVersion.tagName` is redundant but lets us
// use the property that if we don't know `requestedVersion.tagName`,
// then we must know `requestedVersion.cliVersion`. This property is
// required by the type of `getOrFindBundleTagName`.
(requestedVersion.tagName !== undefined
? requestedVersion.tagName
: await getOrFindBundleTagName(requestedVersion, logger)),
apiDetails,
variant,
logger
)),
sourceType: "download",
toolsVersion: requestedVersion.syntheticCliVersion,
};
}
export async function downloadCodeQL(
codeqlURL: string,
maybeCliVersion: string | undefined,
apiDetails: api.GitHubApiDetails,
variant: util.GitHubVariant,
tempDir: string,
logger: Logger
): Promise<{ toolsVersion: string; codeqlFolder: string }> {
const parsedCodeQLURL = new URL(codeqlURL);
const searchParams = new URLSearchParams(parsedCodeQLURL.search);
const headers: OutgoingHttpHeaders = {
accept: "application/octet-stream",
};
// We only want to provide an authorization header if we are downloading
// from the same GitHub instance the Action is running on.
// This avoids leaking Enterprise tokens to dotcom.
// We also don't want to send an authorization header if there's already a token provided in the URL.
if (searchParams.has("token")) {
logger.debug("CodeQL tools URL contains an authorization token.");
} else if (codeqlURL.startsWith(`${apiDetails.url}/`)) {
logger.debug("Providing an authorization token to download CodeQL tools.");
headers.authorization = `token ${apiDetails.auth}`;
} else {
logger.debug("Downloading CodeQL tools without an authorization token.");
}
logger.info(
`Downloading CodeQL tools from ${codeqlURL}. This may take a while.`
);
const dest = path.join(tempDir, uuidV4() as string);
const finalHeaders = Object.assign(
{ "User-Agent": "CodeQL Action" },
headers
);
const codeqlPath = await toolcache.downloadTool(
codeqlURL,
dest,
undefined,
finalHeaders
);
logger.debug(`CodeQL bundle download to ${codeqlPath} complete.`);
const codeqlExtracted = await toolcache.extractTar(codeqlPath);
const bundleVersion = getBundleVersionFromUrl(codeqlURL);
// Try to compute the CLI version for this bundle
const cliVersion: string | undefined =
maybeCliVersion ||
(variant === util.GitHubVariant.DOTCOM &&
(await tryFindCliVersionDotcomOnly(
`codeql-bundle-${bundleVersion}`,
logger
))) ||
undefined;
// Include both the CLI version and the bundle version in the toolcache version number. That way
// if the user requests the same URL again, we can get it from the cache without having to call
// any of the Releases API.
//
// Special case: If the CLI version is a pre-release, then cache the bundle as
// `0.0.0-<bundleVersion>` to avoid the bundle being interpreted as containing a stable CLI
// release.
const toolcacheVersion =
cliVersion && !cliVersion.includes("-")
? `${cliVersion}-${bundleVersion}`
: convertToSemVer(bundleVersion, logger);
return {
toolsVersion: cliVersion || toolcacheVersion,
codeqlFolder: await toolcache.cacheDir(
codeqlExtracted,
"CodeQL",
toolcacheVersion
),
};
}
export function getCodeQLURLVersion(url: string): string {
const match = url.match(/\/codeql-bundle-(.*)\//);
if (match === null || match.length < 2) {
throw new Error(
`Malformed tools url: ${url}. Version could not be inferred`
);
}
return match[1];
}
/**
* Obtains the CodeQL bundle, installs it in the toolcache if appropriate, and extracts it.
*
* @param toolsInput
* @param apiDetails
* @param tempDir
* @param variant
* @param bypassToolcache
* @param defaultCliVersion
* @param logger
* @param checkVersion Whether to check that CodeQL CLI meets the minimum
* version requirement. Must be set to true outside tests.
* @returns the path to the extracted bundle, and the version of the tools
*/
export async function setupCodeQLBundle(
toolsInput: string | undefined,
apiDetails: api.GitHubApiDetails,
tempDir: string,
variant: util.GitHubVariant,
bypassToolcache: boolean,
defaultCliVersion: CodeQLDefaultVersionInfo,
logger: Logger
): Promise<{ codeqlFolder: string; toolsVersion: string }> {
const source = await getCodeQLSource(
toolsInput,
bypassToolcache,
defaultCliVersion,
apiDetails,
variant,
logger
);
let codeqlFolder: string;
let toolsVersion = source.toolsVersion;
switch (source.sourceType) {
case "local":
codeqlFolder = await toolcache.extractTar(source.codeqlTarPath);
break;
case "toolcache":
codeqlFolder = source.codeqlFolder;
logger.debug(`CodeQL found in cache ${codeqlFolder}`);
break;
case "download": {
const result = await downloadCodeQL(
source.codeqlURL,
source.cliVersion,
apiDetails,
variant,
tempDir,
logger
);
toolsVersion = result.toolsVersion;
codeqlFolder = result.codeqlFolder;
break;
}
default:
util.assertNever(source);
}
return { codeqlFolder, toolsVersion };
}

3
src/testing-utils.ts
View File

@@ -204,6 +204,9 @@ export function mockCodeQLVersion(version) {
*/
export function createFeatures(enabledFeatures: Feature[]): FeatureEnablement {
return {
getDefaultCliVersion: async () => {
throw new Error("not implemented");
},
getValue: async (feature) => {
return enabledFeatures.includes(feature);
},