github/codeql-action
1
0
Fork 0
mirror of https://github.com/github/codeql-action.git synced 2026-01-06 22:50:17 +08:00
Code Issues Packages Projects Releases Wiki Activity

Cache explicitly requested bundles with their URL if possible

Browse Source
This commit is contained in:
Henry Mercer
2023-01-12 20:44:05 +00:00
parent c9b1be5115
commit c2e39e078f
9 changed files with 396 additions and 154 deletions
Download Patch File Download Diff File Expand all files Collapse all files

99
src/codeql.test.ts
View File

@@ -83,7 +83,7 @@ test.beforeEach(() => {
* @returns the download URL for the bundle. This can be passed to the tools parameter of
* `codeql.setupCodeQL`.
*/
async function mockDownloadApi({
function mockDownloadApi({
apiDetails = sampleApiDetails,
isPinned,
tagName,
@@ -91,7 +91,7 @@ async function mockDownloadApi({
apiDetails?: GitHubApiDetails;
isPinned?: boolean;
tagName: string;
}): Promise<string> {
}): string {
const platform =
process.platform === "win32"
? "win64"
@@ -130,7 +130,7 @@ async function installIntoToolcache({
tagName: string;
tmpDir: string;
}) {
const url = await mockDownloadApi({ apiDetails, isPinned, tagName });
const url = mockDownloadApi({ apiDetails, isPinned, tagName });
await codeql.setupCodeQL(
cliVersion !== undefined ? undefined : url,
apiDetails,
@@ -145,6 +145,41 @@ async function installIntoToolcache({
);
}
function mockReleaseApi({
apiDetails = sampleApiDetails,
assetNames,
tagName,
}: {
apiDetails?: GitHubApiDetails;
assetNames: string[];
tagName: string;
}): nock.Scope {
return nock(apiDetails.apiURL!)
.get(`/repos/github/codeql-action/releases/tags/${tagName}`)
.reply(200, {
assets: assetNames.map((name) => ({
name,
})),
tag_name: tagName,
});
}
function mockApiDetails(apiDetails: GitHubApiDetails) {
// This is a workaround to mock `api.getApiDetails()` since it doesn't seem to be possible to
// mock this directly. The difficulty is that `getApiDetails()` is called locally in
// `api-client.ts`, but `sinon.stub(api, "getApiDetails")` only affects calls to
// `getApiDetails()` via an imported `api` module.
sinon
.stub(actionsUtil, "getRequiredInput")
.withArgs("token")
.returns(apiDetails.auth);
const requiredEnvParamStub = sinon.stub(util, "getRequiredEnvParam");
requiredEnvParamStub.withArgs("GITHUB_SERVER_URL").returns(apiDetails.url);
requiredEnvParamStub
.withArgs("GITHUB_API_URL")
.returns(apiDetails.apiURL || "");
}
test("downloads and caches explicitly requested bundles that aren't in the toolcache", async (t) => {
await util.withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);
@@ -154,7 +189,7 @@ test("downloads and caches explicitly requested bundles that aren't in the toolc
for (let i = 0; i < versions.length; i++) {
const version = versions[i];
const url = await mockDownloadApi({
const url = mockDownloadApi({
tagName: `codeql-bundle-${version}`,
isPinned: false,
});
@@ -186,7 +221,7 @@ test("downloads an explicitly requested bundle even if a different version is ca
tmpDir,
});
const url = await mockDownloadApi({
const url = mockDownloadApi({
tagName: "codeql-bundle-20200610",
});
const result = await codeql.setupCodeQL(
@@ -204,6 +239,37 @@ test("downloads an explicitly requested bundle even if a different version is ca
});
});
test("tries to cache an explicitly requested bundle with its CLI version number", async (t) => {
await util.withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);
mockApiDetails(sampleApiDetails);
sinon.stub(actionsUtil, "isRunningLocalAction").returns(true);
const releaseApiMock = mockReleaseApi({
assetNames: ["cli-version-2.10.0.txt"],
tagName: "codeql-bundle-20200610",
});
const url = mockDownloadApi({
tagName: "codeql-bundle-20200610",
});
const result = await codeql.setupCodeQL(
url,
sampleApiDetails,
tmpDir,
util.GitHubVariant.DOTCOM,
false,
SAMPLE_DEFAULT_CLI_VERSION,
getRunnerLogger(true),
false
);
t.assert(releaseApiMock.isDone(), "Releases API should have been called");
t.assert(toolcache.find("CodeQL", "2.10.0"));
t.deepEqual(result.toolsVersion, "0.0.0-20200610");
});
});
for (const { isCached, tagName, toolcacheCliVersion } of [
{
isCached: true,
@@ -237,7 +303,7 @@ for (const { isCached, tagName, toolcacheCliVersion } of [
tmpDir,
});
} else {
await mockDownloadApi({
mockDownloadApi({
tagName,
});
sinon.stub(api, "getApiClient").value(() => ({
@@ -314,7 +380,7 @@ for (const variant of [util.GitHubVariant.GHAE, util.GitHubVariant.GHES]) {
tmpDir,
});
await mockDownloadApi({
mockDownloadApi({
tagName: defaults.bundleVersion,
});
const result = await codeql.setupCodeQL(
@@ -349,7 +415,7 @@ test('downloads bundle if "latest" tools specified but not cached', async (t) =>
tmpDir,
});
await mockDownloadApi({
mockDownloadApi({
tagName: defaults.bundleVersion,
});
const result = await codeql.setupCodeQL(
@@ -408,22 +474,7 @@ test("download codeql bundle from github ae endpoint", async (t) => {
path.join(__dirname, `/../src/testdata/codeql-bundle-pinned.tar.gz`)
);
// This is a workaround to mock `api.getApiDetails()` since it doesn't seem to be possible to
// mock this directly. The difficulty is that `getApiDetails()` is called locally in
// `api-client.ts`, but `sinon.stub(api, "getApiDetails")` only affects calls to
// `getApiDetails()` via an imported `api` module.
sinon
.stub(actionsUtil, "getRequiredInput")
.withArgs("token")
.returns(sampleGHAEApiDetails.auth);
const requiredEnvParamStub = sinon.stub(util, "getRequiredEnvParam");
requiredEnvParamStub
.withArgs("GITHUB_SERVER_URL")
.returns(sampleGHAEApiDetails.url);
requiredEnvParamStub
.withArgs("GITHUB_API_URL")
.returns(sampleGHAEApiDetails.apiURL);
mockApiDetails(sampleGHAEApiDetails);
sinon.stub(actionsUtil, "isRunningLocalAction").returns(false);
process.env["GITHUB_ACTION_REPOSITORY"] = "github/codeql-action";

3
src/setup-codeql.test.ts
View File

@@ -118,7 +118,8 @@ test("findCodeQLBundleTagDotcomOnly() errors if no GitHub Release matches marker
getRunnerLogger(true)
),
{
message: "Failed to find a CodeQL bundle release for CLI version 2.12.1.",
message:
"Failed to find a release of the CodeQL tools that contains CodeQL CLI 2.12.1.",
}
);
});

213
src/setup-codeql.ts
View File

@@ -49,21 +49,24 @@ export function getCodeQLActionRepository(logger: Logger): string {
}
/**
* CodeQL bundles are currently tagged in the form `codeql-bundle-yyyymmdd`, so it is not possible
* to directly find the CodeQL bundle release for a particular CLI version.
* Gets the tag name and, if known, the CodeQL CLI version for each CodeQL bundle release.
*
* To get around this, we add a `codeql-version-x.y.z.txt` asset to each bundle release that
* specifies the CLI version for that bundle release.
* CodeQL bundles are currently tagged in the form `codeql-bundle-yyyymmdd`, so it is not possible
* to directly find the CodeQL bundle release for a particular CLI version or find the CodeQL CLI
* version for a particular CodeQL bundle.
*
* To get around this, we add a `cli-version-x.y.z.txt` asset to each bundle release that specifies
* the CLI version for that bundle release. We can then use the GitHub Releases for the CodeQL
* Action as a source of truth.
*
* In the medium term, we should migrate to a tagging scheme that allows us to directly find the
* CodeQL bundle release for a particular CLI version, for example `codeql-bundle-vx.y.z`.
*/
export async function findCodeQLBundleTagDotcomOnly(
cliVersion: string,
async function getCodeQLBundleReleasesDotcomOnly(
logger: Logger
): Promise<string> {
): Promise<Array<{ cliVersion?: string; tagName: string }>> {
logger.debug(
`Trying to find the CodeQL bundle release for CLI version ${cliVersion}.`
`Fetching CodeQL CLI version and CodeQL bundle tag name information for releases of the CodeQL tools.`
);
const apiClient = api.getApiClient();
const codeQLActionRepository = getCodeQLActionRepository(logger);
@@ -73,31 +76,90 @@ export async function findCodeQLBundleTagDotcomOnly(
});
logger.debug(`Found ${releases.length} releases.`);
for (const release of releases) {
return releases.flatMap((release) => {
const cliVersionFileVersions = release.assets
.map((asset) => asset.name.match(/cli-version-(.*)\.txt/)?.[1])
.filter((v) => v)
.map((v) => v as string);
if (cliVersionFileVersions.length === 0) {
logger.debug(
`Ignoring release ${release.tag_name} with no CLI version marker file.`
);
continue;
}
if (cliVersionFileVersions.length > 1) {
logger.warning(
`Ignoring release ${release.tag_name} with multiple CLI version marker files.`
);
continue;
}
if (cliVersionFileVersions[0] === cliVersion) {
return release.tag_name;
return [];
}
return [
{ cliVersion: cliVersionFileVersions[0], tagName: release.tag_name },
];
});
}
async function tryGetCodeQLCliVersionForRelease(
release,
logger: Logger
): Promise<string | undefined> {
const cliVersionsFromMarkerFiles = release.assets
.map((asset) => asset.name.match(/cli-version-(.*)\.txt/)?.[1])
.filter((v) => v)
.map((v) => v as string);
if (cliVersionsFromMarkerFiles.length > 1) {
logger.warning(
`Ignoring release ${release.tag_name} with multiple CLI version marker files.`
);
return undefined;
} else if (cliVersionsFromMarkerFiles.length === 0) {
logger.debug(
`Failed to find the CodeQL CLI version for release ${release.tag_name}.`
);
return undefined;
}
throw new Error(
`Failed to find a CodeQL bundle release for CLI version ${cliVersion}.`
return cliVersionsFromMarkerFiles[0];
}
export async function findCodeQLBundleTagDotcomOnly(
cliVersion: string,
logger: Logger
): Promise<string> {
const filtered = (await getCodeQLBundleReleasesDotcomOnly(logger)).filter(
(release) => release.cliVersion === cliVersion
);
if (filtered.length === 0) {
throw new Error(
`Failed to find a release of the CodeQL tools that contains CodeQL CLI ${cliVersion}.`
);
} else if (filtered.length > 1) {
throw new Error(
`Found multiple releases of the CodeQL tools that contain CodeQL CLI ${cliVersion}. ` +
`Only one such release should exist.`
);
}
return filtered[0].tagName;
}
export async function tryFindCliVersionDotcomOnly(
tagName: string,
logger: Logger
): Promise<string | undefined> {
try {
logger.debug(
`Fetching the GitHub Release for the CodeQL bundle tagged ${tagName}.`
);
const apiClient = api.getApiClient();
const codeQLActionRepository = getCodeQLActionRepository(logger);
const release = await apiClient.repos.getReleaseByTag({
owner: codeQLActionRepository.split("/")[0],
repo: codeQLActionRepository.split("/")[1],
tag: tagName,
});
return tryGetCodeQLCliVersionForRelease(release.data, logger);
} catch (e) {
logger.error(
`Failed to find the CLI version for the CodeQL bundle tagged ${tagName}. Error: ${
e instanceof Error ? e.message : e
}`
);
return undefined;
}
}
async function getCodeQLBundleDownloadURL(
@@ -189,11 +251,11 @@ async function getCodeQLBundleDownloadURL(
return `https://github.com/${CODEQL_DEFAULT_ACTION_REPOSITORY}/releases/download/${tagName}/${codeQLBundleName}`;
}
export function getBundleTagNameFromUrl(url: string): string {
export function getBundleVersionFromUrl(url: string): string {
const match = url.match(/\/codeql-bundle-(.*)\//);
if (match === null || match.length < 2) {
throw new Error(
`Malformed tools url: ${url}. Tag name could not be inferred`
`Malformed tools url: ${url}. Bundle version could not be inferred`
);
}
return match[1];
@@ -216,16 +278,24 @@ export function convertToSemVer(version: string, logger: Logger): string {
}
type CodeQLToolsSource =
| { codeqlTarPath: string; sourceType: "local"; toolsVersion: "local" }
| {
codeqlTarPath: string;
sourceType: "local";
/** Human-readable description of the source of the tools for telemetry purposes. */
toolsVersion: "local";
}
| {
codeqlFolder: string;
sourceType: "toolcache";
/** Human-readable description of the source of the tools for telemetry purposes. */
toolsVersion: string;
}
| {
/** CLI version of the tools, if known. */
cliVersion?: string;
codeqlURL: string;
semanticVersion: string;
sourceType: "download";
/** Human-readable description of the source of the tools for telemetry purposes. */
toolsVersion: string;
};
@@ -331,54 +401,63 @@ export async function getCodeQLSource(
? // case 1
{
cliVersion: defaults.cliVersion,
syntheticCliVersion: defaults.cliVersion,
tagName: defaults.bundleVersion,
variant,
}
: toolsInput !== undefined
? // case 2
{
cliVersion: convertToSemVer(
getBundleTagNameFromUrl(toolsInput),
syntheticCliVersion: convertToSemVer(
getBundleVersionFromUrl(toolsInput),
logger
),
tagName: getBundleTagNameFromUrl(toolsInput),
tagName: `codeql-bundle-${getBundleVersionFromUrl(toolsInput)}`,
url: toolsInput,
variant,
}
: // case 3
defaultCliVersion;
{
...defaultCliVersion,
syntheticCliVersion: defaultCliVersion.cliVersion,
};
// If we find the specified version, we always use that.
let codeqlFolder = toolcache.find("CodeQL", requestedVersion.cliVersion);
let codeqlFolder = toolcache.find(
"CodeQL",
requestedVersion.syntheticCliVersion
);
let tagName: string | undefined = requestedVersion["tagName"];
if (!codeqlFolder) {
logger.debug(
"Didn't find a version of the CodeQL tools in the toolcache with a version number " +
`exactly matching ${requestedVersion.cliVersion}.`
`exactly matching ${requestedVersion.syntheticCliVersion}.`
);
const allVersions = toolcache.findAllVersions("CodeQL");
logger.debug(
`Found the following versions of the CodeQL tools in the toolcache: ${JSON.stringify(
allVersions
)}.`
);
// If there is exactly one version of the CodeQL tools in the toolcache, and that version is
// the form `x.y.z-<tagName>`, then use it.
const candidateVersions = allVersions.filter((version) =>
version.startsWith(`${requestedVersion.cliVersion}-`)
);
if (candidateVersions.length === 1) {
logger.debug("Exactly one candidate version found, using that.");
codeqlFolder = toolcache.find("CodeQL", candidateVersions[0]);
} else {
if (requestedVersion.cliVersion) {
const allVersions = toolcache.findAllVersions("CodeQL");
logger.debug(
"Did not find exactly one version of the CodeQL tools starting with the requested version."
`Found the following versions of the CodeQL tools in the toolcache: ${JSON.stringify(
allVersions
)}.`
);
// If there is exactly one version of the CodeQL tools in the toolcache, and that version is
// the form `x.y.z-<tagName>`, then use it.
const candidateVersions = allVersions.filter((version) =>
version.startsWith(`${requestedVersion.cliVersion}-`)
);
if (candidateVersions.length === 1) {
logger.debug("Exactly one candidate version found, using that.");
codeqlFolder = toolcache.find("CodeQL", candidateVersions[0]);
} else {
logger.debug(
"Did not find exactly one version of the CodeQL tools starting with the requested version."
);
}
}
}
if (!codeqlFolder && !requestedVersion.cliVersion.startsWith("0.0.0")) {
if (!codeqlFolder && requestedVersion.cliVersion) {
// Fall back to accepting a `0.0.0-<tagName>` version if we didn't find the
// `x.y.z` version. This is to support old versions of the toolcache.
//
@@ -398,11 +477,11 @@ export async function getCodeQLSource(
return {
codeqlFolder,
sourceType: "toolcache",
toolsVersion: requestedVersion.cliVersion,
toolsVersion: requestedVersion.syntheticCliVersion,
};
}
logger.debug(
`Did not find CodeQL tools version ${requestedVersion.cliVersion} in the toolcache.`
`Did not find CodeQL tools version ${requestedVersion.syntheticCliVersion} in the toolcache.`
);
// If we don't find the requested version on Enterprise, we may allow a
@@ -410,7 +489,7 @@ export async function getCodeQLSource(
// specified explicitly (in which case we always honor it).
if (variant !== util.GitHubVariant.DOTCOM && !forceLatest && !toolsInput) {
const result = await findOverridingToolsInCache(
requestedVersion.cliVersion,
requestedVersion.syntheticCliVersion,
logger
);
if (result !== undefined) {
@@ -419,24 +498,32 @@ export async function getCodeQLSource(
}
return {
cliVersion: requestedVersion.cliVersion || undefined,
codeqlURL:
requestedVersion["url"] ||
(await getCodeQLBundleDownloadURL(
tagName || (await getOrFindBundleTagName(requestedVersion, logger)),
tagName ||
// The check on `requestedVersion.tagName` is redundant but lets us
// use the property that if we don't know `requestedVersion.tagName`,
// then we must know `requestedVersion.cliVersion`. This property is
// required by the type of `getOrFindBundleTagName`.
(requestedVersion.tagName !== undefined
? requestedVersion.tagName
: await getOrFindBundleTagName(requestedVersion, logger)),
apiDetails,
variant,
logger
)),
semanticVersion: requestedVersion.cliVersion,
sourceType: "download",
toolsVersion: requestedVersion.cliVersion,
toolsVersion: requestedVersion.syntheticCliVersion,
};
}
export async function downloadCodeQL(
codeqlURL: string,
semanticVersion: string,
cliVersion: string | undefined,
apiDetails: api.GitHubApiDetails,
variant: util.GitHubVariant,
tempDir: string,
logger: Logger
): Promise<string> {
@@ -475,7 +562,18 @@ export async function downloadCodeQL(
logger.debug(`CodeQL bundle download to ${codeqlPath} complete.`);
const codeqlExtracted = await toolcache.extractTar(codeqlPath);
return await toolcache.cacheDir(codeqlExtracted, "CodeQL", semanticVersion);
const bundleVersion = getBundleVersionFromUrl(codeqlURL);
// If we have a CLI version, use that. Otherwise, try to find the CLI version from the GitHub Releases
const toolcacheVersion =
cliVersion ||
(variant === util.GitHubVariant.DOTCOM &&
(await tryFindCliVersionDotcomOnly(
`codeql-bundle-${bundleVersion}`,
logger
))) ||
convertToSemVer(bundleVersion, logger);
return await toolcache.cacheDir(codeqlExtracted, "CodeQL", toolcacheVersion);
}
export function getCodeQLURLVersion(url: string): string {
@@ -532,8 +630,9 @@ export async function setupCodeQLBundle(
case "download":
codeqlFolder = await downloadCodeQL(
source.codeqlURL,
source.semanticVersion,
source.cliVersion,
apiDetails,
variant,
tempDir,
logger
);