github/codeql-action
1
0
Fork 0
mirror of https://github.com/github/codeql-action.git synced 2026-01-02 20:50:05 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1221 from github/aeisenberg/ghes-pack-download

Browse Source 
Add support for downloading packs from GHES
This commit is contained in:
Andrew Eisenberg
2022-09-08 10:02:41 -07:00
committed by GitHub
parent 21ae865bef 6085805a3a
commit 9ba4d500aa
26 changed files with 861 additions and 125 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/codeql.test.ts
View File

@@ -26,12 +26,14 @@ const sampleApiDetails = {
auth: "token",
url: "https://github.com",
apiURL: undefined,
registriesAuthTokens: undefined,
};
const sampleGHAEApiDetails = {
auth: "token",
url: "https://example.githubenterprise.com",
apiURL: undefined,
registriesAuthTokens: undefined,
};
let stubConfig: Config;

19
src/codeql.ts
View File

@@ -134,7 +134,10 @@ export interface CodeQL {
/**
* Run 'codeql pack download'.
*/
packDownload(packs: string[]): Promise<PackDownloadOutput>;
packDownload(
packs: string[],
qlconfigFile: string | undefined
): Promise<PackDownloadOutput>;
/**
* Run 'codeql database cleanup'.
@@ -252,6 +255,7 @@ export const CODEQL_VERSION_ML_POWERED_QUERIES = "2.7.5";
const CODEQL_VERSION_LUA_TRACER_CONFIG = "2.10.0";
export const CODEQL_VERSION_CONFIG_FILES = "2.10.1";
const CODEQL_VERSION_LUA_TRACING_GO_WINDOWS_FIXED = "2.10.4";
export const CODEQL_VERSION_GHES_PACK_DOWNLOAD = "2.10.4";
/**
* This variable controls using the new style of tracing from the CodeQL
@@ -1097,11 +1101,22 @@ async function getCodeQLForCmd(
* If no version is specified, then the latest version is
* downloaded. The check to determine what the latest version is is done
* each time this package is requested.
*
* Optionally, a `qlconfigFile` is included. If used, then this file
* is used to determine which registry each pack is downloaded from.
*/
async packDownload(packs: string[]): Promise<PackDownloadOutput> {
async packDownload(
packs: string[],
qlconfigFile: string | undefined
): Promise<PackDownloadOutput> {
const qlconfigArg = qlconfigFile
? [`--qlconfig-file=${qlconfigFile}`]
: ([] as string[]);
const codeqlArgs = [
"pack",
"download",
...qlconfigArg,
"--format=json",
"--resolve-query-specs",
...getExtraOptionsFromEnv(["pack", "download"]),

253
src/config-utils.test.ts
View File

@@ -3,6 +3,7 @@ import * as path from "path";
import * as github from "@actions/github";
import test, { ExecutionContext } from "ava";
import * as yaml from "js-yaml";
import * as sinon from "sinon";
import * as api from "./api-client";
@@ -21,6 +22,7 @@ const sampleApiDetails = {
externalRepoAuth: "token",
url: "https://github.example.com",
apiURL: undefined,
registriesAuthTokens: undefined,
};
const gitHubVersion = { type: util.GitHubVariant.DOTCOM } as util.GitHubVersion;
@@ -89,6 +91,7 @@ test("load empty config", async (t) => {
undefined,
undefined,
undefined,
undefined,
false,
false,
"",
@@ -159,6 +162,7 @@ test("loading config saves config", async (t) => {
undefined,
undefined,
undefined,
undefined,
false,
false,
"",
@@ -194,6 +198,7 @@ test("load input outside of workspace", async (t) => {
undefined,
undefined,
undefined,
undefined,
"../input",
undefined,
false,
@@ -233,6 +238,7 @@ test("load non-local input with invalid repo syntax", async (t) => {
undefined,
undefined,
undefined,
undefined,
configFile,
undefined,
false,
@@ -273,6 +279,7 @@ test("load non-existent input", async (t) => {
languages,
undefined,
undefined,
undefined,
configFile,
undefined,
false,
@@ -379,6 +386,7 @@ test("load non-empty input", async (t) => {
languages,
undefined,
undefined,
undefined,
configFilePath,
undefined,
false,
@@ -449,6 +457,7 @@ test("Default queries are used", async (t) => {
languages,
undefined,
undefined,
undefined,
configFilePath,
undefined,
false,
@@ -527,6 +536,7 @@ test("Queries can be specified in config file", async (t) => {
languages,
undefined,
undefined,
undefined,
configFilePath,
undefined,
false,
@@ -604,6 +614,7 @@ test("Queries from config file can be overridden in workflow file", async (t) =>
languages,
testQueries,
undefined,
undefined,
configFilePath,
undefined,
false,
@@ -679,6 +690,7 @@ test("Queries in workflow file can be used in tandem with the 'disable default q
languages,
testQueries,
undefined,
undefined,
configFilePath,
undefined,
false,
@@ -747,6 +759,7 @@ test("Multiple queries can be specified in workflow file, no config file require
undefined,
undefined,
undefined,
undefined,
false,
false,
"",
@@ -832,6 +845,7 @@ test("Queries in workflow file can be added to the set of queries without overri
languages,
testQueries,
undefined,
undefined,
configFilePath,
undefined,
false,
@@ -915,6 +929,7 @@ test("Invalid queries in workflow file handled correctly", async (t) => {
undefined,
undefined,
undefined,
undefined,
false,
false,
"",
@@ -984,6 +999,7 @@ test("API client used when reading remote config", async (t) => {
languages,
undefined,
undefined,
undefined,
configFile,
undefined,
false,
@@ -1014,6 +1030,7 @@ test("Remote config handles the case where a directory is provided", async (t) =
undefined,
undefined,
undefined,
undefined,
repoReference,
undefined,
false,
@@ -1052,6 +1069,7 @@ test("Invalid format of remote config handled correctly", async (t) => {
undefined,
undefined,
undefined,
undefined,
repoReference,
undefined,
false,
@@ -1096,6 +1114,7 @@ test("No detected languages", async (t) => {
undefined,
undefined,
undefined,
undefined,
false,
false,
"",
@@ -1127,6 +1146,7 @@ test("Unknown languages", async (t) => {
undefined,
undefined,
undefined,
undefined,
false,
false,
"",
@@ -1181,6 +1201,7 @@ test("Config specifies packages", async (t) => {
languages,
undefined,
undefined,
undefined,
configFile,
undefined,
false,
@@ -1241,6 +1262,7 @@ test("Config specifies packages for multiple languages", async (t) => {
languages,
undefined,
undefined,
undefined,
configFile,
undefined,
false,
@@ -1312,6 +1334,7 @@ function doInvalidInputTest(
languages,
undefined,
undefined,
undefined,
configFile,
undefined,
false,
@@ -1901,6 +1924,7 @@ const mlPoweredQueriesMacro = test.macro({
packsInput,
undefined,
undefined,
undefined,
false,
false,
"",
@@ -2208,30 +2232,209 @@ test(
/"a-pack-without-a-scope" is not a valid pack/
);
test("downloadPacks", async (t) => {
const packDownloadStub = sinon.stub();
packDownloadStub.callsFake((packs) => ({
packs,
}));
const codeQL = setCodeQL({
packDownload: packDownloadStub,
test("downloadPacks-no-registries", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
const packDownloadStub = sinon.stub();
packDownloadStub.callsFake((packs) => ({
packs,
}));
const codeQL = setCodeQL({
packDownload: packDownloadStub,
});
const logger = getRunnerLogger(true);
// packs are supplied for go, java, and python
// analyzed languages are java, javascript, and python
await configUtils.downloadPacks(
codeQL,
[Language.javascript, Language.java, Language.python],
{
java: ["a", "b"],
go: ["c", "d"],
python: ["e", "f"],
},
undefined, // registries
sampleApiDetails,
tmpDir,
logger
);
// Expecting packs to be downloaded once for java and once for python
t.deepEqual(packDownloadStub.callCount, 2);
// no config file was created, so pass `undefined` as the config file path
t.deepEqual(packDownloadStub.firstCall.args, [["a", "b"], undefined]);
t.deepEqual(packDownloadStub.secondCall.args, [["e", "f"], undefined]);
});
});
test("downloadPacks-with-registries", async (t) => {
// same thing, but this time include a registries block and
// associated env vars
return await util.withTmpDir(async (tmpDir) => {
process.env.GITHUB_TOKEN = "not-a-token";
process.env.CODEQL_REGISTRIES_AUTH = "not-a-registries-auth";
const logger = getRunnerLogger(true);
const registries = [
{
// no slash
url: "http://ghcr.io",
packages: ["codeql/*", "dsp-testing/*"],
token: "not-a-token",
},
{
// with slash
url: "https://containers.GHEHOSTNAME1/v2/",
packages: "semmle/*",
token: "still-not-a-token",
},
];
// append a slash to the first url
const expectedRegistries = registries.map((r, i) => ({
packages: r.packages,
url: i === 0 ? `${r.url}/` : r.url,
}));
const expectedConfigFile = path.join(tmpDir, "qlconfig.yml");
const packDownloadStub = sinon.stub();
packDownloadStub.callsFake((packs, configFile) => {
t.deepEqual(configFile, expectedConfigFile);
// verify the env vars were set correctly
t.deepEqual(process.env.GITHUB_TOKEN, sampleApiDetails.auth);
t.deepEqual(
process.env.CODEQL_REGISTRIES_AUTH,
"http://ghcr.io=not-a-token,https://containers.GHEHOSTNAME1/v2/=still-not-a-token"
);
// verify the config file contents were set correctly
const config = yaml.load(fs.readFileSync(configFile, "utf8")) as {
registries: configUtils.RegistryConfigNoCredentials[];
};
t.deepEqual(config.registries, expectedRegistries);
return {
packs,
};
});
const codeQL = setCodeQL({
packDownload: packDownloadStub,
getVersion: () => Promise.resolve("2.10.5"),
});
// packs are supplied for go, java, and python
// analyzed languages are java, javascript, and python
await configUtils.downloadPacks(
codeQL,
[Language.javascript, Language.java, Language.python],
{
java: ["a", "b"],
go: ["c", "d"],
python: ["e", "f"],
},
registries,
sampleApiDetails,
tmpDir,
logger
);
// Same packs are downloaded as in previous test
t.deepEqual(packDownloadStub.callCount, 2);
t.deepEqual(packDownloadStub.firstCall.args, [
["a", "b"],
expectedConfigFile,
]);
t.deepEqual(packDownloadStub.secondCall.args, [
["e", "f"],
expectedConfigFile,
]);
// Verify that the env vars were unset.
t.deepEqual(process.env.GITHUB_TOKEN, "not-a-token");
t.deepEqual(process.env.CODEQL_REGISTRIES_AUTH, "not-a-registries-auth");
});
});
test("downloadPacks-with-registries fails on 2.10.3", async (t) => {
// same thing, but this time include a registries block and
// associated env vars
return await util.withTmpDir(async (tmpDir) => {
process.env.GITHUB_TOKEN = "not-a-token";
process.env.CODEQL_REGISTRIES_AUTH = "not-a-registries-auth";
const logger = getRunnerLogger(true);
const registries = [
{
url: "http://ghcr.io",
packages: ["codeql/*", "dsp-testing/*"],
token: "not-a-token",
},
{
url: "https://containers.GHEHOSTNAME1/v2/",
packages: "semmle/*",
token: "still-not-a-token",
},
];
const codeQL = setCodeQL({
getVersion: () => Promise.resolve("2.10.3"),
});
await t.throwsAsync(
async () => {
return await configUtils.downloadPacks(
codeQL,
[Language.javascript, Language.java, Language.python],
{},
registries,
sampleApiDetails,
tmpDir,
logger
);
},
{ instanceOf: Error },
"'registries' input is not supported on CodeQL versions less than 2.10.4."
);
});
});
test("downloadPacks-with-registries fails with invalid registries block", async (t) => {
// same thing, but this time include a registries block and
// associated env vars
return await util.withTmpDir(async (tmpDir) => {
process.env.GITHUB_TOKEN = "not-a-token";
process.env.CODEQL_REGISTRIES_AUTH = "not-a-registries-auth";
const logger = getRunnerLogger(true);
const registries = [
{
// missing url property
packages: ["codeql/*", "dsp-testing/*"],
token: "not-a-token",
},
{
url: "https://containers.GHEHOSTNAME1/v2/",
packages: "semmle/*",
token: "still-not-a-token",
},
];
const codeQL = setCodeQL({
getVersion: () => Promise.resolve("2.10.4"),
});
await t.throwsAsync(
async () => {
return await configUtils.downloadPacks(
codeQL,
[Language.javascript, Language.java, Language.python],
{},
registries as any,
sampleApiDetails,
tmpDir,
logger
);
},
{ instanceOf: Error },
"Invalid 'registries' input. Must be an array of objects with 'url' and 'packages' properties."
);
});
const logger = getRunnerLogger(true);
// packs are supplied for go, java, and python
// analyzed languages are java, javascript, and python
await configUtils.downloadPacks(
codeQL,
[Language.javascript, Language.java, Language.python],
{
java: ["a", "b"],
go: ["c", "d"],
python: ["e", "f"],
},
logger
);
t.deepEqual(packDownloadStub.callCount, 2);
t.deepEqual(packDownloadStub.firstCall.args, [["a", "b"]]);
t.deepEqual(packDownloadStub.secondCall.args, [["e", "f"]]);
});

176
src/config-utils.ts
View File

@@ -9,6 +9,7 @@ import * as semver from "semver";
import * as api from "./api-client";
import {
CodeQL,
CODEQL_VERSION_GHES_PACK_DOWNLOAD,
CODEQL_VERSION_ML_POWERED_QUERIES,
CODEQL_VERSION_ML_POWERED_QUERIES_WINDOWS,
ResolveQueriesOutput,
@@ -61,6 +62,23 @@ export interface UserConfig {
export type QueryFilter = ExcludeQueryFilter | IncludeQueryFilter;
export type RegistryConfigWithCredentials = RegistryConfigNoCredentials & {
// Token to use when downloading packs from this registry.
token: string;
};
/**
* The list of registries and the associated pack globs that determine where each
* pack can be downloaded from.
*/
export interface RegistryConfigNoCredentials {
// URL of a package registry, eg- https://ghcr.io/v2/
url: string;
// List of globs that determine which packs are associated with this registry.
packages: string[] | string;
}
interface ExcludeQueryFilter {
exclude: Record<string, string[] | string>;
}
@@ -1608,6 +1626,7 @@ export async function initConfig(
languagesInput: string | undefined,
queriesInput: string | undefined,
packsInput: string | undefined,
registriesInput: string | undefined,
configFile: string | undefined,
dbLocation: string | undefined,
trapCachingEnabled: boolean,
@@ -1686,7 +1705,16 @@ export async function initConfig(
// happen in the CLI during the `database init` command, so no need
// to download them here.
if (!(await useCodeScanningConfigInCli(codeQL))) {
await downloadPacks(codeQL, config.languages, config.packs, logger);
const registries = parseRegistries(registriesInput);
await downloadPacks(
codeQL,
config.languages,
config.packs,
registries,
apiDetails,
config.tempDir,
logger
);
}
// Save the config so we can easily access it again in the future
@@ -1694,6 +1722,18 @@ export async function initConfig(
return config;
}
function parseRegistries(
registriesInput: string | undefined
): RegistryConfigWithCredentials[] | undefined {
try {
return registriesInput
? (yaml.load(registriesInput) as RegistryConfigWithCredentials[])
: undefined;
} catch (e) {
throw new Error("Invalid registries input. Must be a YAML string.");
}
}
function isLocal(configPath: string): boolean {
// If the path starts with ./, look locally
if (configPath.indexOf("./") === 0) {
@@ -1795,30 +1835,126 @@ export async function downloadPacks(
codeQL: CodeQL,
languages: Language[],
packs: Packs,
registries: RegistryConfigWithCredentials[] | undefined,
apiDetails: api.GitHubApiDetails,
tmpDir: string,
logger: Logger
) {
let numPacksDownloaded = 0;
logger.startGroup("Downloading packs");
for (const language of languages) {
const packsWithVersion = packs[language];
if (packsWithVersion?.length) {
logger.info(`Downloading custom packs for ${language}`);
const results = await codeQL.packDownload(packsWithVersion);
numPacksDownloaded += results.packs.length;
logger.info(
`Downloaded packs: ${results.packs
.map((r) => `${r.name}@${r.version || "latest"}`)
.join(", ")}`
let qlconfigFile: string | undefined;
let registriesAuthTokens: string | undefined;
if (registries) {
if (
!(await codeQlVersionAbove(codeQL, CODEQL_VERSION_GHES_PACK_DOWNLOAD))
) {
throw new Error(
`'registries' input is not supported on CodeQL versions less than ${CODEQL_VERSION_GHES_PACK_DOWNLOAD}.`
);
}
// generate a qlconfig.yml file to hold the registry configs.
const qlconfig = createRegistriesBlock(registries);
qlconfigFile = path.join(tmpDir, "qlconfig.yml");
fs.writeFileSync(qlconfigFile, yaml.dump(qlconfig), "utf8");
registriesAuthTokens = registries
.map((registry) => `${registry.url}=${registry.token}`)
.join(",");
}
await wrapEnvironment(
{
GITHUB_TOKEN: apiDetails.auth,
CODEQL_REGISTRIES_AUTH: registriesAuthTokens,
},
async () => {
let numPacksDownloaded = 0;
logger.startGroup("Downloading packs");
for (const language of languages) {
const packsWithVersion = packs[language];
if (packsWithVersion?.length) {
logger.info(`Downloading custom packs for ${language}`);
const results = await codeQL.packDownload(
packsWithVersion,
qlconfigFile
);
numPacksDownloaded += results.packs.length;
logger.info(
`Downloaded: ${results.packs
.map((r) => `${r.name}@${r.version || "latest"}`)
.join(", ")}`
);
}
}
if (numPacksDownloaded > 0) {
logger.info(
`Downloaded ${numPacksDownloaded} ${packs === 1 ? "pack" : "packs"}`
);
} else {
logger.info("No packs to download");
}
logger.endGroup();
}
);
}
function createRegistriesBlock(registries: RegistryConfigWithCredentials[]): {
registries: RegistryConfigNoCredentials[];
} {
if (
!Array.isArray(registries) ||
registries.some((r) => !r.url || !r.packages)
) {
throw new Error(
"Invalid 'registries' input. Must be an array of objects with 'url' and 'packages' properties."
);
}
// be sure to remove the `token` field from the registry before writing it to disk.
const safeRegistries = registries.map((registry) => ({
// ensure the url ends with a slash to avoid a bug in the CLI 2.10.4
url: !registry?.url.endsWith("/") ? `${registry.url}/` : registry.url,
packages: registry.packages,
}));
const qlconfig = {
registries: safeRegistries,
};
return qlconfig;
}
/**
* Create a temporary environment based on the existing environment and overridden
* by the given environment variables that are passed in as arguments.
*
* Use this new environment in the context of the given operation. After completing
* the operation, restore the original environment.
*
* This function does not support un-setting environment variables.
*
* @param env
* @param operation
*/
async function wrapEnvironment(
env: Record<string, string | undefined>,
operation: Function
) {
// Remember the original env
const oldEnv = { ...process.env };
// Set the new env
for (const [key, value] of Object.entries(env)) {
// Ignore undefined keys
if (value !== undefined) {
process.env[key] = value;
}
}
if (numPacksDownloaded > 0) {
logger.info(
`Downloaded ${numPacksDownloaded} ${packs === 1 ? "pack" : "packs"}`
);
} else {
logger.info("No packs to download");
try {
// Run the operation
await operation();
} finally {
// Restore the old env
for (const [key, value] of Object.entries(oldEnv)) {
process.env[key] = value;
}
}
logger.endGroup();
}

1
src/init-action.ts
View File

@@ -196,6 +196,7 @@ async function run() {
getOptionalInput("languages"),
getOptionalInput("queries"),
getOptionalInput("packs"),
getOptionalInput("registries"),
getOptionalInput("config-file"),
getOptionalInput("db-location"),
await getTrapCachingEnabled(featureFlags),

2
src/init.ts
View File

@@ -42,6 +42,7 @@ export async function initConfig(
languagesInput: string | undefined,
queriesInput: string | undefined,
packsInput: string | undefined,
registriesInput: string | undefined,
configFile: string | undefined,
dbLocation: string | undefined,
trapCachingEnabled: boolean,
@@ -62,6 +63,7 @@ export async function initConfig(
languagesInput,
queriesInput,
packsInput,
registriesInput,
configFile,
dbLocation,
trapCachingEnabled,

1
src/runner.ts
View File

@@ -240,6 +240,7 @@ program
cmd.languages,
cmd.queries,
cmd.packs,
undefined, // we won't support registries in the runner
cmd.configFile,
undefined,
false,