Merge branch 'main' into mbg/analyze/use-upload-sarif

This commit is contained in:
Michael B. Gale
2025-10-22 00:42:55 +01:00
committed by GitHub
77 changed files with 103352 additions and 3978 deletions

View File

@@ -1,12 +1,19 @@
import test from "ava";
import * as sinon from "sinon";
import * as actionsUtil from "./actions-util";
import {
AnalysisKind,
getAnalysisKinds,
parseAnalysisKinds,
supportedAnalysisKinds,
} from "./analyses";
import { getRunnerLogger } from "./logging";
import { setupTests } from "./testing-utils";
import { ConfigurationError } from "./util";
setupTests(test);
test("All known analysis kinds can be parsed successfully", async (t) => {
for (const analysisKind of supportedAnalysisKinds) {
t.deepEqual(await parseAnalysisKinds(analysisKind), [analysisKind]);
@@ -34,3 +41,29 @@ test("Parsing analysis kinds requires at least one analysis kind", async (t) =>
instanceOf: ConfigurationError,
});
});
test("getAnalysisKinds - returns expected analysis kinds for `analysis-kinds` input", async (t) => {
const requiredInputStub = sinon.stub(actionsUtil, "getRequiredInput");
requiredInputStub
.withArgs("analysis-kinds")
.returns("code-scanning,code-quality");
const result = await getAnalysisKinds(getRunnerLogger(true), true);
t.assert(result.includes(AnalysisKind.CodeScanning));
t.assert(result.includes(AnalysisKind.CodeQuality));
});
test("getAnalysisKinds - includes `code-quality` when deprecated `quality-queries` input is used", async (t) => {
const requiredInputStub = sinon.stub(actionsUtil, "getRequiredInput");
requiredInputStub.withArgs("analysis-kinds").returns("code-scanning");
const optionalInputStub = sinon.stub(actionsUtil, "getOptionalInput");
optionalInputStub.withArgs("quality-queries").returns("code-quality");
const result = await getAnalysisKinds(getRunnerLogger(true), true);
t.assert(result.includes(AnalysisKind.CodeScanning));
t.assert(result.includes(AnalysisKind.CodeQuality));
});
test("getAnalysisKinds - throws if `analysis-kinds` input is invalid", async (t) => {
const requiredInputStub = sinon.stub(actionsUtil, "getRequiredInput");
requiredInputStub.withArgs("analysis-kinds").returns("no-such-thing");
await t.throwsAsync(getAnalysisKinds(getRunnerLogger(true), true));
});

View File

@@ -1,4 +1,8 @@
import { fixCodeQualityCategory } from "./actions-util";
import {
fixCodeQualityCategory,
getOptionalInput,
getRequiredInput,
} from "./actions-util";
import { Logger } from "./logging";
import { ConfigurationError } from "./util";
@@ -41,6 +45,55 @@ export async function parseAnalysisKinds(
);
}
// Used to avoid re-parsing the input after we have done it once.
let cachedAnalysisKinds: AnalysisKind[] | undefined;
/**
* Initialises the analysis kinds for the analysis based on the `analysis-kinds` input.
* This function will also use the deprecated `quality-queries` input as an indicator to enable `code-quality`.
* If the `analysis-kinds` input cannot be parsed, a `ConfigurationError` is thrown.
*
* @param logger The logger to use.
* @param skipCache For testing, whether to ignore the cached values (default: false).
*
* @returns The array of enabled analysis kinds.
* @throws A `ConfigurationError` if the `analysis-kinds` input cannot be parsed.
*/
export async function getAnalysisKinds(
logger: Logger,
skipCache: boolean = false,
): Promise<AnalysisKind[]> {
if (!skipCache && cachedAnalysisKinds !== undefined) {
return cachedAnalysisKinds;
}
cachedAnalysisKinds = await parseAnalysisKinds(
getRequiredInput("analysis-kinds"),
);
// Warn that `quality-queries` is deprecated if there is an argument for it.
const qualityQueriesInput = getOptionalInput("quality-queries");
if (qualityQueriesInput !== undefined) {
logger.warning(
"The `quality-queries` input is deprecated and will be removed in a future version of the CodeQL Action. " +
"Use the `analysis-kinds` input to configure different analysis kinds instead.",
);
}
// For backwards compatibility, add Code Quality to the enabled analysis kinds
// if an input to `quality-queries` was specified. We should remove this once
// `quality-queries` is no longer used.
if (
!cachedAnalysisKinds.includes(AnalysisKind.CodeQuality) &&
qualityQueriesInput !== undefined
) {
cachedAnalysisKinds.push(AnalysisKind.CodeQuality);
}
return cachedAnalysisKinds;
}
/** The queries to use for Code Quality analyses. */
export const codeQualityQueries: string[] = ["code-quality"];

View File

@@ -7,12 +7,12 @@ import { getActionVersion, getRequiredInput } from "./actions-util";
import { Logger } from "./logging";
import { getRepositoryNwo, RepositoryNwo } from "./repository";
import {
asHTTPError,
ConfigurationError,
getRequiredEnvParam,
GITHUB_DOTCOM_URL,
GitHubVariant,
GitHubVersion,
isHTTPError,
parseGitHubUrl,
parseMatrixInput,
} from "./util";
@@ -280,22 +280,29 @@ export async function getRepositoryProperties(repositoryNwo: RepositoryNwo) {
}
export function wrapApiConfigurationError(e: unknown) {
if (isHTTPError(e)) {
const httpError = asHTTPError(e);
if (httpError !== undefined) {
if (
e.message.includes("API rate limit exceeded for installation") ||
e.message.includes("commit not found") ||
e.message.includes("Resource not accessible by integration") ||
/ref .* not found in this repository/.test(e.message)
[
/API rate limit exceeded/,
/commit not found/,
/Resource not accessible by integration/,
/ref .* not found in this repository/,
].some((pattern) => pattern.test(httpError.message))
) {
return new ConfigurationError(e.message);
} else if (
e.message.includes("Bad credentials") ||
e.message.includes("Not Found")
return new ConfigurationError(httpError.message);
}
if (
httpError.message.includes("Bad credentials") ||
httpError.message.includes("Not Found")
) {
return new ConfigurationError(
"Please check that your token is valid and has the required permissions: contents: read, security-events: write",
);
}
if (httpError.status === 429) {
return new ConfigurationError("API rate limit exceeded");
}
}
return e;
}

View File

@@ -310,6 +310,20 @@ test("wrapCliConfigurationError - pack cannot be found", (t) => {
t.true(wrappedError instanceof ConfigurationError);
});
test("wrapCliConfigurationError - unknown query file", (t) => {
const commandError = new CommandInvocationError(
"codeql",
["database", "init"],
2,
"my-query-file is not a .ql file, .qls file, a directory, or a query pack specification. See the logs for more details.",
);
const cliError = new CliError(commandError);
const wrappedError = wrapCliConfigurationError(cliError);
t.true(wrappedError instanceof ConfigurationError);
});
test("wrapCliConfigurationError - pack missing auth", (t) => {
const commandError = new CommandInvocationError(
"codeql",

View File

@@ -264,6 +264,9 @@ export const cliErrorsConfig: Record<
new RegExp(
"Query pack .* cannot be found\\. Check the spelling of the pack\\.",
),
new RegExp(
"is not a .ql file, .qls file, a directory, or a query pack specification.",
),
],
},
[CliConfigErrorCategory.PackMissingAuth]: {

View File

@@ -3,7 +3,6 @@ import * as path from "path";
import * as core from "@actions/core";
import * as toolrunner from "@actions/exec/lib/toolrunner";
import { RequestError } from "@octokit/request-error";
import * as yaml from "js-yaml";
import {
@@ -371,11 +370,11 @@ export async function setupCodeQL(
toolsVersion,
zstdAvailability,
};
} catch (e) {
} catch (rawError) {
const e = api.wrapApiConfigurationError(rawError);
const ErrorClass =
e instanceof util.ConfigurationError ||
(e instanceof Error && e.message.includes("ENOSPC")) || // out of disk space
(e instanceof RequestError && e.status === 429) // rate limited
(e instanceof Error && e.message.includes("ENOSPC")) // out of disk space
? util.ConfigurationError
: Error;

View File

@@ -49,10 +49,9 @@ function createTestInitConfigInputs(
return Object.assign(
{},
{
analysisKindsInput: "code-scanning",
analysisKinds: [AnalysisKind.CodeScanning],
languagesInput: undefined,
queriesInput: undefined,
qualityQueriesInput: undefined,
packsInput: undefined,
configFile: undefined,
dbLocation: undefined,
@@ -149,6 +148,7 @@ test("load empty config", async (t) => {
});
const config = await configUtils.initConfig(
createFeatures([]),
createTestInitConfigInputs({
languagesInput: languages,
repository: { owner: "github", repo: "example" },
@@ -188,8 +188,9 @@ test("load code quality config", async (t) => {
});
const config = await configUtils.initConfig(
createFeatures([]),
createTestInitConfigInputs({
analysisKindsInput: "code-quality",
analysisKinds: [AnalysisKind.CodeQuality],
languagesInput: languages,
repository: { owner: "github", repo: "example" },
tempDir,
@@ -272,8 +273,9 @@ test("initActionState doesn't throw if there are queries configured in the repos
await t.notThrowsAsync(async () => {
const config = await configUtils.initConfig(
createFeatures([]),
createTestInitConfigInputs({
analysisKindsInput: "code-quality",
analysisKinds: [AnalysisKind.CodeQuality],
languagesInput: languages,
repository: { owner: "github", repo: "example" },
tempDir,
@@ -310,6 +312,7 @@ test("loading a saved config produces the same config", async (t) => {
t.deepEqual(await configUtils.getConfig(tempDir, logger), undefined);
const config1 = await configUtils.initConfig(
createFeatures([]),
createTestInitConfigInputs({
languagesInput: "javascript,python",
tempDir,
@@ -361,6 +364,7 @@ test("loading config with version mismatch throws", async (t) => {
.returns("does-not-exist");
const config = await configUtils.initConfig(
createFeatures([]),
createTestInitConfigInputs({
languagesInput: "javascript,python",
tempDir,
@@ -389,6 +393,7 @@ test("load input outside of workspace", async (t) => {
return await withTmpDir(async (tempDir) => {
try {
await configUtils.initConfig(
createFeatures([]),
createTestInitConfigInputs({
configFile: "../input",
tempDir,
@@ -416,6 +421,7 @@ test("load non-local input with invalid repo syntax", async (t) => {
try {
await configUtils.initConfig(
createFeatures([]),
createTestInitConfigInputs({
configFile,
tempDir,
@@ -444,6 +450,7 @@ test("load non-existent input", async (t) => {
try {
await configUtils.initConfig(
createFeatures([]),
createTestInitConfigInputs({
languagesInput,
configFile,
@@ -527,6 +534,7 @@ test("load non-empty input", async (t) => {
const configFilePath = createConfigFile(inputFileContents, tempDir);
const actualConfig = await configUtils.initConfig(
createFeatures([]),
createTestInitConfigInputs({
languagesInput,
buildModeInput: "none",
@@ -583,6 +591,7 @@ test("Using config input and file together, config input should be used.", async
const languagesInput = "javascript";
const config = await configUtils.initConfig(
createFeatures([]),
createTestInitConfigInputs({
languagesInput,
configFile: configFilePath,
@@ -633,6 +642,7 @@ test("API client used when reading remote config", async (t) => {
const languagesInput = "javascript";
await configUtils.initConfig(
createFeatures([]),
createTestInitConfigInputs({
languagesInput,
configFile,
@@ -653,6 +663,7 @@ test("Remote config handles the case where a directory is provided", async (t) =
const repoReference = "octo-org/codeql-config/config.yaml@main";
try {
await configUtils.initConfig(
createFeatures([]),
createTestInitConfigInputs({
configFile: repoReference,
tempDir,
@@ -681,6 +692,7 @@ test("Invalid format of remote config handled correctly", async (t) => {
const repoReference = "octo-org/codeql-config/config.yaml@main";
try {
await configUtils.initConfig(
createFeatures([]),
createTestInitConfigInputs({
configFile: repoReference,
tempDir,
@@ -710,6 +722,7 @@ test("No detected languages", async (t) => {
try {
await configUtils.initConfig(
createFeatures([]),
createTestInitConfigInputs({
tempDir,
codeql,
@@ -732,6 +745,7 @@ test("Unknown languages", async (t) => {
try {
await configUtils.initConfig(
createFeatures([]),
createTestInitConfigInputs({
languagesInput,
tempDir,

View File

@@ -11,7 +11,6 @@ import {
CodeQuality,
codeQualityQueries,
CodeScanning,
parseAnalysisKinds,
} from "./analyses";
import * as api from "./api-client";
import { CachingKind, getCachingKind } from "./caching-utils";
@@ -20,6 +19,7 @@ import {
calculateAugmentation,
ExcludeQueryFilter,
generateCodeScanningConfig,
parseUserConfig,
UserConfig,
} from "./config/db-config";
import { shouldPerformDiffInformedAnalysis } from "./diff-informed-analysis-utils";
@@ -373,10 +373,8 @@ export async function getRawLanguages(
/** Inputs required to initialize a configuration. */
export interface InitConfigInputs {
analysisKindsInput: string;
languagesInput: string | undefined;
queriesInput: string | undefined;
qualityQueriesInput: string | undefined;
packsInput: string | undefined;
configFile: string | undefined;
dbLocation: string | undefined;
@@ -396,6 +394,7 @@ export interface InitConfigInputs {
apiDetails: api.GitHubApiCombinedDetails;
features: FeatureEnablement;
repositoryProperties: RepositoryProperties;
analysisKinds: AnalysisKind[];
logger: Logger;
}
@@ -405,10 +404,8 @@ export interface InitConfigInputs {
*/
export async function initActionState(
{
analysisKindsInput,
languagesInput,
queriesInput,
qualityQueriesInput,
packsInput,
buildModeInput,
dbLocation,
@@ -424,22 +421,11 @@ export async function initActionState(
githubVersion,
features,
repositoryProperties,
analysisKinds,
logger,
}: InitConfigInputs,
userConfig: UserConfig,
): Promise<Config> {
const analysisKinds = await parseAnalysisKinds(analysisKindsInput);
// For backwards compatibility, add Code Quality to the enabled analysis kinds
// if an input to `quality-queries` was specified. We should remove this once
// `quality-queries` is no longer used.
if (
!analysisKinds.includes(AnalysisKind.CodeQuality) &&
qualityQueriesInput !== undefined
) {
analysisKinds.push(AnalysisKind.CodeQuality);
}
const languages = await getLanguages(
codeql,
languagesInput,
@@ -540,10 +526,12 @@ async function downloadCacheWithTime(
}
async function loadUserConfig(
logger: Logger,
configFile: string,
workspacePath: string,
apiDetails: api.GitHubApiCombinedDetails,
tempDir: string,
validateConfig: boolean,
): Promise<UserConfig> {
if (isLocal(configFile)) {
if (configFile !== userConfigFromActionPath(tempDir)) {
@@ -556,9 +544,14 @@ async function loadUserConfig(
);
}
}
return getLocalConfig(configFile);
return getLocalConfig(logger, configFile, validateConfig);
} else {
return await getRemoteConfig(configFile, apiDetails);
return await getRemoteConfig(
logger,
configFile,
apiDetails,
validateConfig,
);
}
}
@@ -794,7 +787,10 @@ function hasQueryCustomisation(userConfig: UserConfig): boolean {
* This will parse the config from the user input if present, or generate
* a default config. The parsed config is then stored to a known location.
*/
export async function initConfig(inputs: InitConfigInputs): Promise<Config> {
export async function initConfig(
features: FeatureEnablement,
inputs: InitConfigInputs,
): Promise<Config> {
const { logger, tempDir } = inputs;
// if configInput is set, it takes precedence over configFile
@@ -814,11 +810,14 @@ export async function initConfig(inputs: InitConfigInputs): Promise<Config> {
logger.debug("No configuration file was provided");
} else {
logger.debug(`Using configuration file: ${inputs.configFile}`);
const validateConfig = await features.getValue(Feature.ValidateDbConfig);
userConfig = await loadUserConfig(
logger,
inputs.configFile,
inputs.workspacePath,
inputs.apiDetails,
tempDir,
validateConfig,
);
}
@@ -912,7 +911,11 @@ function isLocal(configPath: string): boolean {
return configPath.indexOf("@") === -1;
}
function getLocalConfig(configFile: string): UserConfig {
function getLocalConfig(
logger: Logger,
configFile: string,
validateConfig: boolean,
): UserConfig {
// Error if the file does not exist
if (!fs.existsSync(configFile)) {
throw new ConfigurationError(
@@ -920,12 +923,19 @@ function getLocalConfig(configFile: string): UserConfig {
);
}
return yaml.load(fs.readFileSync(configFile, "utf8")) as UserConfig;
return parseUserConfig(
logger,
configFile,
fs.readFileSync(configFile, "utf-8"),
validateConfig,
);
}
async function getRemoteConfig(
logger: Logger,
configFile: string,
apiDetails: api.GitHubApiCombinedDetails,
validateConfig: boolean,
): Promise<UserConfig> {
// retrieve the various parts of the config location, and ensure they're present
const format = new RegExp(
@@ -961,9 +971,12 @@ async function getRemoteConfig(
);
}
return yaml.load(
return parseUserConfig(
logger,
configFile,
Buffer.from(fileContents, "base64").toString("binary"),
) as UserConfig;
validateConfig,
);
}
/**

View File

@@ -2,7 +2,13 @@ import test, { ExecutionContext } from "ava";
import { RepositoryProperties } from "../feature-flags/properties";
import { KnownLanguage, Language } from "../languages";
import { prettyPrintPack } from "../util";
import { getRunnerLogger } from "../logging";
import {
checkExpectedLogMessages,
getRecordingLogger,
LoggedMessage,
} from "../testing-utils";
import { ConfigurationError, prettyPrintPack } from "../util";
import * as dbConfig from "./db-config";
@@ -391,3 +397,111 @@ test(
{},
/"a-pack-without-a-scope" is not a valid pack/,
);
test("parseUserConfig - successfully parses valid YAML", (t) => {
const result = dbConfig.parseUserConfig(
getRunnerLogger(true),
"test",
`
paths-ignore:
- "some/path"
queries:
- uses: foo
some-unknown-option: true
`,
true,
);
t.truthy(result);
if (t.truthy(result["paths-ignore"])) {
t.is(result["paths-ignore"].length, 1);
t.is(result["paths-ignore"][0], "some/path");
}
if (t.truthy(result["queries"])) {
t.is(result["queries"].length, 1);
t.deepEqual(result["queries"][0], { uses: "foo" });
}
});
test("parseUserConfig - throws a ConfigurationError if the file is not valid YAML", (t) => {
t.throws(
() =>
dbConfig.parseUserConfig(
getRunnerLogger(true),
"test",
`
paths-ignore:
- "some/path"
queries:
- foo
`,
true,
),
{
instanceOf: ConfigurationError,
},
);
});
test("parseUserConfig - validation isn't picky about `query-filters`", (t) => {
const loggedMessages: LoggedMessage[] = [];
const logger = getRecordingLogger(loggedMessages);
t.notThrows(() =>
dbConfig.parseUserConfig(
logger,
"test",
`
query-filters:
- something
- include: foo
- exclude: bar
`,
true,
),
);
});
test("parseUserConfig - throws a ConfigurationError if validation fails", (t) => {
const loggedMessages: LoggedMessage[] = [];
const logger = getRecordingLogger(loggedMessages);
t.throws(
() =>
dbConfig.parseUserConfig(
logger,
"test",
`
paths-ignore:
- "some/path"
queries: true
`,
true,
),
{
instanceOf: ConfigurationError,
message:
'The configuration file "test" is invalid: instance.queries is not of a type(s) array.',
},
);
const expectedMessages = ["instance.queries is not of a type(s) array"];
checkExpectedLogMessages(t, loggedMessages, expectedMessages);
});
test("parseUserConfig - throws no ConfigurationError if validation should fail, but feature is disabled", (t) => {
const loggedMessages: LoggedMessage[] = [];
const logger = getRecordingLogger(loggedMessages);
t.notThrows(() =>
dbConfig.parseUserConfig(
logger,
"test",
`
paths-ignore:
- "some/path"
queries: true
`,
false,
),
);
});

View File

@@ -1,5 +1,7 @@
import * as path from "path";
import * as yaml from "js-yaml";
import * as jsonschema from "jsonschema";
import * as semver from "semver";
import * as errorMessages from "../error-messages";
@@ -474,3 +476,53 @@ export function generateCodeScanningConfig(
return augmentedConfig;
}
/**
* Attempts to parse `contents` into a `UserConfig` value.
*
* @param logger The logger to use.
* @param pathInput The path to the file where `contents` was obtained from, for use in error messages.
* @param contents The string contents of a YAML file to try and parse as a `UserConfig`.
* @param validateConfig Whether to validate the configuration file against the schema.
* @returns The `UserConfig` corresponding to `contents`, if parsing was successful.
* @throws A `ConfigurationError` if parsing failed.
*/
export function parseUserConfig(
logger: Logger,
pathInput: string,
contents: string,
validateConfig: boolean,
): UserConfig {
try {
const schema =
// eslint-disable-next-line @typescript-eslint/no-require-imports
require("../../src/db-config-schema.json") as jsonschema.Schema;
const doc = yaml.load(contents);
if (validateConfig) {
const result = new jsonschema.Validator().validate(doc, schema);
if (result.errors.length > 0) {
for (const error of result.errors) {
logger.error(error.stack);
}
throw new ConfigurationError(
errorMessages.getInvalidConfigFileMessage(
pathInput,
result.errors.map((e) => e.stack),
),
);
}
}
return doc as UserConfig;
} catch (error) {
if (error instanceof yaml.YAMLException) {
throw new ConfigurationError(
errorMessages.getConfigFileParseErrorMessage(pathInput, error.message),
);
}
throw error;
}
}

View File

@@ -5,6 +5,7 @@ import test from "ava";
import * as sinon from "sinon";
import * as actionsUtil from "./actions-util";
import { AnalysisKind } from "./analyses";
import { GitHubApiDetails } from "./api-client";
import * as apiClient from "./api-client";
import { createStubCodeQL } from "./codeql";
@@ -108,6 +109,39 @@ test("Abort database upload if 'upload-database' input set to false", async (t)
});
});
test("Abort database upload if 'analysis-kinds: code-scanning' is not enabled", async (t) => {
await withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);
sinon
.stub(actionsUtil, "getRequiredInput")
.withArgs("upload-database")
.returns("true");
sinon.stub(gitUtils, "isAnalyzingDefaultBranch").resolves(true);
await mockHttpRequests(201);
const loggedMessages = [];
await uploadDatabases(
testRepoName,
getCodeQL(),
{
...getTestConfig(tmpDir),
analysisKinds: [AnalysisKind.CodeQuality],
},
testApiDetails,
getRecordingLogger(loggedMessages),
);
t.assert(
loggedMessages.find(
(v: LoggedMessage) =>
v.type === "debug" &&
v.message ===
"Not uploading database because 'analysis-kinds: code-scanning' is not enabled.",
) !== undefined,
);
});
});
test("Abort database upload if running against GHES", async (t) => {
await withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);

View File

@@ -1,6 +1,7 @@
import * as fs from "fs";
import * as actionsUtil from "./actions-util";
import { AnalysisKind } from "./analyses";
import { getApiClient, GitHubApiDetails } from "./api-client";
import { type CodeQL } from "./codeql";
import { Config } from "./config-utils";
@@ -22,6 +23,13 @@ export async function uploadDatabases(
return;
}
if (!config.analysisKinds.includes(AnalysisKind.CodeScanning)) {
logger.debug(
`Not uploading database because 'analysis-kinds: ${AnalysisKind.CodeScanning}' is not enabled.`,
);
return;
}
if (util.isInTestMode()) {
logger.debug("In test mode. Skipping database upload.");
return;

145
src/db-config-schema.json Normal file
View File

@@ -0,0 +1,145 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"title": "CodeQL Database Configuration",
"description": "Format of the config file supplied by the user for CodeQL analysis",
"type": "object",
"properties": {
"name": {
"type": "string",
"description": "Name of the configuration"
},
"disable-default-queries": {
"type": "boolean",
"description": "Whether to disable default queries"
},
"queries": {
"type": "array",
"description": "List of additional queries to run",
"items": {
"$ref": "#/definitions/QuerySpec"
}
},
"paths-ignore": {
"type": "array",
"description": "Paths to ignore during analysis",
"items": {
"type": "string"
}
},
"paths": {
"type": "array",
"description": "Paths to include in analysis",
"items": {
"type": "string"
}
},
"packs": {
"description": "Query packs to include. Can be a simple array for single-language analysis or an object with language-specific arrays for multi-language analysis",
"oneOf": [
{
"type": "array",
"items": {
"type": "string"
}
},
{
"type": "object",
"additionalProperties": {
"type": "array",
"items": {
"type": "string"
}
}
}
]
},
"query-filters": {
"type": "array",
"description": "Set of query filters to include and exclude extra queries based on CodeQL query suite include and exclude properties",
"items": {
"$ref": "#/definitions/QueryFilter"
}
}
},
"additionalProperties": true,
"definitions": {
"QuerySpec": {
"type": "object",
"description": "Detailed query specification object",
"properties": {
"name": {
"type": "string",
"description": "Optional name for the query"
},
"uses": {
"type": "string",
"description": "The query or query suite to use"
}
},
"required": ["uses"],
"additionalProperties": false
},
"QueryFilter": {
"description": "Query filter that can either include or exclude queries",
"oneOf": [
{
"$ref": "#/definitions/ExcludeQueryFilter"
},
{
"$ref": "#/definitions/IncludeQueryFilter"
},
{}
]
},
"ExcludeQueryFilter": {
"type": "object",
"description": "Filter to exclude queries",
"properties": {
"exclude": {
"type": "object",
"description": "Queries to exclude",
"additionalProperties": {
"oneOf": [
{
"type": "array",
"items": {
"type": "string"
}
},
{
"type": "string"
}
]
}
}
},
"required": ["exclude"],
"additionalProperties": false
},
"IncludeQueryFilter": {
"type": "object",
"description": "Filter to include queries",
"properties": {
"include": {
"type": "object",
"description": "Queries to include",
"additionalProperties": {
"oneOf": [
{
"type": "array",
"items": {
"type": "string"
}
},
{
"type": "string"
}
]
}
}
},
"required": ["include"],
"additionalProperties": false
}
}
}

View File

@@ -1,6 +1,6 @@
{
"bundleVersion": "codeql-bundle-v2.23.2",
"cliVersion": "2.23.2",
"priorBundleVersion": "codeql-bundle-v2.23.1",
"priorCliVersion": "2.23.1"
"bundleVersion": "codeql-bundle-v2.23.3",
"cliVersion": "2.23.3",
"priorBundleVersion": "codeql-bundle-v2.23.2",
"priorCliVersion": "2.23.2"
}

View File

@@ -47,6 +47,9 @@ export enum EnvVar {
/** Whether the CodeQL Action has already warned the user about low disk space. */
HAS_WARNED_ABOUT_DISK_SPACE = "CODEQL_ACTION_HAS_WARNED_ABOUT_DISK_SPACE",
/** Whether the `setup-codeql` action has been run. */
SETUP_CODEQL_ACTION_HAS_RUN = "CODEQL_ACTION_SETUP_CODEQL_HAS_RUN",
/** Whether the init action has been run. */
INIT_ACTION_HAS_RUN = "CODEQL_ACTION_INIT_HAS_RUN",

View File

@@ -14,6 +14,22 @@ export function getConfigFileDoesNotExistErrorMessage(
return `The configuration file "${configFile}" does not exist`;
}
export function getConfigFileParseErrorMessage(
configFile: string,
message: string,
): string {
return `Cannot parse "${configFile}": ${message}`;
}
export function getInvalidConfigFileMessage(
configFile: string,
messages: string[],
): string {
const andMore =
messages.length > 10 ? `, and ${messages.length - 10} more.` : ".";
return `The configuration file "${configFile}" is invalid: ${messages.slice(0, 10).join(", ")}${andMore}`;
}
export function getConfigFileRepoFormatInvalidMessage(
configFile: string,
): string {

View File

@@ -77,6 +77,7 @@ export enum Feature {
QaTelemetryEnabled = "qa_telemetry_enabled",
ResolveSupportedLanguagesUsingCli = "resolve_supported_languages_using_cli",
UseRepositoryProperties = "use_repository_properties",
ValidateDbConfig = "validate_db_config",
}
export const featureConfig: Record<
@@ -287,6 +288,11 @@ export const featureConfig: Record<
envVar: "CODEQL_ACTION_JAVA_MINIMIZE_DEPENDENCY_JARS",
minimumVersion: "2.23.0",
},
[Feature.ValidateDbConfig]: {
defaultValue: false,
envVar: "CODEQL_ACTION_VALIDATE_DB_CONFIG",
minimumVersion: undefined,
},
};
/**
@@ -651,12 +657,13 @@ class GitHubFeatureFlags {
this.hasAccessedRemoteFeatureFlags = true;
return remoteFlags;
} catch (e) {
if (util.isHTTPError(e) && e.status === 403) {
const httpError = util.asHTTPError(e);
if (httpError?.status === 403) {
this.logger.warning(
"This run of the CodeQL Action does not have permission to access Code Scanning API endpoints. " +
"As a result, it will not be opted into any experimental features. " +
"This could be because the Action is running on a pull request from a fork. If not, " +
`please ensure the Action has the 'security-events: write' permission. Details: ${e.message}`,
`please ensure the Action has the 'security-events: write' permission. Details: ${httpError.message}`,
);
this.hasAccessedRemoteFeatureFlags = false;
return {};

View File

@@ -2,6 +2,7 @@ import test, { ExecutionContext } from "ava";
import * as sinon from "sinon";
import * as actionsUtil from "./actions-util";
import { AnalysisKind } from "./analyses";
import * as codeql from "./codeql";
import * as configUtils from "./config-utils";
import { Feature } from "./feature-flags";
@@ -28,12 +29,13 @@ test("post: init action with debug mode off", async (t) => {
const gitHubVersion: util.GitHubVersion = {
type: util.GitHubVariant.DOTCOM,
};
sinon.stub(configUtils, "getConfig").resolves({
debugMode: false,
gitHubVersion,
languages: [],
packs: [],
} as unknown as configUtils.Config);
sinon.stub(configUtils, "getConfig").resolves(
createTestConfig({
debugMode: false,
gitHubVersion,
languages: [],
}),
);
const uploadAllAvailableDebugArtifactsSpy = sinon.spy();
const printDebugLogsSpy = sinon.spy();
@@ -295,6 +297,17 @@ test("uploading failed SARIF run fails when workflow does not reference github/c
t.truthy(result.upload_failed_run_stack_trace);
});
test("not uploading failed SARIF when `code-scanning` is not an enabled analysis kind", async (t) => {
const result = await testFailedSarifUpload(t, createTestWorkflow([]), {
analysisKinds: [AnalysisKind.CodeQuality],
expectUpload: false,
});
t.is(
result.upload_failed_run_skipped_because,
"Code Scanning is not enabled.",
);
});
function createTestWorkflow(
steps: workflow.WorkflowJobStep[],
): workflow.Workflow {
@@ -327,20 +340,22 @@ async function testFailedSarifUpload(
expectUpload = true,
exportDiagnosticsEnabled = false,
matrix = {},
analysisKinds = [AnalysisKind.CodeScanning],
}: {
category?: string;
databaseExists?: boolean;
expectUpload?: boolean;
exportDiagnosticsEnabled?: boolean;
matrix?: { [key: string]: string };
analysisKinds?: AnalysisKind[];
} = {},
): Promise<initActionPostHelper.UploadFailedSarifResult> {
const config = {
const config = createTestConfig({
analysisKinds,
codeQLCmd: "codeql",
debugMode: true,
languages: [],
packs: [],
} as unknown as configUtils.Config;
});
if (databaseExists) {
config.dbLocation = "path/to/database";
}

View File

@@ -7,7 +7,7 @@ import * as actionsUtil from "./actions-util";
import { CodeScanning } from "./analyses";
import { getApiClient } from "./api-client";
import { CodeQL, getCodeQL } from "./codeql";
import { Config } from "./config-utils";
import { Config, isCodeScanningEnabled } from "./config-utils";
import * as dependencyCaching from "./dependency-caching";
import { EnvVar } from "./environment";
import { Feature, FeatureEnablement } from "./feature-flags";
@@ -139,6 +139,15 @@ export async function tryUploadSarifIfRunFailed(
EnvVar.JOB_STATUS,
process.env[EnvVar.JOB_STATUS] ?? JobStatus.ConfigErrorStatus,
);
// If the only enabled analysis kind is `code-quality`, then we shouldn't
// upload the failed SARIF to Code Scanning.
if (!isCodeScanningEnabled(config)) {
return {
upload_failed_run_skipped_because: "Code Scanning is not enabled.",
};
}
try {
return await maybeUploadFailedSarif(
config,

View File

@@ -15,6 +15,7 @@ import {
getTemporaryDirectory,
persistInputs,
} from "./actions-util";
import { AnalysisKind, getAnalysisKinds } from "./analyses";
import { getGitHubVersion } from "./api-client";
import {
getDependencyCachingEnabled,
@@ -56,6 +57,7 @@ import { ToolsSource } from "./setup-codeql";
import {
ActionName,
InitStatusReport,
InitToolsDownloadFields,
InitWithConfigStatusReport,
createInitWithConfigStatusReport,
createStatusReportBase,
@@ -86,14 +88,29 @@ import {
} from "./util";
import { validateWorkflow } from "./workflow";
/** Fields of the init status report populated when the tools source is `download`. */
interface InitToolsDownloadFields {
/** Time taken to download the bundle, in milliseconds. */
tools_download_duration_ms?: number;
/**
* Whether the relevant tools dotcom feature flags have been misconfigured.
* Only populated if we attempt to determine the default version based on the dotcom feature flags. */
tools_feature_flags_valid?: boolean;
/**
* Sends a status report indicating that the `init` Action is starting.
*
* @param startedAt
* @param config
* @param logger
*/
async function sendStartingStatusReport(
startedAt: Date,
config: Partial<configUtils.Config> | undefined,
logger: Logger,
) {
const statusReportBase = await createStatusReportBase(
ActionName.Init,
"starting",
startedAt,
config,
await checkDiskUsage(logger),
logger,
);
if (statusReportBase !== undefined) {
await sendStatusReport(statusReportBase);
}
}
async function sendCompletedStatusReport(
@@ -210,6 +227,7 @@ async function run() {
? await loadPropertiesFromApi(gitHubVersion, logger, repositoryNwo)
: {};
// Create a unique identifier for this run.
const jobRunUuid = uuidV4();
logger.info(`Job run UUID is ${jobRunUuid}.`);
core.exportVariable(EnvVar.JOB_RUN_UUID, jobRunUuid);
@@ -227,17 +245,30 @@ async function run() {
);
try {
const statusReportBase = await createStatusReportBase(
ActionName.Init,
"starting",
startedAt,
config,
await checkDiskUsage(logger),
logger,
);
if (statusReportBase !== undefined) {
await sendStatusReport(statusReportBase);
// Parsing the `analysis-kinds` input may throw a `ConfigurationError`, which we don't want before
// we have called `sendStartingStatusReport` below. However, we want the analysis kinds for that status
// report. To work around this, we ignore exceptions that are thrown here and then call `getAnalysisKinds`
// a second time later. The second call will then throw the exception again. If `getAnalysisKinds` is
// successful, the results are cached so that we don't duplicate the work in normal runs.
let analysisKinds: AnalysisKind[] | undefined;
try {
analysisKinds = await getAnalysisKinds(logger);
} catch (err) {
logger.debug(
`Failed to parse analysis kinds for 'starting' status report: ${getErrorMessage(err)}`,
);
}
// Send a status report indicating that an analysis is starting.
await sendStartingStatusReport(startedAt, { analysisKinds }, logger);
// Throw a `ConfigurationError` if the `setup-codeql` action has been run.
if (process.env[EnvVar.SETUP_CODEQL_ACTION_HAS_RUN] === "true") {
throw new ConfigurationError(
`The 'init' action should not be run in the same workflow as 'setup-codeql'.`,
);
}
const codeQLDefaultVersionInfo = await features.getDefaultCliVersion(
gitHubVersion.type,
);
@@ -293,21 +324,11 @@ async function run() {
}
}
// Warn that `quality-queries` is deprecated if there is an argument for it.
const qualityQueriesInput = getOptionalInput("quality-queries");
if (qualityQueriesInput !== undefined) {
logger.warning(
"The `quality-queries` input is deprecated and will be removed in a future version of the CodeQL Action. " +
"Use the `analysis-kinds` input to configure different analysis kinds instead.",
);
}
config = await initConfig({
analysisKindsInput: getRequiredInput("analysis-kinds"),
analysisKinds = await getAnalysisKinds(logger);
config = await initConfig(features, {
analysisKinds,
languagesInput: getOptionalInput("languages"),
queriesInput: getOptionalInput("queries"),
qualityQueriesInput,
packsInput: getOptionalInput("packs"),
buildModeInput: getOptionalInput("build-mode"),
configFile,

View File

@@ -61,10 +61,11 @@ export async function initCodeQL(
}
export async function initConfig(
features: FeatureEnablement,
inputs: configUtils.InitConfigInputs,
): Promise<configUtils.Config> {
return await withGroupAsync("Load language configuration", async () => {
return await configUtils.initConfig(inputs);
return await configUtils.initConfig(features, inputs);
});
}

196
src/setup-codeql-action.ts Normal file
View File

@@ -0,0 +1,196 @@
import * as core from "@actions/core";
import { v4 as uuidV4 } from "uuid";
import {
getActionVersion,
getOptionalInput,
getRequiredInput,
getTemporaryDirectory,
} from "./actions-util";
import { getGitHubVersion } from "./api-client";
import { CodeQL } from "./codeql";
import { EnvVar } from "./environment";
import { Features } from "./feature-flags";
import { initCodeQL } from "./init";
import { getActionsLogger, Logger } from "./logging";
import { getRepositoryNwo } from "./repository";
import { ToolsSource } from "./setup-codeql";
import {
ActionName,
InitStatusReport,
InitToolsDownloadFields,
createStatusReportBase,
getActionsStatus,
sendStatusReport,
} from "./status-report";
import { ToolsDownloadStatusReport } from "./tools-download";
import {
checkDiskUsage,
checkForTimeout,
checkGitHubVersionInRange,
getRequiredEnvParam,
initializeEnvironment,
ConfigurationError,
wrapError,
checkActionVersion,
getErrorMessage,
} from "./util";
/**
* Helper function to send a full status report for this action.
*/
async function sendCompletedStatusReport(
startedAt: Date,
toolsDownloadStatusReport: ToolsDownloadStatusReport | undefined,
toolsFeatureFlagsValid: boolean | undefined,
toolsSource: ToolsSource,
toolsVersion: string,
logger: Logger,
error?: Error,
): Promise<void> {
const statusReportBase = await createStatusReportBase(
ActionName.SetupCodeQL,
getActionsStatus(error),
startedAt,
undefined,
await checkDiskUsage(logger),
logger,
error?.message,
error?.stack,
);
if (statusReportBase === undefined) {
return;
}
const initStatusReport: InitStatusReport = {
...statusReportBase,
tools_input: getOptionalInput("tools") || "",
tools_resolved_version: toolsVersion,
tools_source: toolsSource || ToolsSource.Unknown,
workflow_languages: "",
};
const initToolsDownloadFields: InitToolsDownloadFields = {};
if (toolsDownloadStatusReport?.downloadDurationMs !== undefined) {
initToolsDownloadFields.tools_download_duration_ms =
toolsDownloadStatusReport.downloadDurationMs;
}
if (toolsFeatureFlagsValid !== undefined) {
initToolsDownloadFields.tools_feature_flags_valid = toolsFeatureFlagsValid;
}
await sendStatusReport({ ...initStatusReport, ...initToolsDownloadFields });
}
/** The main behaviour of this action. */
async function run(): Promise<void> {
const startedAt = new Date();
const logger = getActionsLogger();
initializeEnvironment(getActionVersion());
let codeql: CodeQL;
let toolsDownloadStatusReport: ToolsDownloadStatusReport | undefined;
let toolsFeatureFlagsValid: boolean | undefined;
let toolsSource: ToolsSource;
let toolsVersion: string;
const apiDetails = {
auth: getRequiredInput("token"),
externalRepoAuth: getOptionalInput("external-repository-token"),
url: getRequiredEnvParam("GITHUB_SERVER_URL"),
apiURL: getRequiredEnvParam("GITHUB_API_URL"),
};
const gitHubVersion = await getGitHubVersion();
checkGitHubVersionInRange(gitHubVersion, logger);
checkActionVersion(getActionVersion(), gitHubVersion);
const repositoryNwo = getRepositoryNwo();
const features = new Features(
gitHubVersion,
repositoryNwo,
getTemporaryDirectory(),
logger,
);
const jobRunUuid = uuidV4();
logger.info(`Job run UUID is ${jobRunUuid}.`);
core.exportVariable(EnvVar.JOB_RUN_UUID, jobRunUuid);
try {
const statusReportBase = await createStatusReportBase(
ActionName.SetupCodeQL,
"starting",
startedAt,
undefined,
await checkDiskUsage(logger),
logger,
);
if (statusReportBase !== undefined) {
await sendStatusReport(statusReportBase);
}
const codeQLDefaultVersionInfo = await features.getDefaultCliVersion(
gitHubVersion.type,
);
toolsFeatureFlagsValid = codeQLDefaultVersionInfo.toolsFeatureFlagsValid;
const initCodeQLResult = await initCodeQL(
getOptionalInput("tools"),
apiDetails,
getTemporaryDirectory(),
gitHubVersion.type,
codeQLDefaultVersionInfo,
features,
logger,
);
codeql = initCodeQLResult.codeql;
toolsDownloadStatusReport = initCodeQLResult.toolsDownloadStatusReport;
toolsVersion = initCodeQLResult.toolsVersion;
toolsSource = initCodeQLResult.toolsSource;
core.setOutput("codeql-path", codeql.getPath());
core.setOutput("codeql-version", (await codeql.getVersion()).version);
core.exportVariable(EnvVar.SETUP_CODEQL_ACTION_HAS_RUN, "true");
} catch (unwrappedError) {
const error = wrapError(unwrappedError);
core.setFailed(error.message);
const statusReportBase = await createStatusReportBase(
ActionName.SetupCodeQL,
error instanceof ConfigurationError ? "user-error" : "failure",
startedAt,
undefined,
await checkDiskUsage(logger),
logger,
error.message,
error.stack,
);
if (statusReportBase !== undefined) {
await sendStatusReport(statusReportBase);
}
return;
}
await sendCompletedStatusReport(
startedAt,
toolsDownloadStatusReport,
toolsFeatureFlagsValid,
toolsSource,
toolsVersion,
logger,
);
}
/** Run the action and catch any unhandled errors. */
async function runWrapper(): Promise<void> {
try {
await run();
} catch (error) {
core.setFailed(`setup-codeql action failed: ${getErrorMessage(error)}`);
}
await checkForTimeout();
}
void runWrapper();

View File

@@ -23,7 +23,6 @@ import { getRepositoryNwo } from "./repository";
import { ToolsSource } from "./setup-codeql";
import {
ConfigurationError,
isHTTPError,
getRequiredEnvParam,
getCachedCodeQlVersion,
isInTestMode,
@@ -33,6 +32,7 @@ import {
BuildMode,
getErrorMessage,
getTestingEnvironment,
asHTTPError,
} from "./util";
export enum ActionName {
@@ -41,6 +41,7 @@ export enum ActionName {
Init = "init",
InitPost = "init-post",
ResolveEnvironment = "resolve-environment",
SetupCodeQL = "setup-codeql",
StartProxy = "start-proxy",
UploadSarif = "upload-sarif",
}
@@ -428,8 +429,9 @@ export async function sendStatusReport<S extends StatusReportBase>(
},
);
} catch (e) {
if (isHTTPError(e)) {
switch (e.status) {
const httpError = asHTTPError(e);
if (httpError !== undefined) {
switch (httpError.status) {
case 403:
if (
getWorkflowEventName() === "push" &&
@@ -442,11 +444,11 @@ export async function sendStatusReport<S extends StatusReportBase>(
`See ${DocUrl.SCANNING_ON_PUSH} for more information on how to configure these events.`,
);
} else {
core.warning(e.message);
core.warning(httpError.message);
}
return;
case 404:
core.warning(e.message);
core.warning(httpError.message);
return;
case 422:
// schema incompatibility when reporting status
@@ -516,6 +518,16 @@ export interface InitWithConfigStatusReport extends InitStatusReport {
config_file: string;
}
/** Fields of the init status report populated when the tools source is `download`. */
export interface InitToolsDownloadFields {
/** Time taken to download the bundle, in milliseconds. */
tools_download_duration_ms?: number;
/**
* Whether the relevant tools dotcom feature flags have been misconfigured.
* Only populated if we attempt to determine the default version based on the dotcom feature flags. */
tools_feature_flags_valid?: boolean;
}
/**
* Composes a `InitWithConfigStatusReport` from the given values.
*

View File

@@ -2,7 +2,7 @@ import { TextDecoder } from "node:util";
import path from "path";
import * as github from "@actions/github";
import { TestFn } from "ava";
import { ExecutionContext, TestFn } from "ava";
import nock from "nock";
import * as sinon from "sinon";
@@ -180,6 +180,23 @@ export function getRecordingLogger(messages: LoggedMessage[]): Logger {
};
}
export function checkExpectedLogMessages(
t: ExecutionContext<any>,
messages: LoggedMessage[],
expectedMessages: string[],
) {
for (const expectedMessage of expectedMessages) {
t.assert(
messages.some(
(msg) =>
typeof msg.message === "string" &&
msg.message.includes(expectedMessage),
),
`Expected '${expectedMessage}' in the logger output, but didn't find it in:\n ${messages.map((m) => ` - '${m.message}'`).join("\n")}`,
);
}
}
/** Mock the HTTP request to the feature flags enablement API endpoint. */
export function mockFeatureFlagApiEndpoint(
responseStatusCode: number,

View File

@@ -13,8 +13,8 @@ import * as gitUtils from "./git-utils";
import { Language } from "./languages";
import { Logger } from "./logging";
import {
asHTTPError,
getErrorMessage,
isHTTPError,
tryGetFolderBytes,
waitForResultWithTimeLimit,
} from "./util";
@@ -236,7 +236,7 @@ export async function cleanupTrapCaches(
}
return { trap_cache_cleanup_size_bytes: totalBytesCleanedUp };
} catch (e) {
if (isHTTPError(e) && e.status === 403) {
if (asHTTPError(e)?.status === 403) {
logger.warning(
"Could not cleanup TRAP caches as the token did not have the required permissions. " +
'To clean up TRAP caches, ensure the token has the "actions:write" permission. ' +

View File

@@ -386,16 +386,17 @@ export async function uploadPayload(
logger.info("Successfully uploaded results");
return response.data.id as string;
} catch (e) {
if (util.isHTTPError(e)) {
switch (e.status) {
const httpError = util.asHTTPError(e);
if (httpError !== undefined) {
switch (httpError.status) {
case 403:
core.warning(e.message || GENERIC_403_MSG);
core.warning(httpError.message || GENERIC_403_MSG);
break;
case 404:
core.warning(e.message || GENERIC_404_MSG);
core.warning(httpError.message || GENERIC_404_MSG);
break;
default:
core.warning(e.message);
core.warning(httpError.message);
break;
}
}

View File

@@ -692,8 +692,22 @@ export class ConfigurationError extends Error {
}
}
export function isHTTPError(arg: any): arg is HTTPError {
return arg?.status !== undefined && Number.isInteger(arg.status);
export function asHTTPError(arg: any): HTTPError | undefined {
if (
typeof arg !== "object" ||
arg === null ||
typeof arg.message !== "string"
) {
return undefined;
}
if (Number.isInteger(arg.status)) {
return new HTTPError(arg.message as string, arg.status as number);
}
// See https://github.com/actions/toolkit/blob/acb230b99a46ed33a3f04a758cd68b47b9a82908/packages/tool-cache/src/tool-cache.ts#L19
if (Number.isInteger(arg.httpStatusCode)) {
return new HTTPError(arg.message as string, arg.httpStatusCode as number);
}
return undefined;
}
let cachedCodeQlVersion: undefined | VersionInfo = undefined;