Merge branch 'main' into update-bundle/codeql-bundle-v2.15.0

This commit is contained in:
Henry Mercer
2023-10-10 17:20:48 +01:00
committed by GitHub
105 changed files with 3145 additions and 901 deletions

View File

@@ -425,3 +425,39 @@ export function getWorkflowRunAttempt(): number {
}
return workflowRunAttempt;
}
/**
* Tries to obtain the output of the `file` command for the file at the specified path.
* The output will vary depending on the type of `file`, which operating system we are running on, etc.
*/
export const getFileType = async (filePath: string): Promise<string> => {
let stderr = "";
let stdout = "";
try {
// The `file` command will output information about the type of file pointed at by `filePath`.
// For binary files, this may include e.g. whether they are static of dynamic binaries.
// The `-L` switch instructs the command to follow symbolic links.
await new toolrunner.ToolRunner(
await safeWhich.safeWhich("file"),
["-L", filePath],
{
silent: true,
listeners: {
stdout: (data) => {
stdout += data.toString();
},
stderr: (data) => {
stderr += data.toString();
},
},
},
).exec();
return stdout.trim();
} catch (e) {
core.info(
`Could not determine type of ${filePath} from ${stdout}. ${stderr}`,
);
throw e;
}
};

View File

@@ -3,6 +3,7 @@ import path from "path";
import { performance } from "perf_hooks";
import * as core from "@actions/core";
import { safeWhich } from "@chrisgavin/safe-which";
import * as actionsUtil from "./actions-util";
import {
@@ -18,6 +19,7 @@ import { runAutobuild } from "./autobuild";
import { getCodeQL } from "./codeql";
import { Config, getConfig } from "./config-utils";
import { uploadDatabases } from "./database-upload";
import { addDiagnostic, makeDiagnostic } from "./diagnostics";
import { EnvVar } from "./environment";
import { Features } from "./feature-flags";
import { Language } from "./languages";
@@ -231,6 +233,44 @@ async function run() {
logger,
);
// Check that `which go` still points at the wrapper script we installed in the `init` Action,
// if the corresponding environment variable is set. This is to ensure that there isn't a step
// in the workflow after the `init` step which installs a different version of Go and takes
// precedence in the PATH, thus potentially circumventing our workaround that allows tracing to work.
const goWrapperPath = process.env[EnvVar.GO_BINARY_LOCATION];
if (
process.env[EnvVar.DID_AUTOBUILD_GOLANG] !== "true" &&
goWrapperPath !== undefined
) {
const goBinaryPath = await safeWhich("go");
if (goWrapperPath !== goBinaryPath) {
core.warning(
`Expected \`which go\` to return ${goWrapperPath}, but got ${goBinaryPath}: please ensure that the correct version of Go is installed before the \`codeql-action/init\` Action is used.`,
);
addDiagnostic(
config,
Language.go,
makeDiagnostic(
"go/workflow/go-installed-after-codeql-init",
"Go was installed after the `codeql-action/init` Action was run",
{
markdownMessage:
"To avoid interfering with the CodeQL analysis, perform all installation steps before calling the `github/codeql-action/init` Action.",
visibility: {
statusPage: true,
telemetry: true,
cliSummaryTable: true,
},
severity: "warning",
},
),
);
}
}
await runAutobuildIfLegacyGoWorkflow(config, logger);
dbCreationTimings = await runFinalize(

View File

@@ -394,14 +394,10 @@ export async function runQueries(
await runPrintLinesOfCode(language);
}
} catch (e) {
logger.info(String(e));
if (e instanceof Error) {
logger.info(e.stack!);
}
statusReport.analyze_failure_language = language;
throw new CodeQLAnalysisError(
statusReport,
`Error running analysis for ${language}: ${e}`,
`Error running analysis for ${language}: ${util.wrapError(e).message}`,
);
}
}

View File

@@ -8,7 +8,6 @@ import {
import { getGitHubVersion } from "./api-client";
import { determineAutobuildLanguages, runAutobuild } from "./autobuild";
import * as configUtils from "./config-utils";
import { EnvVar } from "./environment";
import { Language } from "./languages";
import { Logger, getActionsLogger } from "./logging";
import {
@@ -98,9 +97,6 @@ async function run() {
for (const language of languages) {
currentLanguage = language;
await runAutobuild(language, config, logger);
if (language === Language.go) {
core.exportVariable(EnvVar.DID_AUTOBUILD_GOLANG, "true");
}
}
}
} catch (unwrappedError) {

View File

@@ -4,6 +4,7 @@ import { getTemporaryDirectory, getWorkflowEventName } from "./actions-util";
import { getGitHubVersion } from "./api-client";
import { CodeQL, getCodeQL } from "./codeql";
import * as configUtils from "./config-utils";
import { EnvVar } from "./environment";
import { Feature, featureConfig, Features } from "./feature-flags";
import { isTracedLanguage, Language } from "./languages";
import { Logger } from "./logging";
@@ -150,5 +151,8 @@ export async function runAutobuild(
await setupCppAutobuild(codeQL, logger);
}
await codeQL.runAutobuild(language);
if (language === Language.go) {
core.exportVariable(EnvVar.DID_AUTOBUILD_GOLANG, "true");
}
logger.endGroup();
}

View File

@@ -1133,7 +1133,7 @@ test("database finalize does not override no code found error on CodeQL 2.12.4",
{
message:
'Encountered a fatal error while running "codeql-for-testing database finalize --finalize-dataset --threads=2 --ram=2048 db". ' +
`Exit code was 32 and error was: ${cliMessage}`,
`Exit code was 32 and last log line was: ${cliMessage} See the logs for more details.`,
},
);
});
@@ -1158,7 +1158,26 @@ test("runTool summarizes several fatal errors", async (t) => {
{
message:
'Encountered a fatal error while running "codeql-for-testing database finalize --finalize-dataset --threads=2 --ram=2048 db". ' +
`Exit code was 32 and error was: ${datasetImportError}. Context: ${heapError}.`,
`Exit code was 32 and error was: ${datasetImportError}. Context: ${heapError}. See the logs for more details.`,
},
);
});
test("runTool outputs last line of stderr if fatal error could not be found", async (t) => {
const cliStderr = "line1\nline2\nline3\nline4\nline5";
stubToolRunnerConstructor(32, cliStderr);
const codeqlObject = await codeql.getCodeQLForTesting();
sinon.stub(codeqlObject, "getVersion").resolves(makeVersionInfo("2.12.4"));
// safeWhich throws because of the test CodeQL object.
sinon.stub(safeWhich, "safeWhich").resolves("");
await t.throwsAsync(
async () =>
await codeqlObject.finalizeDatabase("db", "--threads=2", "--ram=2048"),
{
message:
'Encountered a fatal error while running "codeql-for-testing database finalize --finalize-dataset --threads=2 --ram=2048 db". ' +
"Exit code was 32 and last log line was: line5. See the logs for more details.",
},
);
});

View File

@@ -54,15 +54,27 @@ export class CommandInvocationError extends Error {
cmd: string,
args: string[],
public exitCode: number,
public error: string,
public output: string,
public stderr: string,
public stdout: string,
) {
const prettyCommand = [cmd, ...args]
.map((x) => (x.includes(" ") ? `'${x}'` : x))
.join(" ");
const fatalErrors = extractFatalErrors(stderr);
const lastLine = stderr.trim().split("\n").pop()?.trim();
let error = fatalErrors
? ` and error was: ${fatalErrors.trim()}`
: lastLine
? ` and last log line was: ${lastLine}`
: "";
if (error[error.length - 1] !== ".") {
error += ".";
}
super(
`Encountered a fatal error while running "${prettyCommand}". ` +
`Exit code was ${exitCode} and error was: ${error.trim()}`,
`Exit code was ${exitCode}${error} See the logs for more details.`,
);
}
}
@@ -128,7 +140,7 @@ export interface CodeQL {
*/
resolveBuildEnvironment(
workingDir: string | undefined,
language: Language,
language: string,
): Promise<ResolveBuildEnvironmentOutput>;
/**
@@ -624,6 +636,7 @@ export async function getCodeQLForCmd(
"--db-cluster",
config.dbLocation,
`--source-root=${sourceRoot}`,
...(await getLanguageAliasingArguments(this)),
...extraArgs,
...getExtraOptionsFromEnv(["database", "init"]),
],
@@ -737,20 +750,12 @@ export async function getCodeQLForCmd(
}
},
async betterResolveLanguages() {
const extraArgs: string[] = [];
if (
await util.codeQlVersionAbove(this, CODEQL_VERSION_LANGUAGE_ALIASING)
) {
extraArgs.push("--extractor-include-aliases");
}
const codeqlArgs = [
"resolve",
"languages",
"--format=betterjson",
"--extractor-options-verbosity=4",
...extraArgs,
...(await getLanguageAliasingArguments(this)),
...getExtraOptionsFromEnv(["resolve", "languages"]),
];
const output = await runTool(cmd, codeqlArgs);
@@ -787,12 +792,13 @@ export async function getCodeQLForCmd(
},
async resolveBuildEnvironment(
workingDir: string | undefined,
language: Language,
language: string,
) {
const codeqlArgs = [
"resolve",
"build-environment",
`--language=${language}`,
...(await getLanguageAliasingArguments(this)),
...getExtraOptionsFromEnv(["resolve", "build-environment"]),
];
if (workingDir !== undefined) {
@@ -1082,6 +1088,7 @@ export async function getCodeQLForCmd(
"extractor",
"--format=json",
`--language=${language}`,
...(await getLanguageAliasingArguments(this)),
...getExtraOptionsFromEnv(["resolve", "extractor"]),
],
{
@@ -1243,7 +1250,6 @@ async function runTool(
...(opts.stdin ? { input: Buffer.from(opts.stdin || "") } : {}),
}).exec();
if (exitCode !== 0) {
error = extractFatalErrors(error) || error;
throw new CommandInvocationError(cmd, args, exitCode, error, output);
}
return output;
@@ -1459,7 +1465,7 @@ function isNoCodeFoundError(e: CommandInvocationError): boolean {
*/
const javascriptNoCodeFoundWarning =
"No JavaScript or TypeScript code found.";
return e.exitCode === 32 || e.error.includes(javascriptNoCodeFoundWarning);
return e.exitCode === 32 || e.stderr.includes(javascriptNoCodeFoundWarning);
}
async function isDiagnosticsExportInvalidSarifFixed(
@@ -1470,3 +1476,10 @@ async function isDiagnosticsExportInvalidSarifFixed(
CODEQL_VERSION_DIAGNOSTICS_EXPORT_FIXED,
);
}
async function getLanguageAliasingArguments(codeql: CodeQL): Promise<string[]> {
if (await util.codeQlVersionAbove(codeql, CODEQL_VERSION_LANGUAGE_ALIASING)) {
return ["--extractor-include-aliases"];
}
return [];
}

109
src/diagnostics.ts Normal file
View File

@@ -0,0 +1,109 @@
import { mkdirSync, writeFileSync } from "fs";
import path from "path";
import { Config } from "./config-utils";
import { Language } from "./languages";
import { getActionsLogger } from "./logging";
import { getCodeQLDatabasePath } from "./util";
/** Represents a diagnostic message for the tool status page, etc. */
export interface DiagnosticMessage {
/** ISO 8601 timestamp */
timestamp: string;
source: {
/**
* An identifier under which it makes sense to group this diagnostic message.
* This is used to build the SARIF reporting descriptor object.
*/
id: string;
/** Display name for the ID. This is used to build the SARIF reporting descriptor object. */
name: string;
/**
* Name of the CodeQL extractor. This is used to identify which tool component the reporting
* descriptor object should be nested under in SARIF.
*/
extractorName?: string;
};
/** GitHub flavored Markdown formatted message. Should include inline links to any help pages. */
markdownMessage?: string;
/** Plain text message. Used by components where the string processing needed to support Markdown is cumbersome. */
plaintextMessage?: string;
/** List of help links intended to supplement the `plaintextMessage`. */
helpLinks?: string[];
/** SARIF severity */
severity?: "error" | "warning" | "note";
visibility?: {
/** True if the message should be displayed on the status page (defaults to false) */
statusPage?: boolean;
/**
* True if the message should be counted in the diagnostics summary table printed by `codeql database analyze`
* (defaults to false)
*/
cliSummaryTable?: boolean;
/** True if the message should be sent to telemetry (defaults to false) */
telemetry?: boolean;
};
location?: {
/** Path to the affected file if appropriate, relative to the source root */
file?: string;
startLine?: number;
startColumn?: number;
endLine?: number;
endColumn?: number;
};
/** Structured metadata about the diagnostic message */
attributes?: { [key: string]: any };
}
/**
* Constructs a new diagnostic message with the specified id and name, as well as optional additional data.
*
* @param id An identifier under which it makes sense to group this diagnostic message.
* @param name Display name for the ID.
* @param data Optional additional data to initialize the diagnostic with.
* @returns Returns the new diagnostic message.
*/
export function makeDiagnostic(
id: string,
name: string,
data: Partial<DiagnosticMessage> | undefined = undefined,
): DiagnosticMessage {
return {
...data,
timestamp: data?.timestamp ?? new Date().toISOString(),
source: { ...data?.source, id, name },
};
}
/**
* Writes the given diagnostic to the database.
*
* @param config The configuration that tells us where to store the diagnostic.
* @param diagnostic The diagnostic message to add to the database.
*/
export function addDiagnostic(
config: Config,
language: Language,
diagnostic: DiagnosticMessage,
) {
const logger = getActionsLogger();
const diagnosticsPath = path.resolve(
getCodeQLDatabasePath(config, language),
"diagnostic",
"codeql-action",
);
try {
// Create the directory if it doesn't exist yet.
mkdirSync(diagnosticsPath, { recursive: true });
const jsonPath = path.resolve(
diagnosticsPath,
`codeql-action-${diagnostic.timestamp}.json`,
);
writeFileSync(jsonPath, JSON.stringify(diagnostic));
} catch (err) {
logger.warning(`Unable to write diagnostic message to database: ${err}`);
}
}

View File

@@ -62,4 +62,11 @@ export enum EnvVar {
* rather that the init action.
*/
WORKFLOW_STARTED_AT = "CODEQL_WORKFLOW_STARTED_AT",
/**
* The path where we initially discovered the Go binary in the system path
* before replacing it with a wrapper script. We check this later to ensure
* that it hasn't been tampered with by a late e.g. `setup-go` step.
*/
GO_BINARY_LOCATION = "CODEQL_ACTION_GO_BINARY",
}

View File

@@ -1,10 +1,13 @@
import * as fs from "fs";
import * as path from "path";
import * as core from "@actions/core";
import { safeWhich } from "@chrisgavin/safe-which";
import { v4 as uuidV4 } from "uuid";
import {
getActionVersion,
getFileType,
getOptionalInput,
getRequiredInput,
getTemporaryDirectory,
@@ -14,7 +17,13 @@ import { CodeQL } from "./codeql";
import * as configUtils from "./config-utils";
import { EnvVar } from "./environment";
import { Feature, Features } from "./feature-flags";
import { initCodeQL, initConfig, installPythonDeps, runInit } from "./init";
import {
checkInstallPython311,
initCodeQL,
initConfig,
installPythonDeps,
runInit,
} from "./init";
import { Language } from "./languages";
import { getActionsLogger, Logger } from "./logging";
import { parseRepositoryNwo } from "./repository";
@@ -25,6 +34,7 @@ import {
getActionsStatus,
sendStatusReport,
} from "./status-report";
import { ToolsFeature, isSupportedToolsFeature } from "./tools-features";
import { getTotalCacheSize } from "./trap-caching";
import {
checkDiskUsage,
@@ -273,6 +283,8 @@ async function run() {
logger,
);
await checkInstallPython311(config.languages, codeql);
if (
config.languages.includes(Language.python) &&
getRequiredInput("setup-python-dependencies") === "true"
@@ -312,6 +324,9 @@ async function run() {
}
try {
// Query CLI for supported features
const versionInfo = await codeql.getVersion();
// Forward Go flags
const goFlags = process.env["GOFLAGS"];
if (goFlags) {
@@ -321,6 +336,53 @@ async function run() {
);
}
// Go 1.21 and above ships with statically linked binaries on Linux. CodeQL cannot currently trace custom builds
// where the entry point is a statically linked binary. Until that is fixed, we work around the problem by
// replacing the `go` binary with a shell script that invokes the actual `go` binary. Since the shell is typically
// dynamically linked, this provides a suitable entry point for the CodeQL tracer.
if (
config.languages.includes(Language.go) &&
process.platform === "linux" &&
!isSupportedToolsFeature(
versionInfo,
ToolsFeature.IndirectTracingSupportsStaticBinaries,
)
) {
try {
const goBinaryPath = await safeWhich("go");
const fileOutput = await getFileType(goBinaryPath);
if (fileOutput.includes("statically linked")) {
logger.debug(`Applying static binary workaround for Go`);
// Create a directory that we can add to the system PATH.
const tempBinPath = path.resolve(
getTemporaryDirectory(),
"codeql-action-go-tracing",
"bin",
);
fs.mkdirSync(tempBinPath, { recursive: true });
core.addPath(tempBinPath);
// Write the wrapper script to the directory we just added to the PATH.
const goWrapperPath = path.resolve(tempBinPath, "go");
fs.writeFileSync(
goWrapperPath,
`#!/bin/bash\n\nexec ${goBinaryPath} "$@"`,
);
fs.chmodSync(goWrapperPath, "755");
// Store the original location of our wrapper script somewhere where we can
// later retrieve it from and cross-check that it hasn't been changed.
core.exportVariable(EnvVar.GO_BINARY_LOCATION, goWrapperPath);
}
} catch (e) {
logger.warning(
`Analyzing Go on Linux, but failed to install wrapper script. Tracing custom builds may fail: ${e}`,
);
}
}
// Limit RAM and threads for extractors. When running extractors, the CodeQL CLI obeys the
// CODEQL_RAM and CODEQL_THREADS environment variables to decide how much RAM and how many
// threads it would ask extractors to use. See help text for the "--ram" and "--threads"

View File

@@ -13,6 +13,7 @@ import {
FeatureEnablement,
useCodeScanningConfigInCli,
} from "./feature-flags";
import { Language } from "./languages";
import { Logger } from "./logging";
import { RepositoryNwo } from "./repository";
import { ToolsSource } from "./setup-codeql";
@@ -181,6 +182,30 @@ function processError(e: any): Error {
return e;
}
/**
* If we are running python 3.12+ on windows, we need to switch to python 3.11.
* This check happens in a powershell script.
*/
export async function checkInstallPython311(
languages: Language[],
codeql: CodeQL,
) {
if (
languages.includes(Language.python) &&
process.platform === "win32" &&
!(await codeql.getVersion()).features?.supportsPython312
) {
const script = path.resolve(
__dirname,
"../python-setup",
"check_python12.ps1",
);
await new toolrunner.ToolRunner(await safeWhich.safeWhich("powershell"), [
script,
]).exec();
}
}
export async function installPythonDeps(codeql: CodeQL, logger: Logger) {
logger.startGroup("Setup Python dependencies");

View File

@@ -8,7 +8,6 @@ import {
import { getGitHubVersion } from "./api-client";
import { CommandInvocationError } from "./codeql";
import * as configUtils from "./config-utils";
import { Language, parseLanguage } from "./languages";
import { getActionsLogger } from "./logging";
import { runResolveBuildEnvironment } from "./resolve-environment";
import {
@@ -44,16 +43,6 @@ async function run() {
return;
}
const language: Language | undefined = parseLanguage(
getRequiredInput("language"),
);
if (language === undefined) {
throw new Error(
`Did not recognize the language "${getRequiredInput("language")}".`,
);
}
const gitHubVersion = await getGitHubVersion();
checkGitHubVersionInRange(gitHubVersion, logger);
@@ -69,7 +58,7 @@ async function run() {
config.codeQLCmd,
logger,
workingDirectory,
language,
getRequiredInput("language"),
);
core.setOutput(ENVIRONMENT_OUTPUT_NAME, result);
} catch (unwrappedError) {

View File

@@ -1,5 +1,9 @@
import { CODEQL_VERSION_RESOLVE_ENVIRONMENT, getCodeQL } from "./codeql";
import { Language } from "./languages";
import {
CODEQL_VERSION_LANGUAGE_ALIASING,
CODEQL_VERSION_RESOLVE_ENVIRONMENT,
getCodeQL,
} from "./codeql";
import { parseLanguage } from "./languages";
import { Logger } from "./logging";
import * as util from "./util";
@@ -7,11 +11,27 @@ export async function runResolveBuildEnvironment(
cmd: string,
logger: Logger,
workingDir: string | undefined,
language: Language,
languageInput: string,
) {
logger.startGroup(`Attempting to resolve build environment for ${language}`);
logger.startGroup(
`Attempting to resolve build environment for ${languageInput}`,
);
const codeql = await getCodeQL(cmd);
let language = languageInput;
// If the CodeQL CLI version in use supports language aliasing, give the CLI the raw language
// input. Otherwise, parse the language input and give the CLI the parsed language.
if (
!(await util.codeQlVersionAbove(codeql, CODEQL_VERSION_LANGUAGE_ALIASING))
) {
const parsedLanguage = parseLanguage(languageInput)?.toString();
if (parsedLanguage === undefined) {
throw new Error(`Did not recognize the language '${languageInput}'.`);
}
language = parsedLanguage;
}
let result = {};
// If the CodeQL version in use does not support the `resolve build-environment`

View File

@@ -553,7 +553,7 @@ export async function downloadCodeQL(
logger.debug("Downloading CodeQL tools without an authorization token.");
}
logger.info(
`Downloading CodeQL tools from ${codeqlURL}. This may take a while.`,
`Downloading CodeQL tools from ${codeqlURL} . This may take a while.`,
);
const dest = path.join(tempDir, uuidV4());

View File

@@ -0,0 +1,24 @@
import test from "ava";
import { makeVersionInfo } from "./testing-utils";
import { ToolsFeature, isSupportedToolsFeature } from "./tools-features";
test("isSupportedToolsFeature", async (t) => {
const versionInfo = makeVersionInfo("1.0.0");
t.false(
isSupportedToolsFeature(
versionInfo,
ToolsFeature.IndirectTracingSupportsStaticBinaries,
),
);
versionInfo.features = { indirectTracingSupportsStaticBinaries: true };
t.true(
isSupportedToolsFeature(
versionInfo,
ToolsFeature.IndirectTracingSupportsStaticBinaries,
),
);
});

19
src/tools-features.ts Normal file
View File

@@ -0,0 +1,19 @@
import { VersionInfo } from "./codeql";
export enum ToolsFeature {
IndirectTracingSupportsStaticBinaries = "indirectTracingSupportsStaticBinaries",
}
/**
* Determines if the given feature is supported by the CLI.
*
* @param versionInfo Version information, including features, returned by the CLI.
* @param feature The feature to check for.
* @returns True if the feature is supported or false otherwise.
*/
export function isSupportedToolsFeature(
versionInfo: VersionInfo,
feature: ToolsFeature,
): boolean {
return !!versionInfo.features && versionInfo.features[feature];
}