github/codeql-action
1
0
Fork 0
mirror of https://github.com/github/codeql-action.git synced 2025-12-23 15:50:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

Refactor: Pull out generic tool invocation functionality

Browse Source
This commit is contained in:
Henry Mercer
2024-10-01 14:39:04 +01:00
parent cf5b0a9041
commit 1aa7f6f05d
15 changed files with 256 additions and 177 deletions
Download Patch File Download Diff File Expand all files Collapse all files

78
lib/actions-util.js generated
View File

@@ -23,7 +23,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
return result;
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.getFileType = exports.FileCmdNotFoundError = exports.determineMergeBaseCommitOid = exports.getCommitOid = exports.getOptionalInput = exports.getRequiredInput = void 0;
exports.CommandInvocationError = exports.getFileType = exports.FileCmdNotFoundError = exports.determineMergeBaseCommitOid = exports.getCommitOid = exports.getOptionalInput = exports.getRequiredInput = void 0;
exports.getTemporaryDirectory = getTemporaryDirectory;
exports.getRef = getRef;
exports.getActionVersion = getActionVersion;
@@ -37,6 +37,9 @@ exports.getUploadValue = getUploadValue;
exports.getWorkflowRunID = getWorkflowRunID;
exports.getWorkflowRunAttempt = getWorkflowRunAttempt;
exports.isSelfHostedRunner = isSelfHostedRunner;
exports.prettyPrintInvocation = prettyPrintInvocation;
exports.ensureEndsInPeriod = ensureEndsInPeriod;
exports.runTool = runTool;
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const core = __importStar(require("@actions/core"));
@@ -429,4 +432,77 @@ exports.getFileType = getFileType;
function isSelfHostedRunner() {
return process.env.RUNNER_ENVIRONMENT === "self-hosted";
}
function prettyPrintInvocation(cmd, args) {
return [cmd, ...args].map((x) => (x.includes(" ") ? `'${x}'` : x)).join(" ");
}
/**
* An error from a tool invocation, with associated exit code, stderr, etc.
*/
class CommandInvocationError extends Error {
constructor(cmd, args, exitCode, stderr, stdout) {
const prettyCommand = prettyPrintInvocation(cmd, args);
const lastLine = ensureEndsInPeriod(stderr.trim().split("\n").pop()?.trim() || "n/a");
super(`Failed to run "${prettyCommand}". ` +
`Exit code was ${exitCode} and last log line was: ${lastLine} See the logs for more details.`);
this.cmd = cmd;
this.args = args;
this.exitCode = exitCode;
this.stderr = stderr;
this.stdout = stdout;
}
}
exports.CommandInvocationError = CommandInvocationError;
function ensureEndsInPeriod(text) {
return text[text.length - 1] === "." ? text : `${text}.`;
}
/**
* A constant defining the maximum number of characters we will keep from
* the programs stderr for logging.
*
* This serves two purposes:
* 1. It avoids an OOM if a program fails in a way that results it
* printing many log lines.
* 2. It avoids us hitting the limit of how much data we can send in our
* status reports on GitHub.com.
*/
const MAX_STDERR_BUFFER_SIZE = 20000;
/**
* Runs a CLI tool.
*
* @returns Standard output produced by the tool.
* @throws A `CommandInvocationError` if the tool exits with a non-zero status code.
*/
async function runTool(cmd, args = [], opts = {}) {
let stdout = "";
let stderr = "";
process.stdout.write(`[command]${cmd} ${args.join(" ")}\n`);
const exitCode = await new toolrunner.ToolRunner(cmd, args, {
ignoreReturnCode: true,
listeners: {
stdout: (data) => {
stdout += data.toString("utf8");
if (!opts.noStreamStdout) {
process.stdout.write(data);
}
},
stderr: (data) => {
let readStartIndex = 0;
// If the error is too large, then we only take the last MAX_STDERR_BUFFER_SIZE characters
if (data.length - MAX_STDERR_BUFFER_SIZE > 0) {
// Eg: if we have MAX_STDERR_BUFFER_SIZE the start index should be 2.
readStartIndex = data.length - MAX_STDERR_BUFFER_SIZE + 1;
}
stderr += data.toString("utf8", readStartIndex);
// Mimic the standard behavior of the toolrunner by writing stderr to stdout
process.stdout.write(data);
},
},
silent: true,
...(opts.stdin ? { input: Buffer.from(opts.stdin || "") } : {}),
}).exec();
if (exitCode !== 0) {
throw new CommandInvocationError(cmd, args, exitCode, stderr, stdout);
}
return stdout;
}
//# sourceMappingURL=actions-util.js.map

2
lib/actions-util.js.map
View File

File diff suppressed because one or more lines are too long

31
lib/cli-errors.js generated
View File

@@ -1,26 +1,24 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.cliErrorsConfig = exports.CliConfigErrorCategory = exports.CommandInvocationError = void 0;
exports.cliErrorsConfig = exports.CliConfigErrorCategory = exports.CliError = void 0;
exports.getCliConfigCategoryIfExists = getCliConfigCategoryIfExists;
exports.wrapCliConfigurationError = wrapCliConfigurationError;
const actions_util_1 = require("./actions-util");
const doc_url_1 = require("./doc-url");
const util_1 = require("./util");
/**
* A class of Error that we can classify as an error stemming from a CLI
* invocation, with associated exit code, stderr,etc.
* An error from a CodeQL CLI invocation, with associated exit code, stderr, etc.
*/
class CommandInvocationError extends Error {
constructor(cmd, args, exitCode, stderr, stdout) {
const prettyCommand = [cmd, ...args]
.map((x) => (x.includes(" ") ? `'${x}'` : x))
.join(" ");
class CliError extends Error {
constructor({ cmd, args, exitCode, stderr }) {
const prettyCommand = (0, actions_util_1.prettyPrintInvocation)(cmd, args);
const fatalErrors = extractFatalErrors(stderr);
const autobuildErrors = extractAutobuildErrors(stderr);
let message;
if (fatalErrors) {
message =
`Encountered a fatal error while running "${prettyCommand}". ` +
`Exit code was ${exitCode} and error was: ${ensureEndsInPeriod(fatalErrors.trim())} See the logs for more details.`;
`Exit code was ${exitCode} and error was: ${(0, actions_util_1.ensureEndsInPeriod)(fatalErrors.trim())} See the logs for more details.`;
}
else if (autobuildErrors) {
message =
@@ -29,7 +27,7 @@ class CommandInvocationError extends Error {
`Encountered the following error: ${autobuildErrors}`;
}
else {
const lastLine = ensureEndsInPeriod(stderr.trim().split("\n").pop()?.trim() || "n/a");
const lastLine = (0, actions_util_1.ensureEndsInPeriod)(stderr.trim().split("\n").pop()?.trim() || "n/a");
message =
`Encountered a fatal error while running "${prettyCommand}". ` +
`Exit code was ${exitCode} and last log line was: ${lastLine} See the logs for more details.`;
@@ -37,10 +35,9 @@ class CommandInvocationError extends Error {
super(message);
this.exitCode = exitCode;
this.stderr = stderr;
this.stdout = stdout;
}
}
exports.CommandInvocationError = CommandInvocationError;
exports.CliError = CliError;
/**
* Provide a better error message from the stderr of a CLI invocation that failed with a fatal
* error.
@@ -89,10 +86,10 @@ function extractFatalErrors(error) {
}
const isOneLiner = !fatalErrors.some((e) => e.includes("\n"));
if (isOneLiner) {
fatalErrors = fatalErrors.map(ensureEndsInPeriod);
fatalErrors = fatalErrors.map(actions_util_1.ensureEndsInPeriod);
}
return [
ensureEndsInPeriod(lastError),
(0, actions_util_1.ensureEndsInPeriod)(lastError),
"Context:",
...fatalErrors.reverse(),
].join(isOneLiner ? " " : "\n");
@@ -109,9 +106,6 @@ function extractAutobuildErrors(error) {
}
return errorLines.join("\n") || undefined;
}
function ensureEndsInPeriod(text) {
return text[text.length - 1] === "." ? text : `${text}.`;
}
/** Error messages from the CLI that we consider configuration errors and handle specially. */
var CliConfigErrorCategory;
(function (CliConfigErrorCategory) {
@@ -267,9 +261,6 @@ function getCliConfigCategoryIfExists(cliError) {
* simply returns the original error.
*/
function wrapCliConfigurationError(cliError) {
if (!(cliError instanceof CommandInvocationError)) {
return cliError;
}
const cliConfigErrorCategory = getCliConfigCategoryIfExists(cliError);
if (cliConfigErrorCategory === undefined) {
return cliError;

2
lib/cli-errors.js.map
View File

File diff suppressed because one or more lines are too long

80
lib/codeql.js generated
View File

@@ -241,7 +241,7 @@ async function getCodeQLForCmd(cmd, checkVersion) {
async getVersion() {
let result = util.getCachedCodeQlVersion();
if (result === undefined) {
const output = await runTool(cmd, ["version", "--format=json"]);
const output = await runCli(cmd, ["version", "--format=json"]);
try {
result = JSON.parse(output);
}
@@ -253,7 +253,7 @@ async function getCodeQLForCmd(cmd, checkVersion) {
return result;
},
async printVersion() {
await runTool(cmd, ["version", "--format=json"]);
await runCli(cmd, ["version", "--format=json"]);
},
async supportsFeature(feature) {
return (0, tools_features_1.isSupportedToolsFeature)(await this.getVersion(), feature);
@@ -290,7 +290,7 @@ async function getCodeQLForCmd(cmd, checkVersion) {
const overwriteFlag = (0, tools_features_1.isSupportedToolsFeature)(await this.getVersion(), tools_features_1.ToolsFeature.ForceOverwrite)
? "--force-overwrite"
: "--overwrite";
await runTool(cmd, [
await runCli(cmd, [
"database",
"init",
overwriteFlag,
@@ -325,10 +325,10 @@ async function getCodeQLForCmd(cmd, checkVersion) {
// When `DYLD_INSERT_LIBRARIES` is set in the environment for a step,
// the Actions runtime introduces its own workaround for SIP
// (https://github.com/actions/runner/pull/416).
await runTool(autobuildCmd);
await runCli(autobuildCmd);
},
async extractScannedLanguage(config, language) {
await runTool(cmd, [
await runCli(cmd, [
"database",
"trace-command",
"--index-traceless-dbs",
@@ -343,7 +343,7 @@ async function getCodeQLForCmd(cmd, checkVersion) {
applyAutobuildAzurePipelinesTimeoutFix();
}
try {
await runTool(cmd, [
await runCli(cmd, [
"database",
"trace-command",
"--use-build-mode",
@@ -378,7 +378,7 @@ async function getCodeQLForCmd(cmd, checkVersion) {
...getExtraOptionsFromEnv(["database", "finalize"]),
databasePath,
];
await runTool(cmd, args);
await runCli(cmd, args);
},
async resolveLanguages() {
const codeqlArgs = [
@@ -387,7 +387,7 @@ async function getCodeQLForCmd(cmd, checkVersion) {
"--format=json",
...getExtraOptionsFromEnv(["resolve", "languages"]),
];
const output = await runTool(cmd, codeqlArgs);
const output = await runCli(cmd, codeqlArgs);
try {
return JSON.parse(output);
}
@@ -404,7 +404,7 @@ async function getCodeQLForCmd(cmd, checkVersion) {
...(await getLanguageAliasingArguments(this)),
...getExtraOptionsFromEnv(["resolve", "languages"]),
];
const output = await runTool(cmd, codeqlArgs);
const output = await runCli(cmd, codeqlArgs);
try {
return JSON.parse(output);
}
@@ -423,7 +423,7 @@ async function getCodeQLForCmd(cmd, checkVersion) {
if (extraSearchPath !== undefined) {
codeqlArgs.push("--additional-packs", extraSearchPath);
}
const output = await runTool(cmd, codeqlArgs);
const output = await runCli(cmd, codeqlArgs);
try {
return JSON.parse(output);
}
@@ -442,7 +442,7 @@ async function getCodeQLForCmd(cmd, checkVersion) {
if (workingDir !== undefined) {
codeqlArgs.push("--working-dir", workingDir);
}
const output = await runTool(cmd, codeqlArgs);
const output = await runCli(cmd, codeqlArgs);
try {
return JSON.parse(output);
}
@@ -466,7 +466,7 @@ async function getCodeQLForCmd(cmd, checkVersion) {
if (await util.codeQlVersionAtLeast(this, feature_flags_1.CODEQL_VERSION_FINE_GRAINED_PARALLELISM)) {
codeqlArgs.push("--intra-layer-parallelism");
}
await runTool(cmd, codeqlArgs);
await runCli(cmd, codeqlArgs);
},
async databaseInterpretResults(databasePath, querySuitePaths, sarifFile, addSnippetsFlag, threadsFlag, verbosityFlag, automationDetailsId, config, features) {
const shouldExportDiagnostics = await features.getValue(feature_flags_1.Feature.ExportDiagnosticsEnabled, this);
@@ -512,7 +512,7 @@ async function getCodeQLForCmd(cmd, checkVersion) {
}
// Capture the stdout, which contains the analysis summary. Don't stream it to the Actions
// logs to avoid printing it twice.
return await runTool(cmd, codeqlArgs, {
return await runCli(cmd, codeqlArgs, {
noStreamStdout: true,
});
},
@@ -523,7 +523,7 @@ async function getCodeQLForCmd(cmd, checkVersion) {
...getExtraOptionsFromEnv(["database", "print-baseline"]),
databasePath,
];
return await runTool(cmd, codeqlArgs);
return await runCli(cmd, codeqlArgs);
},
/**
* Download specified packs into the package cache. If the specified
@@ -551,7 +551,7 @@ async function getCodeQLForCmd(cmd, checkVersion) {
...getExtraOptionsFromEnv(["pack", "download"]),
...packs,
];
const output = await runTool(cmd, codeqlArgs);
const output = await runCli(cmd, codeqlArgs);
try {
const parsedOutput = JSON.parse(output);
if (Array.isArray(parsedOutput.packs) &&
@@ -580,7 +580,7 @@ async function getCodeQLForCmd(cmd, checkVersion) {
`${cacheCleanupFlag}=${cleanupLevel}`,
...getExtraOptionsFromEnv(["database", "cleanup"]),
];
await runTool(cmd, codeqlArgs);
await runCli(cmd, codeqlArgs);
},
async databaseBundle(databasePath, outputFilePath, databaseName) {
const args = [
@@ -662,7 +662,7 @@ async function getCodeQLForCmd(cmd, checkVersion) {
if (mergeRunsFromEqualCategory) {
args.push("--sarif-merge-runs-from-equal-category");
}
await runTool(cmd, args);
await runCli(cmd, args);
},
};
// To ensure that status reports include the CodeQL CLI version wherever
@@ -742,48 +742,16 @@ function getExtraOptions(options, paths, pathInfo) {
: getExtraOptions(options?.[paths[0]], paths?.slice(1), pathInfo.concat(paths[0]));
return all.concat(specific);
}
/*
* A constant defining the maximum number of characters we will keep from
* the programs stderr for logging. This serves two purposes:
* (1) It avoids an OOM if a program fails in a way that results it
* printing many log lines.
* (2) It avoids us hitting the limit of how much data we can send in our
* status reports on GitHub.com.
*/
const maxErrorSize = 20_000;
async function runTool(cmd, args = [], opts = {}) {
let stdout = "";
let stderr = "";
process.stdout.write(`[command]${cmd} ${args.join(" ")}\n`);
const exitCode = await new toolrunner.ToolRunner(cmd, args, {
ignoreReturnCode: true,
listeners: {
stdout: (data) => {
stdout += data.toString("utf8");
if (!opts.noStreamStdout) {
process.stdout.write(data);
async function runCli(cmd, args = [], opts = {}) {
try {
return await (0, actions_util_1.runTool)(cmd, args, opts);
}
},
stderr: (data) => {
let readStartIndex = 0;
// If the error is too large, then we only take the last 20,000 characters
if (data.length - maxErrorSize > 0) {
// Eg: if we have 20,000 the start index should be 2.
readStartIndex = data.length - maxErrorSize + 1;
catch (e) {
if (e instanceof actions_util_1.CommandInvocationError) {
throw (0, cli_errors_1.wrapCliConfigurationError)(new cli_errors_1.CliError(e));
}
stderr += data.toString("utf8", readStartIndex);
// Mimic the standard behavior of the toolrunner by writing stderr to stdout
process.stdout.write(data);
},
},
silent: true,
...(opts.stdin ? { input: Buffer.from(opts.stdin || "") } : {}),
}).exec();
if (exitCode !== 0) {
const e = new cli_errors_1.CommandInvocationError(cmd, args, exitCode, stderr, stdout);
throw (0, cli_errors_1.wrapCliConfigurationError)(e);
throw e;
}
return stdout;
}
/**
* Generates a code scanning configuration that is to be used for a scan.

2
lib/codeql.js.map
View File

File diff suppressed because one or more lines are too long

2
lib/codeql.test.js generated
View File

@@ -591,7 +591,7 @@ for (const { codeqlVersion, flagPassed, githubVersion, negativeFlagPassed, } of
// safeWhich throws because of the test CodeQL object.
sinon.stub(safeWhich, "safeWhich").resolves("");
await t.throwsAsync(async () => await codeqlObject.databaseRunQueries(stubConfig.dbLocation, []), {
instanceOf: cli_errors_1.CommandInvocationError,
instanceOf: cli_errors_1.CliError,
message: `Encountered a fatal error while running "codeql-for-testing database run-queries --expect-discarded-cache --min-disk-free=1024 -v --intra-layer-parallelism". Exit code was 1 and error was: Oops! A fatal internal error occurred. Details:
com.semmle.util.exception.CatastrophicError: An error occurred while evaluating ControlFlowGraph::ControlFlow::Root.isRootOf/1#dispred#f610e6ed/2@86282cc8
Severe disk cache trouble (corruption or out of space) at /home/runner/work/_temp/codeql_databases/go/db-go/default/cache/pages/28/33.pack: Failed to write item to disk. See the logs for more details.`,

2
lib/codeql.test.js.map
View File

File diff suppressed because one or more lines are too long

2
lib/resolve-environment-action.js generated
View File

@@ -55,7 +55,7 @@ async function run() {
}
catch (unwrappedError) {
const error = (0, util_1.wrapError)(unwrappedError);
if (error instanceof cli_errors_1.CommandInvocationError) {
if (error instanceof cli_errors_1.CliError) {
// If the CLI failed to run successfully for whatever reason,
// we just return an empty JSON object and proceed with the workflow.
core.setOutput(ENVIRONMENT_OUTPUT_NAME, {});

2
lib/resolve-environment-action.js.map
View File

@@ -1 +1 @@
{"version":3,"file":"resolve-environment-action.js","sourceRoot":"","sources":["../src/resolve-environment-action.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AAAA,oDAAsC;AAEtC,iDAKwB;AACxB,6CAAgD;AAChD,6CAAsD;AACtD,iDAAmD;AACnD,uCAA6C;AAC7C,+DAAmE;AACnE,mDAKyB;AACzB,iCAOgB;AAEhB,MAAM,uBAAuB,GAAG,aAAa,CAAC;AAE9C,KAAK,UAAU,GAAG;IAChB,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;IAC7B,MAAM,MAAM,GAAG,IAAA,0BAAgB,GAAE,CAAC;IAElC,IAAI,MAA0B,CAAC;IAE/B,IAAI,CAAC;QACH,MAAM,gBAAgB,GAAG,MAAM,IAAA,sCAAsB,EACnD,0BAAU,CAAC,kBAAkB,EAC7B,UAAU,EACV,SAAS,EACT,MAAM,EACN,MAAM,IAAA,qBAAc,EAAC,MAAM,CAAC,EAC5B,MAAM,CACP,CAAC;QACF,IAAI,gBAAgB,KAAK,SAAS,EAAE,CAAC;YACnC,MAAM,IAAA,gCAAgB,EAAC,gBAAgB,CAAC,CAAC;QAC3C,CAAC;QAED,MAAM,aAAa,GAAG,MAAM,IAAA,6BAAgB,GAAE,CAAC;QAC/C,IAAA,gCAAyB,EAAC,aAAa,EAAE,MAAM,CAAC,CAAC;QACjD,IAAA,yBAAkB,EAAC,IAAA,+BAAgB,GAAE,EAAE,aAAa,CAAC,CAAC;QAEtD,MAAM,GAAG,MAAM,IAAA,wBAAS,EAAC,IAAA,oCAAqB,GAAE,EAAE,MAAM,CAAC,CAAC;QAC1D,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CACb,yFAAyF,CAC1F,CAAC;QACJ,CAAC;QAED,MAAM,gBAAgB,GAAG,IAAA,+BAAgB,EAAC,mBAAmB,CAAC,CAAC;QAC/D,MAAM,MAAM,GAAG,MAAM,IAAA,gDAA0B,EAC7C,MAAM,CAAC,SAAS,EAChB,MAAM,EACN,gBAAgB,EAChB,IAAA,+BAAgB,EAAC,UAAU,CAAC,CAC7B,CAAC;QACF,IAAI,CAAC,SAAS,CAAC,uBAAuB,EAAE,MAAM,CAAC,CAAC;IAClD,CAAC;IAAC,OAAO,cAAc,EAAE,CAAC;QACxB,MAAM,KAAK,GAAG,IAAA,gBAAS,EAAC,cAAc,CAAC,CAAC;QAExC,IAAI,KAAK,YAAY,mCAAsB,EAAE,CAAC;YAC5C,6DAA6D;YAC7D,qEAAqE;YACrE,IAAI,CAAC,SAAS,CAAC,uBAAuB,EAAE,EAAE,CAAC,CAAC;YAC5C,MAAM,CAAC,OAAO,CACZ,wFAAwF,KAAK,CAAC,OAAO,EAAE,CACxG,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,kFAAkF;YAClF,IAAI,CAAC,SAAS,CACZ,wFAAwF,KAAK,CAAC,OAAO,EAAE,CACxG,CAAC;YAEF,MAAM,gBAAgB,GAAG,MAAM,IAAA,sCAAsB,EACnD,0BAAU,CAAC,kBAAkB,EAC7B,IAAA,gCAAgB,EAAC,KAAK,CAAC,EACvB,SAAS,EACT,MAAM,EACN,MAAM,IAAA,qBAAc,EAAC,MAAM,CAAC,EAC5B,MAAM,EACN,KAAK,CAAC,OAAO,EACb,KAAK,CAAC,KAAK,CACZ,CAAC;YACF,IAAI,gBAAgB,KAAK,SAAS,EAAE,CAAC;gBACnC,MAAM,IAAA,gCAAgB,EAAC,gBAAgB,CAAC,CAAC;YAC3C,CAAC;QACH,CAAC;QAED,OAAO;IACT,CAAC;IAED,MAAM,gBAAgB,GAAG,MAAM,IAAA,sCAAsB,EACnD,0BAAU,CAAC,kBAAkB,EAC7B,SAAS,EACT,SAAS,EACT,MAAM,EACN,MAAM,IAAA,qBAAc,EAAC,MAAM,CAAC,EAC5B,MAAM,CACP,CAAC;IACF,IAAI,gBAAgB,KAAK,SAAS,EAAE,CAAC;QACnC,MAAM,IAAA,gCAAgB,EAAC,gBAAgB,CAAC,CAAC;IAC3C,CAAC;AACH,CAAC;AAED,KAAK,UAAU,UAAU;IACvB,IAAI,CAAC;QACH,MAAM,GAAG,EAAE,CAAC;IACd,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,CAAC,SAAS,CACZ,GAAG,0BAAU,CAAC,kBAAkB,mBAAmB,IAAA,sBAAe,EAChE,KAAK,CACN,EAAE,CACJ,CAAC;IACJ,CAAC;IACD,MAAM,IAAA,sBAAe,GAAE,CAAC;AAC1B,CAAC;AAED,KAAK,UAAU,EAAE,CAAC"}
{"version":3,"file":"resolve-environment-action.js","sourceRoot":"","sources":["../src/resolve-environment-action.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AAAA,oDAAsC;AAEtC,iDAKwB;AACxB,6CAAgD;AAChD,6CAAwC;AACxC,iDAAmD;AACnD,uCAA6C;AAC7C,+DAAmE;AACnE,mDAKyB;AACzB,iCAOgB;AAEhB,MAAM,uBAAuB,GAAG,aAAa,CAAC;AAE9C,KAAK,UAAU,GAAG;IAChB,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;IAC7B,MAAM,MAAM,GAAG,IAAA,0BAAgB,GAAE,CAAC;IAElC,IAAI,MAA0B,CAAC;IAE/B,IAAI,CAAC;QACH,MAAM,gBAAgB,GAAG,MAAM,IAAA,sCAAsB,EACnD,0BAAU,CAAC,kBAAkB,EAC7B,UAAU,EACV,SAAS,EACT,MAAM,EACN,MAAM,IAAA,qBAAc,EAAC,MAAM,CAAC,EAC5B,MAAM,CACP,CAAC;QACF,IAAI,gBAAgB,KAAK,SAAS,EAAE,CAAC;YACnC,MAAM,IAAA,gCAAgB,EAAC,gBAAgB,CAAC,CAAC;QAC3C,CAAC;QAED,MAAM,aAAa,GAAG,MAAM,IAAA,6BAAgB,GAAE,CAAC;QAC/C,IAAA,gCAAyB,EAAC,aAAa,EAAE,MAAM,CAAC,CAAC;QACjD,IAAA,yBAAkB,EAAC,IAAA,+BAAgB,GAAE,EAAE,aAAa,CAAC,CAAC;QAEtD,MAAM,GAAG,MAAM,IAAA,wBAAS,EAAC,IAAA,oCAAqB,GAAE,EAAE,MAAM,CAAC,CAAC;QAC1D,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CACb,yFAAyF,CAC1F,CAAC;QACJ,CAAC;QAED,MAAM,gBAAgB,GAAG,IAAA,+BAAgB,EAAC,mBAAmB,CAAC,CAAC;QAC/D,MAAM,MAAM,GAAG,MAAM,IAAA,gDAA0B,EAC7C,MAAM,CAAC,SAAS,EAChB,MAAM,EACN,gBAAgB,EAChB,IAAA,+BAAgB,EAAC,UAAU,CAAC,CAC7B,CAAC;QACF,IAAI,CAAC,SAAS,CAAC,uBAAuB,EAAE,MAAM,CAAC,CAAC;IAClD,CAAC;IAAC,OAAO,cAAc,EAAE,CAAC;QACxB,MAAM,KAAK,GAAG,IAAA,gBAAS,EAAC,cAAc,CAAC,CAAC;QAExC,IAAI,KAAK,YAAY,qBAAQ,EAAE,CAAC;YAC9B,6DAA6D;YAC7D,qEAAqE;YACrE,IAAI,CAAC,SAAS,CAAC,uBAAuB,EAAE,EAAE,CAAC,CAAC;YAC5C,MAAM,CAAC,OAAO,CACZ,wFAAwF,KAAK,CAAC,OAAO,EAAE,CACxG,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,kFAAkF;YAClF,IAAI,CAAC,SAAS,CACZ,wFAAwF,KAAK,CAAC,OAAO,EAAE,CACxG,CAAC;YAEF,MAAM,gBAAgB,GAAG,MAAM,IAAA,sCAAsB,EACnD,0BAAU,CAAC,kBAAkB,EAC7B,IAAA,gCAAgB,EAAC,KAAK,CAAC,EACvB,SAAS,EACT,MAAM,EACN,MAAM,IAAA,qBAAc,EAAC,MAAM,CAAC,EAC5B,MAAM,EACN,KAAK,CAAC,OAAO,EACb,KAAK,CAAC,KAAK,CACZ,CAAC;YACF,IAAI,gBAAgB,KAAK,SAAS,EAAE,CAAC;gBACnC,MAAM,IAAA,gCAAgB,EAAC,gBAAgB,CAAC,CAAC;YAC3C,CAAC;QACH,CAAC;QAED,OAAO;IACT,CAAC;IAED,MAAM,gBAAgB,GAAG,MAAM,IAAA,sCAAsB,EACnD,0BAAU,CAAC,kBAAkB,EAC7B,SAAS,EACT,SAAS,EACT,MAAM,EACN,MAAM,IAAA,qBAAc,EAAC,MAAM,CAAC,EAC5B,MAAM,CACP,CAAC;IACF,IAAI,gBAAgB,KAAK,SAAS,EAAE,CAAC;QACnC,MAAM,IAAA,gCAAgB,EAAC,gBAAgB,CAAC,CAAC;IAC3C,CAAC;AACH,CAAC;AAED,KAAK,UAAU,UAAU;IACvB,IAAI,CAAC;QACH,MAAM,GAAG,EAAE,CAAC;IACd,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,CAAC,SAAS,CACZ,GAAG,0BAAU,CAAC,kBAAkB,mBAAmB,IAAA,sBAAe,EAChE,KAAK,CACN,EAAE,CACJ,CAAC;IACJ,CAAC;IACD,MAAM,IAAA,sBAAe,GAAE,CAAC;AAC1B,CAAC;AAED,KAAK,UAAU,EAAE,CAAC"}

86
src/actions-util.ts
View File

@@ -480,3 +480,89 @@ export const getFileType = async (filePath: string): Promise<string> => {
export function isSelfHostedRunner() {
return process.env.RUNNER_ENVIRONMENT === "self-hosted";
}
export function prettyPrintInvocation(cmd: string, args: string[]): string {
return [cmd, ...args].map((x) => (x.includes(" ") ? `'${x}'` : x)).join(" ");
}
/**
* An error from a tool invocation, with associated exit code, stderr, etc.
*/
export class CommandInvocationError extends Error {
constructor(
public cmd: string,
public args: string[],
public exitCode: number,
public stderr: string,
public stdout: string,
) {
const prettyCommand = prettyPrintInvocation(cmd, args);
const lastLine = ensureEndsInPeriod(
stderr.trim().split("\n").pop()?.trim() || "n/a",
);
super(
`Failed to run "${prettyCommand}". ` +
`Exit code was ${exitCode} and last log line was: ${lastLine} See the logs for more details.`,
);
}
}
export function ensureEndsInPeriod(text: string): string {
return text[text.length - 1] === "." ? text : `${text}.`;
}
/**
* A constant defining the maximum number of characters we will keep from
* the programs stderr for logging.
*
* This serves two purposes:
* 1. It avoids an OOM if a program fails in a way that results it
* printing many log lines.
* 2. It avoids us hitting the limit of how much data we can send in our
* status reports on GitHub.com.
*/
const MAX_STDERR_BUFFER_SIZE = 20000;
/**
* Runs a CLI tool.
*
* @returns Standard output produced by the tool.
* @throws A `CommandInvocationError` if the tool exits with a non-zero status code.
*/
export async function runTool(
cmd: string,
args: string[] = [],
opts: { stdin?: string; noStreamStdout?: boolean } = {},
): Promise<string> {
let stdout = "";
let stderr = "";
process.stdout.write(`[command]${cmd} ${args.join(" ")}\n`);
const exitCode = await new toolrunner.ToolRunner(cmd, args, {
ignoreReturnCode: true,
listeners: {
stdout: (data: Buffer) => {
stdout += data.toString("utf8");
if (!opts.noStreamStdout) {
process.stdout.write(data);
}
},
stderr: (data: Buffer) => {
let readStartIndex = 0;
// If the error is too large, then we only take the last MAX_STDERR_BUFFER_SIZE characters
if (data.length - MAX_STDERR_BUFFER_SIZE > 0) {
// Eg: if we have MAX_STDERR_BUFFER_SIZE the start index should be 2.
readStartIndex = data.length - MAX_STDERR_BUFFER_SIZE + 1;
}
stderr += data.toString("utf8", readStartIndex);
// Mimic the standard behavior of the toolrunner by writing stderr to stdout
process.stdout.write(data);
},
},
silent: true,
...(opts.stdin ? { input: Buffer.from(opts.stdin || "") } : {}),
}).exec();
if (exitCode !== 0) {
throw new CommandInvocationError(cmd, args, exitCode, stderr, stdout);
}
return stdout;
}

39
src/cli-errors.ts
View File

@@ -1,21 +1,20 @@
import {
CommandInvocationError,
ensureEndsInPeriod,
prettyPrintInvocation,
} from "./actions-util";
import { DocUrl } from "./doc-url";
import { ConfigurationError } from "./util";
/**
* A class of Error that we can classify as an error stemming from a CLI
* invocation, with associated exit code, stderr,etc.
* An error from a CodeQL CLI invocation, with associated exit code, stderr, etc.
*/
export class CommandInvocationError extends Error {
constructor(
cmd: string,
args: string[],
public exitCode: number,
public stderr: string,
public stdout: string,
) {
const prettyCommand = [cmd, ...args]
.map((x) => (x.includes(" ") ? `'${x}'` : x))
.join(" ");
export class CliError extends Error {
public readonly exitCode: number;
public readonly stderr: string;
constructor({ cmd, args, exitCode, stderr }: CommandInvocationError) {
const prettyCommand = prettyPrintInvocation(cmd, args);
const fatalErrors = extractFatalErrors(stderr);
const autobuildErrors = extractAutobuildErrors(stderr);
@@ -42,6 +41,8 @@ export class CommandInvocationError extends Error {
}
super(message);
this.exitCode = exitCode;
this.stderr = stderr;
}
}
@@ -115,10 +116,6 @@ function extractAutobuildErrors(error: string): string | undefined {
return errorLines.join("\n") || undefined;
}
function ensureEndsInPeriod(text: string): string {
return text[text.length - 1] === "." ? text : `${text}.`;
}
/** Error messages from the CLI that we consider configuration errors and handle specially. */
export enum CliConfigErrorCategory {
AutobuildError = "AutobuildError",
@@ -282,7 +279,7 @@ export const cliErrorsConfig: Record<
* if not, return undefined.
*/
export function getCliConfigCategoryIfExists(
cliError: CommandInvocationError,
cliError: CliError,
): CliConfigErrorCategory | undefined {
for (const [category, configuration] of Object.entries(cliErrorsConfig)) {
if (
@@ -308,11 +305,7 @@ export function getCliConfigCategoryIfExists(
* error message appended, if it exists in a known set of configuration errors. Otherwise,
* simply returns the original error.
*/
export function wrapCliConfigurationError(cliError: Error): Error {
if (!(cliError instanceof CommandInvocationError)) {
return cliError;
}
export function wrapCliConfigurationError(cliError: CliError): Error {
const cliConfigErrorCategory = getCliConfigCategoryIfExists(cliError);
if (cliConfigErrorCategory === undefined) {
return cliError;

4
src/codeql.test.ts
View File

@@ -12,7 +12,7 @@ import * as sinon from "sinon";
import * as actionsUtil from "./actions-util";
import { GitHubApiDetails } from "./api-client";
import { CommandInvocationError } from "./cli-errors";
import { CliError } from "./cli-errors";
import * as codeql from "./codeql";
import { AugmentationProperties, Config } from "./config-utils";
import * as defaults from "./defaults.json";
@@ -961,7 +961,7 @@ test("runTool recognizes fatal internal errors", async (t) => {
async () =>
await codeqlObject.databaseRunQueries(stubConfig.dbLocation, []),
{
instanceOf: CommandInvocationError,
instanceOf: CliError,
message: `Encountered a fatal error while running "codeql-for-testing database run-queries --expect-discarded-cache --min-disk-free=1024 -v --intra-layer-parallelism". Exit code was 1 and error was: Oops! A fatal internal error occurred. Details:
com.semmle.util.exception.CatastrophicError: An error occurred while evaluating ControlFlowGraph::ControlFlow::Root.isRootOf/1#dispred#f610e6ed/2@86282cc8
Severe disk cache trouble (corruption or out of space) at /home/runner/work/_temp/codeql_databases/go/db-go/default/cache/pages/28/33.pack: Failed to write item to disk. See the logs for more details.`,

91
src/codeql.ts
View File

@@ -7,15 +7,14 @@ import * as yaml from "js-yaml";
import * as semver from "semver";
import {
CommandInvocationError,
getActionVersion,
getOptionalInput,
isAnalyzingDefaultBranch,
runTool,
} from "./actions-util";
import * as api from "./api-client";
import {
CommandInvocationError,
wrapCliConfigurationError,
} from "./cli-errors";
import { CliError, wrapCliConfigurationError } from "./cli-errors";
import { type Config } from "./config-utils";
import { DocUrl } from "./doc-url";
import { EnvVar } from "./environment";
@@ -544,7 +543,7 @@ export async function getCodeQLForCmd(
async getVersion() {
let result = util.getCachedCodeQlVersion();
if (result === undefined) {
const output = await runTool(cmd, ["version", "--format=json"]);
const output = await runCli(cmd, ["version", "--format=json"]);
try {
result = JSON.parse(output) as VersionInfo;
} catch {
@@ -557,7 +556,7 @@ export async function getCodeQLForCmd(
return result;
},
async printVersion() {
await runTool(cmd, ["version", "--format=json"]);
await runCli(cmd, ["version", "--format=json"]);
},
async supportsFeature(feature: ToolsFeature) {
return isSupportedToolsFeature(await this.getVersion(), feature);
@@ -627,7 +626,7 @@ export async function getCodeQLForCmd(
? "--force-overwrite"
: "--overwrite";
await runTool(
await runCli(
cmd,
[
"database",
@@ -674,10 +673,10 @@ export async function getCodeQLForCmd(
// When `DYLD_INSERT_LIBRARIES` is set in the environment for a step,
// the Actions runtime introduces its own workaround for SIP
// (https://github.com/actions/runner/pull/416).
await runTool(autobuildCmd);
await runCli(autobuildCmd);
},
async extractScannedLanguage(config: Config, language: Language) {
await runTool(cmd, [
await runCli(cmd, [
"database",
"trace-command",
"--index-traceless-dbs",
@@ -692,7 +691,7 @@ export async function getCodeQLForCmd(
applyAutobuildAzurePipelinesTimeoutFix();
}
try {
await runTool(cmd, [
await runCli(cmd, [
"database",
"trace-command",
"--use-build-mode",
@@ -731,7 +730,7 @@ export async function getCodeQLForCmd(
...getExtraOptionsFromEnv(["database", "finalize"]),
databasePath,
];
await runTool(cmd, args);
await runCli(cmd, args);
},
async resolveLanguages() {
const codeqlArgs = [
@@ -740,7 +739,7 @@ export async function getCodeQLForCmd(
"--format=json",
...getExtraOptionsFromEnv(["resolve", "languages"]),
];
const output = await runTool(cmd, codeqlArgs);
const output = await runCli(cmd, codeqlArgs);
try {
return JSON.parse(output) as ResolveLanguagesOutput;
@@ -759,7 +758,7 @@ export async function getCodeQLForCmd(
...(await getLanguageAliasingArguments(this)),
...getExtraOptionsFromEnv(["resolve", "languages"]),
];
const output = await runTool(cmd, codeqlArgs);
const output = await runCli(cmd, codeqlArgs);
try {
return JSON.parse(output) as BetterResolveLanguagesOutput;
@@ -783,7 +782,7 @@ export async function getCodeQLForCmd(
if (extraSearchPath !== undefined) {
codeqlArgs.push("--additional-packs", extraSearchPath);
}
const output = await runTool(cmd, codeqlArgs);
const output = await runCli(cmd, codeqlArgs);
try {
return JSON.parse(output) as ResolveQueriesOutput;
@@ -805,7 +804,7 @@ export async function getCodeQLForCmd(
if (workingDir !== undefined) {
codeqlArgs.push("--working-dir", workingDir);
}
const output = await runTool(cmd, codeqlArgs);
const output = await runCli(cmd, codeqlArgs);
try {
return JSON.parse(output) as ResolveBuildEnvironmentOutput;
@@ -839,7 +838,7 @@ export async function getCodeQLForCmd(
) {
codeqlArgs.push("--intra-layer-parallelism");
}
await runTool(cmd, codeqlArgs);
await runCli(cmd, codeqlArgs);
},
async databaseInterpretResults(
databasePath: string,
@@ -911,7 +910,7 @@ export async function getCodeQLForCmd(
}
// Capture the stdout, which contains the analysis summary. Don't stream it to the Actions
// logs to avoid printing it twice.
return await runTool(cmd, codeqlArgs, {
return await runCli(cmd, codeqlArgs, {
noStreamStdout: true,
});
},
@@ -922,7 +921,7 @@ export async function getCodeQLForCmd(
...getExtraOptionsFromEnv(["database", "print-baseline"]),
databasePath,
];
return await runTool(cmd, codeqlArgs);
return await runCli(cmd, codeqlArgs);
},
/**
@@ -956,7 +955,7 @@ export async function getCodeQLForCmd(
...packs,
];
const output = await runTool(cmd, codeqlArgs);
const output = await runCli(cmd, codeqlArgs);
try {
const parsedOutput: PackDownloadOutput = JSON.parse(output);
@@ -994,7 +993,7 @@ export async function getCodeQLForCmd(
`${cacheCleanupFlag}=${cleanupLevel}`,
...getExtraOptionsFromEnv(["database", "cleanup"]),
];
await runTool(cmd, codeqlArgs);
await runCli(cmd, codeqlArgs);
},
async databaseBundle(
databasePath: string,
@@ -1103,7 +1102,7 @@ export async function getCodeQLForCmd(
args.push("--sarif-merge-runs-from-equal-category");
}
await runTool(cmd, args);
await runCli(cmd, args);
},
};
// To ensure that status reports include the CodeQL CLI version wherever
@@ -1216,53 +1215,19 @@ export function getExtraOptions(
return all.concat(specific);
}
/*
* A constant defining the maximum number of characters we will keep from
* the programs stderr for logging. This serves two purposes:
* (1) It avoids an OOM if a program fails in a way that results it
* printing many log lines.
* (2) It avoids us hitting the limit of how much data we can send in our
* status reports on GitHub.com.
*/
const maxErrorSize = 20_000;
async function runTool(
async function runCli(
cmd: string,
args: string[] = [],
opts: { stdin?: string; noStreamStdout?: boolean } = {},
) {
let stdout = "";
let stderr = "";
process.stdout.write(`[command]${cmd} ${args.join(" ")}\n`);
const exitCode = await new toolrunner.ToolRunner(cmd, args, {
ignoreReturnCode: true,
listeners: {
stdout: (data: Buffer) => {
stdout += data.toString("utf8");
if (!opts.noStreamStdout) {
process.stdout.write(data);
): Promise<string> {
try {
return await runTool(cmd, args, opts);
} catch (e) {
if (e instanceof CommandInvocationError) {
throw wrapCliConfigurationError(new CliError(e));
}
},
stderr: (data: Buffer) => {
let readStartIndex = 0;
// If the error is too large, then we only take the last 20,000 characters
if (data.length - maxErrorSize > 0) {
// Eg: if we have 20,000 the start index should be 2.
readStartIndex = data.length - maxErrorSize + 1;
throw e;
}
stderr += data.toString("utf8", readStartIndex);
// Mimic the standard behavior of the toolrunner by writing stderr to stdout
process.stdout.write(data);
},
},
silent: true,
...(opts.stdin ? { input: Buffer.from(opts.stdin || "") } : {}),
}).exec();
if (exitCode !== 0) {
const e = new CommandInvocationError(cmd, args, exitCode, stderr, stdout);
throw wrapCliConfigurationError(e);
}
return stdout;
}
/**

4
src/resolve-environment-action.ts
View File

@@ -7,7 +7,7 @@ import {
getTemporaryDirectory,
} from "./actions-util";
import { getGitHubVersion } from "./api-client";
import { CommandInvocationError } from "./cli-errors";
import { CliError } from "./cli-errors";
import { Config, getConfig } from "./config-utils";
import { getActionsLogger } from "./logging";
import { runResolveBuildEnvironment } from "./resolve-environment";
@@ -69,7 +69,7 @@ async function run() {
} catch (unwrappedError) {
const error = wrapError(unwrappedError);
if (error instanceof CommandInvocationError) {
if (error instanceof CliError) {
// If the CLI failed to run successfully for whatever reason,
// we just return an empty JSON object and proceed with the workflow.
core.setOutput(ENVIRONMENT_OUTPUT_NAME, {});