mirror of
https://github.com/github/codeql-action.git
synced 2026-01-04 13:40:23 +08:00
Merge remote-tracking branch 'origin/main' into dbartol/actions-analysis
This commit is contained in:
@@ -324,3 +324,41 @@ test("determineBaseBranchHeadCommitOid other error", async (t) => {
|
||||
|
||||
infoStub.restore();
|
||||
});
|
||||
|
||||
test("decodeGitFilePath unquoted strings", async (t) => {
|
||||
t.deepEqual(actionsUtil.decodeGitFilePath("foo"), "foo");
|
||||
t.deepEqual(actionsUtil.decodeGitFilePath("foo bar"), "foo bar");
|
||||
t.deepEqual(actionsUtil.decodeGitFilePath("foo\\\\bar"), "foo\\\\bar");
|
||||
t.deepEqual(actionsUtil.decodeGitFilePath('foo\\"bar'), 'foo\\"bar');
|
||||
t.deepEqual(actionsUtil.decodeGitFilePath("foo\\001bar"), "foo\\001bar");
|
||||
t.deepEqual(actionsUtil.decodeGitFilePath("foo\\abar"), "foo\\abar");
|
||||
t.deepEqual(actionsUtil.decodeGitFilePath("foo\\bbar"), "foo\\bbar");
|
||||
t.deepEqual(actionsUtil.decodeGitFilePath("foo\\fbar"), "foo\\fbar");
|
||||
t.deepEqual(actionsUtil.decodeGitFilePath("foo\\nbar"), "foo\\nbar");
|
||||
t.deepEqual(actionsUtil.decodeGitFilePath("foo\\rbar"), "foo\\rbar");
|
||||
t.deepEqual(actionsUtil.decodeGitFilePath("foo\\tbar"), "foo\\tbar");
|
||||
t.deepEqual(actionsUtil.decodeGitFilePath("foo\\vbar"), "foo\\vbar");
|
||||
t.deepEqual(
|
||||
actionsUtil.decodeGitFilePath("\\a\\b\\f\\n\\r\\t\\v"),
|
||||
"\\a\\b\\f\\n\\r\\t\\v",
|
||||
);
|
||||
});
|
||||
|
||||
test("decodeGitFilePath quoted strings", async (t) => {
|
||||
t.deepEqual(actionsUtil.decodeGitFilePath('"foo"'), "foo");
|
||||
t.deepEqual(actionsUtil.decodeGitFilePath('"foo bar"'), "foo bar");
|
||||
t.deepEqual(actionsUtil.decodeGitFilePath('"foo\\\\bar"'), "foo\\bar");
|
||||
t.deepEqual(actionsUtil.decodeGitFilePath('"foo\\"bar"'), 'foo"bar');
|
||||
t.deepEqual(actionsUtil.decodeGitFilePath('"foo\\001bar"'), "foo\x01bar");
|
||||
t.deepEqual(actionsUtil.decodeGitFilePath('"foo\\abar"'), "foo\x07bar");
|
||||
t.deepEqual(actionsUtil.decodeGitFilePath('"foo\\bbar"'), "foo\bbar");
|
||||
t.deepEqual(actionsUtil.decodeGitFilePath('"foo\\fbar"'), "foo\fbar");
|
||||
t.deepEqual(actionsUtil.decodeGitFilePath('"foo\\nbar"'), "foo\nbar");
|
||||
t.deepEqual(actionsUtil.decodeGitFilePath('"foo\\rbar"'), "foo\rbar");
|
||||
t.deepEqual(actionsUtil.decodeGitFilePath('"foo\\tbar"'), "foo\tbar");
|
||||
t.deepEqual(actionsUtil.decodeGitFilePath('"foo\\vbar"'), "foo\vbar");
|
||||
t.deepEqual(
|
||||
actionsUtil.decodeGitFilePath('"\\a\\b\\f\\n\\r\\t\\v"'),
|
||||
"\x07\b\f\n\r\t\v",
|
||||
);
|
||||
});
|
||||
|
||||
@@ -56,6 +56,7 @@ async function runGitCommand(
|
||||
): Promise<string> {
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
core.debug(`Running git command: git ${args.join(" ")}`);
|
||||
try {
|
||||
await new toolrunner.ToolRunner(await safeWhich.safeWhich("git"), args, {
|
||||
silent: true,
|
||||
@@ -161,6 +162,159 @@ export const determineBaseBranchHeadCommitOid = async function (
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Deepen the git history of the given ref by one level. Errors are logged.
|
||||
*
|
||||
* This function uses the `checkout_path` to determine the repository path and
|
||||
* works only when called from `analyze` or `upload-sarif`.
|
||||
*/
|
||||
export const deepenGitHistory = async function () {
|
||||
try {
|
||||
await runGitCommand(
|
||||
getOptionalInput("checkout_path"),
|
||||
["fetch", "--no-tags", "--deepen=1"],
|
||||
"Cannot deepen the shallow repository.",
|
||||
);
|
||||
} catch {
|
||||
// Errors are already logged by runGitCommand()
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Fetch the given remote branch. Errors are logged.
|
||||
*
|
||||
* This function uses the `checkout_path` to determine the repository path and
|
||||
* works only when called from `analyze` or `upload-sarif`.
|
||||
*/
|
||||
export const gitFetch = async function (branch: string, extraFlags: string[]) {
|
||||
try {
|
||||
await runGitCommand(
|
||||
getOptionalInput("checkout_path"),
|
||||
["fetch", "--no-tags", ...extraFlags, "origin", `${branch}:${branch}`],
|
||||
`Cannot fetch ${branch}.`,
|
||||
);
|
||||
} catch {
|
||||
// Errors are already logged by runGitCommand()
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Compute the all merge bases between the given refs. Returns an empty array
|
||||
* if no merge base is found, or if there is an error.
|
||||
*
|
||||
* This function uses the `checkout_path` to determine the repository path and
|
||||
* works only when called from `analyze` or `upload-sarif`.
|
||||
*/
|
||||
export const getAllGitMergeBases = async function (
|
||||
refs: string[],
|
||||
): Promise<string[]> {
|
||||
try {
|
||||
const stdout = await runGitCommand(
|
||||
getOptionalInput("checkout_path"),
|
||||
["merge-base", "--all", ...refs],
|
||||
`Cannot get merge base of ${refs}.`,
|
||||
);
|
||||
return stdout.trim().split("\n");
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Compute the diff hunk headers between the two given refs.
|
||||
*
|
||||
* This function uses the `checkout_path` to determine the repository path and
|
||||
* works only when called from `analyze` or `upload-sarif`.
|
||||
*
|
||||
* @returns an array of diff hunk headers (one element per line), or undefined
|
||||
* if the action was not triggered by a pull request, or if the diff could not
|
||||
* be determined.
|
||||
*/
|
||||
export const getGitDiffHunkHeaders = async function (
|
||||
fromRef: string,
|
||||
toRef: string,
|
||||
): Promise<string[] | undefined> {
|
||||
let stdout = "";
|
||||
try {
|
||||
stdout = await runGitCommand(
|
||||
getOptionalInput("checkout_path"),
|
||||
[
|
||||
"-c",
|
||||
"core.quotePath=false",
|
||||
"diff",
|
||||
"--no-renames",
|
||||
"--irreversible-delete",
|
||||
"-U0",
|
||||
fromRef,
|
||||
toRef,
|
||||
],
|
||||
`Cannot get diff from ${fromRef} to ${toRef}.`,
|
||||
);
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const headers: string[] = [];
|
||||
for (const line of stdout.split("\n")) {
|
||||
if (
|
||||
line.startsWith("--- ") ||
|
||||
line.startsWith("+++ ") ||
|
||||
line.startsWith("@@ ")
|
||||
) {
|
||||
headers.push(line);
|
||||
}
|
||||
}
|
||||
return headers;
|
||||
};
|
||||
|
||||
/**
|
||||
* Decode, if necessary, a file path produced by Git. See
|
||||
* https://git-scm.com/docs/git-config#Documentation/git-config.txt-corequotePath
|
||||
* for details on how Git encodes file paths with special characters.
|
||||
*
|
||||
* This function works only for Git output with `core.quotePath=false`.
|
||||
*/
|
||||
export const decodeGitFilePath = function (filePath: string): string {
|
||||
if (filePath.startsWith('"') && filePath.endsWith('"')) {
|
||||
filePath = filePath.substring(1, filePath.length - 1);
|
||||
return filePath.replace(
|
||||
/\\([abfnrtv\\"]|[0-7]{1,3})/g,
|
||||
(_match, seq: string) => {
|
||||
switch (seq[0]) {
|
||||
case "a":
|
||||
return "\x07";
|
||||
case "b":
|
||||
return "\b";
|
||||
case "f":
|
||||
return "\f";
|
||||
case "n":
|
||||
return "\n";
|
||||
case "r":
|
||||
return "\r";
|
||||
case "t":
|
||||
return "\t";
|
||||
case "v":
|
||||
return "\v";
|
||||
case "\\":
|
||||
return "\\";
|
||||
case '"':
|
||||
return '"';
|
||||
default:
|
||||
// Both String.fromCharCode() and String.fromCodePoint() works only
|
||||
// for constructing an entire character at once. If a Unicode
|
||||
// character is encoded as a sequence of escaped bytes, calling these
|
||||
// methods sequentially on the individual byte values would *not*
|
||||
// produce the original multi-byte Unicode character. As a result,
|
||||
// this implementation works only with the Git option core.quotePath
|
||||
// set to false.
|
||||
return String.fromCharCode(parseInt(seq, 8));
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
return filePath;
|
||||
};
|
||||
|
||||
/**
|
||||
* Get the ref currently being analyzed.
|
||||
*/
|
||||
@@ -472,6 +626,11 @@ export function isSelfHostedRunner() {
|
||||
return process.env.RUNNER_ENVIRONMENT === "self-hosted";
|
||||
}
|
||||
|
||||
/** Determines whether we are running in default setup. */
|
||||
export function isDefaultSetup(): boolean {
|
||||
return getWorkflowEventName() === "dynamic";
|
||||
}
|
||||
|
||||
export function prettyPrintInvocation(cmd: string, args: string[]): string {
|
||||
return [cmd, ...args].map((x) => (x.includes(" ") ? `'${x}'` : x)).join(" ");
|
||||
}
|
||||
@@ -559,3 +718,29 @@ export async function runTool(
|
||||
}
|
||||
return stdout;
|
||||
}
|
||||
|
||||
const persistedInputsKey = "persisted_inputs";
|
||||
|
||||
/**
|
||||
* Persists all inputs to the action as state that can be retrieved later in the post-action.
|
||||
* This would be simplified if actions/runner#3514 is addressed.
|
||||
* https://github.com/actions/runner/issues/3514
|
||||
*/
|
||||
export const persistInputs = function () {
|
||||
const inputEnvironmentVariables = Object.entries(process.env).filter(
|
||||
([name]) => name.startsWith("INPUT_"),
|
||||
);
|
||||
core.saveState(persistedInputsKey, JSON.stringify(inputEnvironmentVariables));
|
||||
};
|
||||
|
||||
/**
|
||||
* Restores all inputs to the action from the persisted state.
|
||||
*/
|
||||
export const restoreInputs = function () {
|
||||
const persistedInputs = core.getState(persistedInputsKey);
|
||||
if (persistedInputs) {
|
||||
for (const [name, value] of JSON.parse(persistedInputs)) {
|
||||
process.env[name] = value;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
*/
|
||||
import * as core from "@actions/core";
|
||||
|
||||
import { getTemporaryDirectory } from "./actions-util";
|
||||
import * as actionsUtil from "./actions-util";
|
||||
import { getGitHubVersion } from "./api-client";
|
||||
import { getConfig } from "./config-utils";
|
||||
import * as debugArtifacts from "./debug-artifacts";
|
||||
@@ -21,6 +21,7 @@ import {
|
||||
|
||||
async function runWrapper() {
|
||||
try {
|
||||
actionsUtil.restoreInputs();
|
||||
const logger = getActionsLogger();
|
||||
const gitHubVersion = await getGitHubVersion();
|
||||
checkGitHubVersionInRange(gitHubVersion, logger);
|
||||
@@ -30,14 +31,17 @@ async function runWrapper() {
|
||||
const features = new Features(
|
||||
gitHubVersion,
|
||||
repositoryNwo,
|
||||
getTemporaryDirectory(),
|
||||
actionsUtil.getTemporaryDirectory(),
|
||||
logger,
|
||||
);
|
||||
|
||||
// Upload SARIF artifacts if we determine that this is a first-party analysis run.
|
||||
// For third-party runs, this artifact will be uploaded in the `upload-sarif-post` step.
|
||||
if (process.env[EnvVar.INIT_ACTION_HAS_RUN] === "true") {
|
||||
const config = await getConfig(getTemporaryDirectory(), logger);
|
||||
const config = await getConfig(
|
||||
actionsUtil.getTemporaryDirectory(),
|
||||
logger,
|
||||
);
|
||||
if (config !== undefined) {
|
||||
await withGroup("Uploading combined SARIF debug artifact", () =>
|
||||
debugArtifacts.uploadCombinedSarifArtifacts(
|
||||
|
||||
@@ -3,6 +3,7 @@ import path from "path";
|
||||
import { performance } from "perf_hooks";
|
||||
|
||||
import * as core from "@actions/core";
|
||||
import * as github from "@actions/github";
|
||||
|
||||
import * as actionsUtil from "./actions-util";
|
||||
import {
|
||||
@@ -12,13 +13,16 @@ import {
|
||||
runCleanup,
|
||||
runFinalize,
|
||||
runQueries,
|
||||
setupDiffInformedQueryRun,
|
||||
warnIfGoInstalledAfterInit,
|
||||
} from "./analyze";
|
||||
import { getApiDetails, getGitHubVersion } from "./api-client";
|
||||
import { runAutobuild } from "./autobuild";
|
||||
import { getTotalCacheSize, shouldStoreCache } from "./caching-utils";
|
||||
import { getCodeQL } from "./codeql";
|
||||
import { Config, getConfig } from "./config-utils";
|
||||
import { uploadDatabases } from "./database-upload";
|
||||
import { uploadDependencyCaches } from "./dependency-caching";
|
||||
import { EnvVar } from "./environment";
|
||||
import { Features } from "./feature-flags";
|
||||
import { Language } from "./languages";
|
||||
@@ -34,7 +38,6 @@ import {
|
||||
} from "./status-report";
|
||||
import {
|
||||
cleanupTrapCaches,
|
||||
getTotalCacheSize,
|
||||
TrapCacheCleanupStatusReport,
|
||||
uploadTrapCaches,
|
||||
} from "./trap-caching";
|
||||
@@ -92,7 +95,7 @@ async function sendStatusReport(
|
||||
...report,
|
||||
trap_cache_upload_duration_ms: Math.round(trapCacheUploadTime || 0),
|
||||
trap_cache_upload_size_bytes: Math.round(
|
||||
await getTotalCacheSize(config.trapCaches, logger),
|
||||
await getTotalCacheSize(Object.values(config.trapCaches), logger),
|
||||
),
|
||||
};
|
||||
await statusReport.sendStatusReport(trapCacheUploadStatusReport);
|
||||
@@ -199,6 +202,10 @@ async function run() {
|
||||
let didUploadTrapCaches = false;
|
||||
util.initializeEnvironment(actionsUtil.getActionVersion());
|
||||
|
||||
// Make inputs accessible in the `post` step, details at
|
||||
// https://github.com/github/codeql-action/issues/2553
|
||||
actionsUtil.persistInputs();
|
||||
|
||||
const logger = getActionsLogger();
|
||||
try {
|
||||
const statusReportBase = await createStatusReportBase(
|
||||
@@ -256,6 +263,17 @@ async function run() {
|
||||
logger,
|
||||
);
|
||||
|
||||
const pull_request = github.context.payload.pull_request;
|
||||
const diffRangePackDir =
|
||||
pull_request &&
|
||||
(await setupDiffInformedQueryRun(
|
||||
pull_request.base.ref as string,
|
||||
pull_request.head.ref as string,
|
||||
codeql,
|
||||
logger,
|
||||
features,
|
||||
));
|
||||
|
||||
await warnIfGoInstalledAfterInit(config, logger);
|
||||
await runAutobuildIfLegacyGoWorkflow(config, logger);
|
||||
|
||||
@@ -274,6 +292,7 @@ async function run() {
|
||||
memory,
|
||||
util.getAddSnippetsFlag(actionsUtil.getRequiredInput("add-snippets")),
|
||||
threads,
|
||||
diffRangePackDir,
|
||||
actionsUtil.getOptionalInput("category"),
|
||||
config,
|
||||
logger,
|
||||
@@ -324,6 +343,11 @@ async function run() {
|
||||
logger,
|
||||
);
|
||||
|
||||
// Store dependency cache(s) if dependency caching is enabled.
|
||||
if (shouldStoreCache(config.dependencyCachingEnabled)) {
|
||||
await uploadDependencyCaches(config, logger);
|
||||
}
|
||||
|
||||
// We don't upload results in test mode, so don't wait for processing
|
||||
if (util.isInTestMode()) {
|
||||
logger.debug("In test mode. Waiting for processing is disabled.");
|
||||
|
||||
@@ -101,6 +101,7 @@ test("status report fields", async (t) => {
|
||||
addSnippetsFlag,
|
||||
threadsFlag,
|
||||
undefined,
|
||||
undefined,
|
||||
config,
|
||||
getRunnerLogger(true),
|
||||
createFeatures([Feature.QaTelemetryEnabled]),
|
||||
|
||||
228
src/analyze.ts
228
src/analyze.ts
@@ -6,6 +6,7 @@ import { safeWhich } from "@chrisgavin/safe-which";
|
||||
import del from "del";
|
||||
import * as yaml from "js-yaml";
|
||||
|
||||
import * as actionsUtil from "./actions-util";
|
||||
import { setupCppAutobuild } from "./autobuild";
|
||||
import {
|
||||
CODEQL_VERSION_ANALYSIS_SUMMARY_V2,
|
||||
@@ -17,7 +18,7 @@ import { addDiagnostic, makeDiagnostic } from "./diagnostics";
|
||||
import { EnvVar } from "./environment";
|
||||
import { FeatureEnablement, Feature } from "./feature-flags";
|
||||
import { isScannedLanguage, Language } from "./languages";
|
||||
import { Logger } from "./logging";
|
||||
import { Logger, withGroup } from "./logging";
|
||||
import { DatabaseCreationTimings, EventReport } from "./status-report";
|
||||
import { ToolsFeature } from "./tools-features";
|
||||
import { endTracingForCluster } from "./tracer-config";
|
||||
@@ -234,12 +235,224 @@ async function finalizeDatabaseCreation(
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Set up the diff-informed analysis feature.
|
||||
*
|
||||
* @param baseRef The base branch name, used for calculating the diff range.
|
||||
* @param headRef The head branch name, used for calculating the diff range.
|
||||
* @param codeql
|
||||
* @param logger
|
||||
* @param features
|
||||
* @returns Absolute path to the directory containing the extension pack for
|
||||
* the diff range information, or `undefined` if the feature is disabled.
|
||||
*/
|
||||
export async function setupDiffInformedQueryRun(
|
||||
baseRef: string,
|
||||
headRef: string,
|
||||
codeql: CodeQL,
|
||||
logger: Logger,
|
||||
features: FeatureEnablement,
|
||||
): Promise<string | undefined> {
|
||||
if (!(await features.getValue(Feature.DiffInformedQueries, codeql))) {
|
||||
return undefined;
|
||||
}
|
||||
return await withGroup("Generating diff range extension pack", async () => {
|
||||
const diffRanges = await getPullRequestEditedDiffRanges(
|
||||
baseRef,
|
||||
headRef,
|
||||
logger,
|
||||
);
|
||||
return writeDiffRangeDataExtensionPack(logger, diffRanges);
|
||||
});
|
||||
}
|
||||
|
||||
interface DiffThunkRange {
|
||||
path: string;
|
||||
startLine: number;
|
||||
endLine: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the file line ranges that were added or modified in the pull request.
|
||||
*
|
||||
* @param baseRef The base branch name, used for calculating the diff range.
|
||||
* @param headRef The head branch name, used for calculating the diff range.
|
||||
* @param logger
|
||||
* @returns An array of tuples, where each tuple contains the absolute path of a
|
||||
* file, the start line and the end line (both 1-based and inclusive) of an
|
||||
* added or modified range in that file. Returns `undefined` if the action was
|
||||
* not triggered by a pull request or if there was an error.
|
||||
*/
|
||||
async function getPullRequestEditedDiffRanges(
|
||||
baseRef: string,
|
||||
headRef: string,
|
||||
logger: Logger,
|
||||
): Promise<DiffThunkRange[] | undefined> {
|
||||
const checkoutPath = actionsUtil.getOptionalInput("checkout_path");
|
||||
if (checkoutPath === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
// To compute the merge bases between the base branch and the PR topic branch,
|
||||
// we need to fetch the commit graph from the branch heads to those merge
|
||||
// babes. The following 4-step procedure does so while limiting the amount of
|
||||
// history fetched.
|
||||
|
||||
// Step 1: Deepen from the PR merge commit to the base branch head and the PR
|
||||
// topic branch head, so that the PR merge commit is no longer considered a
|
||||
// grafted commit.
|
||||
await actionsUtil.deepenGitHistory();
|
||||
// Step 2: Fetch the base branch shallow history. This step ensures that the
|
||||
// base branch name is present in the local repository. Normally the base
|
||||
// branch name would be added by Step 4. However, if the base branch head is
|
||||
// an ancestor of the PR topic branch head, Step 4 would fail without doing
|
||||
// anything, so we need to fetch the base branch explicitly.
|
||||
await actionsUtil.gitFetch(baseRef, ["--depth=1"]);
|
||||
// Step 3: Fetch the PR topic branch history, stopping when we reach commits
|
||||
// that are reachable from the base branch head.
|
||||
await actionsUtil.gitFetch(headRef, [`--shallow-exclude=${baseRef}`]);
|
||||
// Step 4: Fetch the base branch history, stopping when we reach commits that
|
||||
// are reachable from the PR topic branch head.
|
||||
await actionsUtil.gitFetch(baseRef, [`--shallow-exclude=${headRef}`]);
|
||||
// Step 5: Deepen the history so that we have the merge bases between the base
|
||||
// branch and the PR topic branch.
|
||||
await actionsUtil.deepenGitHistory();
|
||||
|
||||
// To compute the exact same diff as GitHub would compute for the PR, we need
|
||||
// to use the same merge base as GitHub. That is easy to do if there is only
|
||||
// one merge base, which is by far the most common case. If there are multiple
|
||||
// merge bases, we stop without producing a diff range.
|
||||
const mergeBases = await actionsUtil.getAllGitMergeBases([baseRef, headRef]);
|
||||
logger.info(`Merge bases: ${mergeBases.join(", ")}`);
|
||||
if (mergeBases.length !== 1) {
|
||||
logger.info(
|
||||
"Cannot compute diff range because baseRef and headRef " +
|
||||
`have ${mergeBases.length} merge bases (instead of exactly 1).`,
|
||||
);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const diffHunkHeaders = await actionsUtil.getGitDiffHunkHeaders(
|
||||
mergeBases[0],
|
||||
headRef,
|
||||
);
|
||||
if (diffHunkHeaders === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const results = new Array<DiffThunkRange>();
|
||||
|
||||
let changedFile = "";
|
||||
for (const line of diffHunkHeaders) {
|
||||
if (line.startsWith("+++ ")) {
|
||||
const filePath = actionsUtil.decodeGitFilePath(line.substring(4));
|
||||
if (filePath.startsWith("b/")) {
|
||||
// The file was edited: track all hunks in the file
|
||||
changedFile = filePath.substring(2);
|
||||
} else if (filePath === "/dev/null") {
|
||||
// The file was deleted: skip all hunks in the file
|
||||
changedFile = "";
|
||||
} else {
|
||||
logger.warning(`Failed to parse diff hunk header line: ${line}`);
|
||||
return undefined;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (line.startsWith("@@ ")) {
|
||||
if (changedFile === "") continue;
|
||||
|
||||
const match = line.match(/^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@/);
|
||||
if (match === null) {
|
||||
logger.warning(`Failed to parse diff hunk header line: ${line}`);
|
||||
return undefined;
|
||||
}
|
||||
const startLine = parseInt(match[1], 10);
|
||||
const numLines = parseInt(match[2], 10);
|
||||
if (numLines === 0) {
|
||||
// The hunk was a deletion: skip it
|
||||
continue;
|
||||
}
|
||||
const endLine = startLine + (numLines || 1) - 1;
|
||||
results.push({
|
||||
path: path.join(checkoutPath, changedFile),
|
||||
startLine,
|
||||
endLine,
|
||||
});
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create an extension pack in the temporary directory that contains the file
|
||||
* line ranges that were added or modified in the pull request.
|
||||
*
|
||||
* @param logger
|
||||
* @param ranges The file line ranges, as returned by
|
||||
* `getPullRequestEditedDiffRanges`.
|
||||
* @returns The absolute path of the directory containing the extension pack, or
|
||||
* `undefined` if no extension pack was created.
|
||||
*/
|
||||
function writeDiffRangeDataExtensionPack(
|
||||
logger: Logger,
|
||||
ranges: DiffThunkRange[] | undefined,
|
||||
): string | undefined {
|
||||
if (ranges === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const diffRangeDir = path.join(
|
||||
actionsUtil.getTemporaryDirectory(),
|
||||
"pr-diff-range",
|
||||
);
|
||||
fs.mkdirSync(diffRangeDir);
|
||||
fs.writeFileSync(
|
||||
path.join(diffRangeDir, "qlpack.yml"),
|
||||
`
|
||||
name: codeql-action/pr-diff-range
|
||||
version: 0.0.0
|
||||
library: true
|
||||
extensionTargets:
|
||||
codeql/util: '*'
|
||||
dataExtensions:
|
||||
- pr-diff-range.yml
|
||||
`,
|
||||
);
|
||||
|
||||
const header = `
|
||||
extensions:
|
||||
- addsTo:
|
||||
pack: codeql/util
|
||||
extensible: restrictAlertsTo
|
||||
data:
|
||||
`;
|
||||
|
||||
let data = ranges
|
||||
.map((range) => ` - ["${range[0]}", ${range[1]}, ${range[2]}]\n`)
|
||||
.join("");
|
||||
if (!data) {
|
||||
// Ensure that the data extension is not empty, so that a pull request with
|
||||
// no edited lines would exclude (instead of accepting) all alerts.
|
||||
data = ' - ["", 0, 0]\n';
|
||||
}
|
||||
|
||||
const extensionContents = header + data;
|
||||
const extensionFilePath = path.join(diffRangeDir, "pr-diff-range.yml");
|
||||
fs.writeFileSync(extensionFilePath, extensionContents);
|
||||
logger.debug(
|
||||
`Wrote pr-diff-range extension pack to ${extensionFilePath}:\n${extensionContents}`,
|
||||
);
|
||||
|
||||
return diffRangeDir;
|
||||
}
|
||||
|
||||
// Runs queries and creates sarif files in the given folder
|
||||
export async function runQueries(
|
||||
sarifFolder: string,
|
||||
memoryFlag: string,
|
||||
addSnippetsFlag: string,
|
||||
threadsFlag: string,
|
||||
diffRangePackDir: string | undefined,
|
||||
automationDetailsId: string | undefined,
|
||||
config: configUtils.Config,
|
||||
logger: Logger,
|
||||
@@ -247,8 +460,18 @@ export async function runQueries(
|
||||
): Promise<QueriesStatusReport> {
|
||||
const statusReport: QueriesStatusReport = {};
|
||||
|
||||
const dataExtensionFlags = diffRangePackDir
|
||||
? [
|
||||
`--additional-packs=${diffRangePackDir}`,
|
||||
"--extension-packs=codeql-action/pr-diff-range",
|
||||
]
|
||||
: [];
|
||||
const sarifRunPropertyFlag = diffRangePackDir
|
||||
? "--sarif-run-property=incrementalMode=diff-informed"
|
||||
: undefined;
|
||||
|
||||
const codeql = await getCodeQL(config.codeQLCmd);
|
||||
const queryFlags = [memoryFlag, threadsFlag];
|
||||
const queryFlags = [memoryFlag, threadsFlag, ...dataExtensionFlags];
|
||||
|
||||
for (const language of config.languages) {
|
||||
try {
|
||||
@@ -336,6 +559,7 @@ export async function runQueries(
|
||||
addSnippetsFlag,
|
||||
threadsFlag,
|
||||
enableDebugLogging ? "-vv" : "-v",
|
||||
sarifRunPropertyFlag,
|
||||
automationDetailsId,
|
||||
config,
|
||||
features,
|
||||
|
||||
89
src/caching-utils.ts
Normal file
89
src/caching-utils.ts
Normal file
@@ -0,0 +1,89 @@
|
||||
import * as core from "@actions/core";
|
||||
|
||||
import { getOptionalInput, isDefaultSetup } from "./actions-util";
|
||||
import { EnvVar } from "./environment";
|
||||
import { Logger } from "./logging";
|
||||
import { isHostedRunner, tryGetFolderBytes } from "./util";
|
||||
|
||||
/**
|
||||
* Returns the total size of all the specified paths.
|
||||
* @param paths The paths for which to calculate the total size.
|
||||
* @param logger A logger to record some informational messages to.
|
||||
* @returns The total size of all specified paths.
|
||||
*/
|
||||
export async function getTotalCacheSize(
|
||||
paths: string[],
|
||||
logger: Logger,
|
||||
): Promise<number> {
|
||||
const sizes = await Promise.all(
|
||||
paths.map((cacheDir) => tryGetFolderBytes(cacheDir, logger)),
|
||||
);
|
||||
return sizes.map((a) => a || 0).reduce((a, b) => a + b, 0);
|
||||
}
|
||||
|
||||
/* Enumerates caching modes. */
|
||||
export enum CachingKind {
|
||||
/** Do not restore or store any caches. */
|
||||
None = "none",
|
||||
/** Store caches, but do not restore any existing ones. */
|
||||
Store = "store",
|
||||
/** Restore existing caches, but do not store any new ones. */
|
||||
Restore = "restore",
|
||||
/** Restore existing caches, and store new ones. */
|
||||
Full = "full",
|
||||
}
|
||||
|
||||
/** Returns a value indicating whether new caches should be stored, based on `kind`. */
|
||||
export function shouldStoreCache(kind: CachingKind): boolean {
|
||||
return kind === CachingKind.Full || kind === CachingKind.Store;
|
||||
}
|
||||
|
||||
/** Returns a value indicating whether existing caches should be restored, based on `kind`. */
|
||||
export function shouldRestoreCache(kind: CachingKind): boolean {
|
||||
return kind === CachingKind.Full || kind === CachingKind.Restore;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses the `upload` input into an `UploadKind`.
|
||||
*/
|
||||
export function getCachingKind(input: string | undefined): CachingKind {
|
||||
switch (input) {
|
||||
case undefined:
|
||||
case "none":
|
||||
case "off":
|
||||
case "false":
|
||||
return CachingKind.None;
|
||||
case "full":
|
||||
case "on":
|
||||
case "true":
|
||||
return CachingKind.Full;
|
||||
case "store":
|
||||
return CachingKind.Store;
|
||||
case "restore":
|
||||
return CachingKind.Restore;
|
||||
default:
|
||||
core.warning(
|
||||
`Unrecognized 'dependency-caching' input: ${input}. Defaulting to 'none'.`,
|
||||
);
|
||||
return CachingKind.None;
|
||||
}
|
||||
}
|
||||
|
||||
/** Determines whether dependency caching is enabled. */
|
||||
export function getDependencyCachingEnabled(): CachingKind {
|
||||
// If the workflow specified something always respect that
|
||||
const dependencyCaching =
|
||||
getOptionalInput("dependency-caching") ||
|
||||
process.env[EnvVar.DEPENDENCY_CACHING];
|
||||
if (dependencyCaching !== undefined) return getCachingKind(dependencyCaching);
|
||||
|
||||
// On self-hosted runners which may have dependencies installed centrally, disable caching by default
|
||||
if (!isHostedRunner()) return CachingKind.None;
|
||||
|
||||
// Disable in advanced workflows by default.
|
||||
if (!isDefaultSetup()) return CachingKind.None;
|
||||
|
||||
// On hosted runners, disable dependency caching by default.
|
||||
// TODO: Review later whether we can enable this by default.
|
||||
return CachingKind.None;
|
||||
}
|
||||
@@ -839,6 +839,7 @@ for (const {
|
||||
"",
|
||||
"",
|
||||
"-v",
|
||||
undefined,
|
||||
"",
|
||||
Object.assign({}, stubConfig, { gitHubVersion: githubVersion }),
|
||||
createFeatures([]),
|
||||
|
||||
@@ -166,6 +166,7 @@ export interface CodeQL {
|
||||
addSnippetsFlag: string,
|
||||
threadsFlag: string,
|
||||
verbosityFlag: string | undefined,
|
||||
sarifRunPropertyFlag: string | undefined,
|
||||
automationDetailsId: string | undefined,
|
||||
config: Config,
|
||||
features: FeatureEnablement,
|
||||
@@ -834,6 +835,7 @@ export async function getCodeQLForCmd(
|
||||
addSnippetsFlag: string,
|
||||
threadsFlag: string,
|
||||
verbosityFlag: string,
|
||||
sarifRunPropertyFlag: string | undefined,
|
||||
automationDetailsId: string | undefined,
|
||||
config: Config,
|
||||
features: FeatureEnablement,
|
||||
@@ -861,6 +863,9 @@ export async function getCodeQLForCmd(
|
||||
...(await getJobRunUuidSarifOptions(this)),
|
||||
...getExtraOptionsFromEnv(["database", "interpret-results"]),
|
||||
];
|
||||
if (sarifRunPropertyFlag !== undefined) {
|
||||
codeqlArgs.push(sarifRunPropertyFlag);
|
||||
}
|
||||
if (automationDetailsId !== undefined) {
|
||||
codeqlArgs.push("--sarif-category", automationDetailsId);
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import * as yaml from "js-yaml";
|
||||
import * as sinon from "sinon";
|
||||
|
||||
import * as api from "./api-client";
|
||||
import { CachingKind } from "./caching-utils";
|
||||
import {
|
||||
CodeQL,
|
||||
getCachedCodeQL,
|
||||
@@ -52,6 +53,7 @@ function createTestInitConfigInputs(
|
||||
configInput: undefined,
|
||||
buildModeInput: undefined,
|
||||
trapCachingEnabled: false,
|
||||
dependencyCachingEnabled: CachingKind.None,
|
||||
debugMode: false,
|
||||
debugArtifactName: "",
|
||||
debugDatabaseName: "",
|
||||
@@ -347,6 +349,7 @@ test("load non-empty input", async (t) => {
|
||||
augmentationProperties: configUtils.defaultAugmentationProperties,
|
||||
trapCaches: {},
|
||||
trapCacheDownloadTime: 0,
|
||||
dependencyCachingEnabled: CachingKind.None,
|
||||
};
|
||||
|
||||
const languagesInput = "javascript";
|
||||
|
||||
@@ -6,6 +6,7 @@ import * as yaml from "js-yaml";
|
||||
import * as semver from "semver";
|
||||
|
||||
import * as api from "./api-client";
|
||||
import { CachingKind, getCachingKind } from "./caching-utils";
|
||||
import { CodeQL } from "./codeql";
|
||||
import { Feature, FeatureEnablement } from "./feature-flags";
|
||||
import { Language, parseLanguage } from "./languages";
|
||||
@@ -139,6 +140,9 @@ export interface Config {
|
||||
* Time taken to download TRAP caches. Used for status reporting.
|
||||
*/
|
||||
trapCacheDownloadTime: number;
|
||||
|
||||
/** A value indicating how dependency caching should be used. */
|
||||
dependencyCachingEnabled: CachingKind;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -396,6 +400,7 @@ export interface InitConfigInputs {
|
||||
configInput: string | undefined;
|
||||
buildModeInput: string | undefined;
|
||||
trapCachingEnabled: boolean;
|
||||
dependencyCachingEnabled: string | undefined;
|
||||
debugMode: boolean;
|
||||
debugArtifactName: string;
|
||||
debugDatabaseName: string;
|
||||
@@ -428,6 +433,7 @@ export async function getDefaultConfig({
|
||||
buildModeInput,
|
||||
dbLocation,
|
||||
trapCachingEnabled,
|
||||
dependencyCachingEnabled,
|
||||
debugMode,
|
||||
debugArtifactName,
|
||||
debugDatabaseName,
|
||||
@@ -479,6 +485,7 @@ export async function getDefaultConfig({
|
||||
augmentationProperties,
|
||||
trapCaches,
|
||||
trapCacheDownloadTime,
|
||||
dependencyCachingEnabled: getCachingKind(dependencyCachingEnabled),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -512,6 +519,7 @@ async function loadConfig({
|
||||
configFile,
|
||||
dbLocation,
|
||||
trapCachingEnabled,
|
||||
dependencyCachingEnabled,
|
||||
debugMode,
|
||||
debugArtifactName,
|
||||
debugDatabaseName,
|
||||
@@ -583,6 +591,7 @@ async function loadConfig({
|
||||
augmentationProperties,
|
||||
trapCaches,
|
||||
trapCacheDownloadTime,
|
||||
dependencyCachingEnabled: getCachingKind(dependencyCachingEnabled),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"bundleVersion": "codeql-bundle-v2.19.1",
|
||||
"cliVersion": "2.19.1",
|
||||
"priorBundleVersion": "codeql-bundle-v2.19.0",
|
||||
"priorCliVersion": "2.19.0"
|
||||
"bundleVersion": "codeql-bundle-v2.19.2",
|
||||
"cliVersion": "2.19.2",
|
||||
"priorBundleVersion": "codeql-bundle-v2.19.1",
|
||||
"priorCliVersion": "2.19.1"
|
||||
}
|
||||
|
||||
223
src/dependency-caching.ts
Normal file
223
src/dependency-caching.ts
Normal file
@@ -0,0 +1,223 @@
|
||||
import * as os from "os";
|
||||
import { join } from "path";
|
||||
|
||||
import * as actionsCache from "@actions/cache";
|
||||
import * as glob from "@actions/glob";
|
||||
|
||||
import { getTotalCacheSize } from "./caching-utils";
|
||||
import { Config } from "./config-utils";
|
||||
import { EnvVar } from "./environment";
|
||||
import { Language } from "./languages";
|
||||
import { Logger } from "./logging";
|
||||
import { getRequiredEnvParam } from "./util";
|
||||
|
||||
/**
|
||||
* Caching configuration for a particular language.
|
||||
*/
|
||||
interface CacheConfig {
|
||||
/** The paths of directories on the runner that should be included in the cache. */
|
||||
paths: string[];
|
||||
/**
|
||||
* Patterns for the paths of files whose contents affect which dependencies are used
|
||||
* by a project. We find all files which match these patterns, calculate a hash for
|
||||
* their contents, and use that hash as part of the cache key.
|
||||
*/
|
||||
hash: string[];
|
||||
}
|
||||
|
||||
const CODEQL_DEPENDENCY_CACHE_PREFIX = "codeql-dependencies";
|
||||
const CODEQL_DEPENDENCY_CACHE_VERSION = 1;
|
||||
|
||||
/**
|
||||
* Default caching configurations per language.
|
||||
*/
|
||||
const CODEQL_DEFAULT_CACHE_CONFIG: { [language: string]: CacheConfig } = {
|
||||
java: {
|
||||
paths: [
|
||||
// Maven
|
||||
join(os.homedir(), ".m2", "repository"),
|
||||
// Gradle
|
||||
join(os.homedir(), ".gradle", "caches"),
|
||||
],
|
||||
hash: [
|
||||
// Maven
|
||||
"**/pom.xml",
|
||||
// Gradle
|
||||
"**/*.gradle*",
|
||||
"**/gradle-wrapper.properties",
|
||||
"buildSrc/**/Versions.kt",
|
||||
"buildSrc/**/Dependencies.kt",
|
||||
"gradle/*.versions.toml",
|
||||
"**/versions.properties",
|
||||
],
|
||||
},
|
||||
csharp: {
|
||||
paths: [join(os.homedir(), ".nuget", "packages")],
|
||||
hash: [
|
||||
// NuGet
|
||||
"**/packages.lock.json",
|
||||
// Paket
|
||||
"**/paket.lock",
|
||||
],
|
||||
},
|
||||
go: {
|
||||
paths: [join(os.homedir(), "go", "pkg", "mod")],
|
||||
hash: ["**/go.sum"],
|
||||
},
|
||||
};
|
||||
|
||||
async function makeGlobber(patterns: string[]): Promise<glob.Globber> {
|
||||
return glob.create(patterns.join("\n"));
|
||||
}
|
||||
|
||||
/**
|
||||
* Attempts to restore dependency caches for the languages being analyzed.
|
||||
*
|
||||
* @param languages The languages being analyzed.
|
||||
* @param logger A logger to record some informational messages to.
|
||||
* @returns A list of languages for which dependency caches were restored.
|
||||
*/
|
||||
export async function downloadDependencyCaches(
|
||||
languages: Language[],
|
||||
logger: Logger,
|
||||
): Promise<Language[]> {
|
||||
const restoredCaches: Language[] = [];
|
||||
|
||||
for (const language of languages) {
|
||||
const cacheConfig = CODEQL_DEFAULT_CACHE_CONFIG[language];
|
||||
|
||||
if (cacheConfig === undefined) {
|
||||
logger.info(
|
||||
`Skipping download of dependency cache for ${language} as we have no caching configuration for it.`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check that we can find files to calculate the hash for the cache key from, so we don't end up
|
||||
// with an empty string.
|
||||
const globber = await makeGlobber(cacheConfig.hash);
|
||||
|
||||
if ((await globber.glob()).length === 0) {
|
||||
logger.info(
|
||||
`Skipping download of dependency cache for ${language} as we cannot calculate a hash for the cache key.`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
const primaryKey = await cacheKey(language, cacheConfig);
|
||||
const restoreKeys: string[] = [await cachePrefix(language)];
|
||||
|
||||
logger.info(
|
||||
`Downloading cache for ${language} with key ${primaryKey} and restore keys ${restoreKeys.join(
|
||||
", ",
|
||||
)}`,
|
||||
);
|
||||
|
||||
const hitKey = await actionsCache.restoreCache(
|
||||
cacheConfig.paths,
|
||||
primaryKey,
|
||||
restoreKeys,
|
||||
);
|
||||
|
||||
if (hitKey !== undefined) {
|
||||
logger.info(`Cache hit on key ${hitKey} for ${language}.`);
|
||||
restoredCaches.push(language);
|
||||
} else {
|
||||
logger.info(`No suitable cache found for ${language}.`);
|
||||
}
|
||||
}
|
||||
|
||||
return restoredCaches;
|
||||
}
|
||||
|
||||
/**
|
||||
* Attempts to store caches for the languages that were analyzed.
|
||||
*
|
||||
* @param config The configuration for this workflow.
|
||||
* @param logger A logger to record some informational messages to.
|
||||
*/
|
||||
export async function uploadDependencyCaches(config: Config, logger: Logger) {
|
||||
for (const language of config.languages) {
|
||||
const cacheConfig = CODEQL_DEFAULT_CACHE_CONFIG[language];
|
||||
|
||||
if (cacheConfig === undefined) {
|
||||
logger.info(
|
||||
`Skipping upload of dependency cache for ${language} as we have no caching configuration for it.`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check that we can find files to calculate the hash for the cache key from, so we don't end up
|
||||
// with an empty string.
|
||||
const globber = await makeGlobber(cacheConfig.hash);
|
||||
|
||||
if ((await globber.glob()).length === 0) {
|
||||
logger.info(
|
||||
`Skipping upload of dependency cache for ${language} as we cannot calculate a hash for the cache key.`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Calculate the size of the files that we would store in the cache. We use this to determine whether the
|
||||
// cache should be saved or not. For example, if there are no files to store, then we skip creating the
|
||||
// cache. In the future, we could also:
|
||||
// - Skip uploading caches with a size below some threshold: this makes sense for avoiding the overhead
|
||||
// of storing and restoring small caches, but does not help with alert wobble if a package repository
|
||||
// cannot be reached in a given run.
|
||||
// - Skip uploading caches with a size above some threshold: this could be a concern if other workflows
|
||||
// use the cache quota that we compete with. In that case, we do not wish to use up all of the quota
|
||||
// with the dependency caches. For this, we could use the Cache API to check whether other workflows
|
||||
// are using the quota and how full it is.
|
||||
const size = await getTotalCacheSize(cacheConfig.paths, logger);
|
||||
|
||||
// Skip uploading an empty cache.
|
||||
if (size === 0) {
|
||||
logger.info(
|
||||
`Skipping upload of dependency cache for ${language} since it is empty.`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
const key = await cacheKey(language, cacheConfig);
|
||||
|
||||
logger.info(
|
||||
`Uploading cache of size ${size} for ${language} with key ${key}`,
|
||||
);
|
||||
|
||||
await actionsCache.saveCache(cacheConfig.paths, key);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Computes a cache key for the specified language.
|
||||
*
|
||||
* @param language The language being analyzed.
|
||||
* @param cacheConfig The cache configuration for the language.
|
||||
* @returns A cache key capturing information about the project(s) being analyzed in the specified language.
|
||||
*/
|
||||
async function cacheKey(
|
||||
language: Language,
|
||||
cacheConfig: CacheConfig,
|
||||
): Promise<string> {
|
||||
const hash = await glob.hashFiles(cacheConfig.hash.join("\n"));
|
||||
return `${await cachePrefix(language)}${hash}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Constructs a prefix for the cache key, comprised of a CodeQL-specific prefix, a version number that
|
||||
* can be changed to invalidate old caches, the runner's operating system, and the specified language name.
|
||||
*
|
||||
* @param language The language being analyzed.
|
||||
* @returns The prefix that identifies what a cache is for.
|
||||
*/
|
||||
async function cachePrefix(language: Language): Promise<string> {
|
||||
const runnerOs = getRequiredEnvParam("RUNNER_OS");
|
||||
const customPrefix = process.env[EnvVar.DEPENDENCY_CACHING_PREFIX];
|
||||
let prefix = CODEQL_DEPENDENCY_CACHE_PREFIX;
|
||||
|
||||
if (customPrefix !== undefined && customPrefix.length > 0) {
|
||||
prefix = `${prefix}-${customPrefix}`;
|
||||
}
|
||||
|
||||
return `${prefix}-${CODEQL_DEPENDENCY_CACHE_VERSION}-${runnerOs}-${language}-`;
|
||||
}
|
||||
@@ -98,4 +98,17 @@ export enum EnvVar {
|
||||
* We check this later to ensure that it hasn't been tampered with by a late e.g. `setup-go` step.
|
||||
*/
|
||||
GO_BINARY_LOCATION = "CODEQL_ACTION_GO_BINARY",
|
||||
|
||||
/**
|
||||
* Used as an alternative to the `dependency-caching` input for the `init` Action.
|
||||
* Useful for experiments where it is easier to set an environment variable than
|
||||
* change the inputs to the Action.
|
||||
*/
|
||||
DEPENDENCY_CACHING = "CODEQL_ACTION_DEPENDENCY_CACHING",
|
||||
|
||||
/**
|
||||
* An optional string to add into the cache key used by dependency caching.
|
||||
* Useful for testing purposes where multiple caches may be stored in the same repository.
|
||||
*/
|
||||
DEPENDENCY_CACHING_PREFIX = "CODEQL_ACTION_DEPENDENCY_CACHE_PREFIX",
|
||||
}
|
||||
|
||||
@@ -60,6 +60,36 @@ test(`All features are disabled if running against GHES`, async (t) => {
|
||||
});
|
||||
});
|
||||
|
||||
test(`Feature flags are requested in Proxima`, async (t) => {
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
const loggedMessages = [];
|
||||
const features = setUpFeatureFlagTests(
|
||||
tmpDir,
|
||||
getRecordingLogger(loggedMessages),
|
||||
{ type: GitHubVariant.GHE_DOTCOM },
|
||||
);
|
||||
|
||||
mockFeatureFlagApiEndpoint(200, initializeFeatures(true));
|
||||
|
||||
for (const feature of Object.values(Feature)) {
|
||||
// Ensure we have gotten a response value back from the Mock API
|
||||
t.assert(
|
||||
await features.getValue(feature, includeCodeQlIfRequired(feature)),
|
||||
);
|
||||
}
|
||||
|
||||
// And that we haven't bailed preemptively.
|
||||
t.assert(
|
||||
loggedMessages.find(
|
||||
(v: LoggedMessage) =>
|
||||
v.type === "debug" &&
|
||||
v.message ===
|
||||
"Not running against github.com. Disabling all toggleable features.",
|
||||
) === undefined,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test("API response missing and features use default value", async (t) => {
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
const loggedMessages: LoggedMessage[] = [];
|
||||
|
||||
@@ -50,7 +50,9 @@ export interface FeatureEnablement {
|
||||
export enum Feature {
|
||||
ArtifactV4Upgrade = "artifact_v4_upgrade",
|
||||
CleanupTrapCaches = "cleanup_trap_caches",
|
||||
CppBuildModeNone = "cpp_build_mode_none",
|
||||
CppDependencyInstallation = "cpp_dependency_installation_enabled",
|
||||
DiffInformedQueries = "diff_informed_queries",
|
||||
DisableCsharpBuildless = "disable_csharp_buildless",
|
||||
DisableJavaBuildlessEnabled = "disable_java_buildless_enabled",
|
||||
DisableKotlinAnalysisEnabled = "disable_kotlin_analysis_enabled",
|
||||
@@ -102,12 +104,23 @@ export const featureConfig: Record<
|
||||
envVar: "CODEQL_ACTION_CLEANUP_TRAP_CACHES",
|
||||
minimumVersion: undefined,
|
||||
},
|
||||
[Feature.CppBuildModeNone]: {
|
||||
defaultValue: false,
|
||||
envVar: "CODEQL_EXTRACTOR_CPP_BUILD_MODE_NONE",
|
||||
minimumVersion: undefined,
|
||||
},
|
||||
[Feature.CppDependencyInstallation]: {
|
||||
defaultValue: false,
|
||||
envVar: "CODEQL_EXTRACTOR_CPP_AUTOINSTALL_DEPENDENCIES",
|
||||
legacyApi: true,
|
||||
minimumVersion: "2.15.0",
|
||||
},
|
||||
[Feature.DiffInformedQueries]: {
|
||||
defaultValue: false,
|
||||
envVar: "CODEQL_ACTION_DIFF_INFORMED_QUERIES",
|
||||
minimumVersion: undefined,
|
||||
toolsFeature: ToolsFeature.DatabaseInterpretResultsSupportsSarifRunProperty,
|
||||
},
|
||||
[Feature.DisableCsharpBuildless]: {
|
||||
defaultValue: false,
|
||||
envVar: "CODEQL_ACTION_DISABLE_CSHARP_BUILDLESS",
|
||||
@@ -472,7 +485,10 @@ class GitHubFeatureFlags {
|
||||
|
||||
private async loadApiResponse(): Promise<GitHubFeatureFlagsApiResponse> {
|
||||
// Do nothing when not running against github.com
|
||||
if (this.gitHubVersion.type !== util.GitHubVariant.DOTCOM) {
|
||||
if (
|
||||
this.gitHubVersion.type !== util.GitHubVariant.DOTCOM &&
|
||||
this.gitHubVersion.type !== util.GitHubVariant.GHE_DOTCOM
|
||||
) {
|
||||
this.logger.debug(
|
||||
"Not running against github.com. Disabling all toggleable features.",
|
||||
);
|
||||
|
||||
@@ -6,7 +6,11 @@
|
||||
|
||||
import * as core from "@actions/core";
|
||||
|
||||
import { getTemporaryDirectory, printDebugLogs } from "./actions-util";
|
||||
import {
|
||||
restoreInputs,
|
||||
getTemporaryDirectory,
|
||||
printDebugLogs,
|
||||
} from "./actions-util";
|
||||
import { getGitHubVersion } from "./api-client";
|
||||
import { Config, getConfig } from "./config-utils";
|
||||
import * as debugArtifacts from "./debug-artifacts";
|
||||
@@ -42,6 +46,9 @@ async function runWrapper() {
|
||||
| initActionPostHelper.UploadFailedSarifResult
|
||||
| undefined;
|
||||
try {
|
||||
// Restore inputs from `init` Action.
|
||||
restoreInputs();
|
||||
|
||||
const gitHubVersion = await getGitHubVersion();
|
||||
checkGitHubVersionInRange(gitHubVersion, logger);
|
||||
|
||||
|
||||
@@ -12,10 +12,17 @@ import {
|
||||
getOptionalInput,
|
||||
getRequiredInput,
|
||||
getTemporaryDirectory,
|
||||
persistInputs,
|
||||
} from "./actions-util";
|
||||
import { getGitHubVersion } from "./api-client";
|
||||
import {
|
||||
getDependencyCachingEnabled,
|
||||
getTotalCacheSize,
|
||||
shouldRestoreCache,
|
||||
} from "./caching-utils";
|
||||
import { CodeQL } from "./codeql";
|
||||
import * as configUtils from "./config-utils";
|
||||
import { downloadDependencyCaches } from "./dependency-caching";
|
||||
import {
|
||||
addDiagnostic,
|
||||
flushDiagnostics,
|
||||
@@ -45,7 +52,6 @@ import {
|
||||
import { ZstdAvailability } from "./tar";
|
||||
import { ToolsDownloadStatusReport } from "./tools-download";
|
||||
import { ToolsFeature } from "./tools-features";
|
||||
import { getTotalCacheSize } from "./trap-caching";
|
||||
import {
|
||||
checkDiskUsage,
|
||||
checkForTimeout,
|
||||
@@ -224,7 +230,7 @@ async function sendCompletedStatusReport(
|
||||
packs: JSON.stringify(packs),
|
||||
trap_cache_languages: Object.keys(config.trapCaches).join(","),
|
||||
trap_cache_download_size_bytes: Math.round(
|
||||
await getTotalCacheSize(config.trapCaches, logger),
|
||||
await getTotalCacheSize(Object.values(config.trapCaches), logger),
|
||||
),
|
||||
trap_cache_download_duration_ms: Math.round(config.trapCacheDownloadTime),
|
||||
query_filters: JSON.stringify(
|
||||
@@ -250,6 +256,9 @@ async function run() {
|
||||
const logger = getActionsLogger();
|
||||
initializeEnvironment(getActionVersion());
|
||||
|
||||
// Make inputs accessible in the `post` step.
|
||||
persistInputs();
|
||||
|
||||
let config: configUtils.Config | undefined;
|
||||
let codeql: CodeQL;
|
||||
let toolsDownloadStatusReport: ToolsDownloadStatusReport | undefined;
|
||||
@@ -335,6 +344,7 @@ async function run() {
|
||||
dbLocation: getOptionalInput("db-location"),
|
||||
configInput: getOptionalInput("config"),
|
||||
trapCachingEnabled: getTrapCachingEnabled(),
|
||||
dependencyCachingEnabled: getDependencyCachingEnabled(),
|
||||
// Debug mode is enabled if:
|
||||
// - The `init` Action is passed `debug: true`.
|
||||
// - Actions step debugging is enabled (e.g. by [enabling debug logging for a rerun](https://docs.github.com/en/actions/managing-workflow-runs/re-running-workflows-and-jobs#re-running-all-the-jobs-in-a-workflow),
|
||||
@@ -542,6 +552,21 @@ async function run() {
|
||||
}
|
||||
}
|
||||
|
||||
// Set CODEQL_EXTRACTOR_CPP_BUILD_MODE_NONE
|
||||
if (config.languages.includes(Language.cpp)) {
|
||||
const bmnVar = "CODEQL_EXTRACTOR_CPP_BUILD_MODE_NONE";
|
||||
const value =
|
||||
process.env[bmnVar] ||
|
||||
(await features.getValue(Feature.CppBuildModeNone, codeql));
|
||||
logger.info(`Setting C++ build-mode: none to ${value}`);
|
||||
core.exportVariable(bmnVar, value);
|
||||
}
|
||||
|
||||
// Restore dependency cache(s), if they exist.
|
||||
if (shouldRestoreCache(config.dependencyCachingEnabled)) {
|
||||
await downloadDependencyCaches(config.languages, logger);
|
||||
}
|
||||
|
||||
// For CLI versions <2.15.1, build tracing caused errors in MacOS ARM machines with
|
||||
// System Integrity Protection (SIP) disabled.
|
||||
if (
|
||||
|
||||
@@ -141,7 +141,7 @@ test("getCodeQLSource correctly returns bundled CLI version when tools == latest
|
||||
|
||||
// Afterwards, ensure that we see the deprecation message in the log.
|
||||
const expected_message: string =
|
||||
"`tools: latest` has been renamed to `tools: linked`, but the old name is still supported for now. No action is required.";
|
||||
"`tools: latest` has been renamed to `tools: linked`, but the old name is still supported. No action is required.";
|
||||
t.assert(
|
||||
loggedMessages.some(
|
||||
(msg) =>
|
||||
@@ -249,3 +249,13 @@ test("setupCodeQLBundle logs the CodeQL CLI version being used when asked to dow
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test('tryGetTagNameFromUrl extracts the right tag name for a repo name containing "codeql-bundle"', (t) => {
|
||||
t.is(
|
||||
setupCodeql.tryGetTagNameFromUrl(
|
||||
"https://github.com/org/codeql-bundle-testing/releases/download/codeql-bundle-v2.19.0/codeql-bundle-linux64.tar.zst",
|
||||
getRunnerLogger(true),
|
||||
),
|
||||
"codeql-bundle-v2.19.0",
|
||||
);
|
||||
});
|
||||
|
||||
@@ -138,12 +138,30 @@ function tryGetBundleVersionFromTagName(
|
||||
return match[1];
|
||||
}
|
||||
|
||||
function tryGetTagNameFromUrl(url: string, logger: Logger): string | undefined {
|
||||
const match = url.match(/\/(codeql-bundle-.*)\//);
|
||||
if (match === null || match.length < 2) {
|
||||
export function tryGetTagNameFromUrl(
|
||||
url: string,
|
||||
logger: Logger,
|
||||
): string | undefined {
|
||||
const matches = [...url.matchAll(/\/(codeql-bundle-[^/]*)\//g)];
|
||||
if (!matches.length) {
|
||||
logger.debug(`Could not determine tag name for URL ${url}.`);
|
||||
return undefined;
|
||||
}
|
||||
// Example: https://github.com/org/codeql-bundle-testing/releases/download/codeql-bundle-v2.19.0/codeql-bundle-linux64.tar.zst
|
||||
// We require a trailing forward slash to be part of the match, so the last match gives us the tag
|
||||
// name. An alternative approach would be to also match against `/releases/`, but this approach
|
||||
// assumes less about the structure of the URL.
|
||||
const match = matches[matches.length - 1];
|
||||
|
||||
if (match === null || match.length !== 2) {
|
||||
logger.debug(
|
||||
`Could not determine tag name for URL ${url}. Matched ${JSON.stringify(
|
||||
match,
|
||||
)}.`,
|
||||
);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return match[1];
|
||||
}
|
||||
|
||||
@@ -274,13 +292,12 @@ export async function getCodeQLSource(
|
||||
toolsInput && CODEQL_BUNDLE_VERSION_ALIAS.includes(toolsInput);
|
||||
if (forceShippedTools) {
|
||||
logger.info(
|
||||
`Overriding the version of the CodeQL tools by ${defaultCliVersion.cliVersion}, the version shipped with the Action since ` +
|
||||
`tools: ${toolsInput} was requested.`,
|
||||
`'tools: ${toolsInput}' was requested, so using CodeQL version ${defaultCliVersion.cliVersion}, the version shipped with the Action.`,
|
||||
);
|
||||
|
||||
if (toolsInput === "latest") {
|
||||
logger.warning(
|
||||
"`tools: latest` has been renamed to `tools: linked`, but the old name is still supported for now. No action is required.",
|
||||
"`tools: latest` has been renamed to `tools: linked`, but the old name is still supported. No action is required.",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,6 +20,8 @@ import {
|
||||
|
||||
async function runWrapper() {
|
||||
try {
|
||||
// Restore inputs from `start-proxy` Action.
|
||||
actionsUtil.restoreInputs();
|
||||
const pid = core.getState("proxy-process-pid");
|
||||
if (pid) {
|
||||
process.kill(Number(pid));
|
||||
|
||||
@@ -91,6 +91,9 @@ function generateCertificateAuthority(): CertificateAuthority {
|
||||
}
|
||||
|
||||
async function runWrapper() {
|
||||
// Make inputs accessible in the `post` step.
|
||||
actionsUtil.persistInputs();
|
||||
|
||||
const logger = getActionsLogger();
|
||||
|
||||
// Setup logging for the proxy
|
||||
|
||||
23
src/tar.ts
23
src/tar.ts
@@ -125,27 +125,30 @@ export async function extract(
|
||||
"Could not determine tar version, which is required to extract a Zstandard archive.",
|
||||
);
|
||||
}
|
||||
return await extractTarZst(
|
||||
fs.createReadStream(tarPath),
|
||||
tarVersion,
|
||||
logger,
|
||||
);
|
||||
return await extractTarZst(tarPath, tarVersion, logger);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract a compressed tar archive
|
||||
*
|
||||
* @param file path to the tar
|
||||
* @param tar tar stream, or path to the tar
|
||||
* @param dest destination directory. Optional.
|
||||
* @returns path to the destination directory
|
||||
*/
|
||||
export async function extractTarZst(
|
||||
tarStream: stream.Readable,
|
||||
tar: stream.Readable | string,
|
||||
tarVersion: TarVersion,
|
||||
logger: Logger,
|
||||
): Promise<string> {
|
||||
const dest = await createExtractFolder();
|
||||
logger.debug(
|
||||
`Extracting to ${dest}.${
|
||||
tar instanceof stream.Readable
|
||||
? ` Input stream has high water mark ${tar.readableHighWaterMark}.`
|
||||
: ""
|
||||
}`,
|
||||
);
|
||||
|
||||
try {
|
||||
// Initialize args
|
||||
@@ -157,7 +160,7 @@ export async function extractTarZst(
|
||||
args.push("--overwrite");
|
||||
}
|
||||
|
||||
args.push("-f", "-", "-C", dest);
|
||||
args.push("-f", tar instanceof stream.Readable ? "-" : tar, "-C", dest);
|
||||
|
||||
process.stdout.write(`[command]tar ${args.join(" ")}\n`);
|
||||
|
||||
@@ -175,7 +178,9 @@ export async function extractTarZst(
|
||||
process.stdout.write(data);
|
||||
});
|
||||
|
||||
tarStream.pipe(tarProcess.stdin);
|
||||
if (tar instanceof stream.Readable) {
|
||||
tar.pipe(tarProcess.stdin);
|
||||
}
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
tarProcess.on("exit", (code) => {
|
||||
|
||||
@@ -331,6 +331,7 @@ export function createTestConfig(overrides: Partial<Config>): Config {
|
||||
},
|
||||
trapCaches: {},
|
||||
trapCacheDownloadTime: 0,
|
||||
dependencyCachingEnabled: false,
|
||||
},
|
||||
overrides,
|
||||
);
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { IncomingMessage, OutgoingHttpHeaders } from "http";
|
||||
import { IncomingMessage, OutgoingHttpHeaders, RequestOptions } from "http";
|
||||
import * as path from "path";
|
||||
import { performance } from "perf_hooks";
|
||||
|
||||
@@ -11,6 +11,11 @@ import { formatDuration, Logger } from "./logging";
|
||||
import * as tar from "./tar";
|
||||
import { cleanUpGlob } from "./util";
|
||||
|
||||
/**
|
||||
* High watermark to use when streaming the download and extraction of the CodeQL tools.
|
||||
*/
|
||||
export const STREAMING_HIGH_WATERMARK_BYTES = 4 * 1024 * 1024; // 4 MiB
|
||||
|
||||
/**
|
||||
* Timing information for the download and extraction of the CodeQL tools when
|
||||
* we fully download the bundle before extracting.
|
||||
@@ -86,6 +91,7 @@ export async function downloadAndExtract(
|
||||
|
||||
if (
|
||||
compressionMethod === "zstd" &&
|
||||
process.platform === "linux" &&
|
||||
(await features.getValue(Feature.ZstdBundleStreamingExtraction))
|
||||
) {
|
||||
logger.info(`Streaming the extraction of the CodeQL bundle.`);
|
||||
@@ -182,7 +188,14 @@ async function downloadAndExtractZstdWithStreaming(
|
||||
headers,
|
||||
);
|
||||
const response = await new Promise<IncomingMessage>((resolve) =>
|
||||
https.get(codeqlURL, { headers }, (r) => resolve(r)),
|
||||
https.get(
|
||||
codeqlURL,
|
||||
{
|
||||
headers,
|
||||
highWaterMark: STREAMING_HIGH_WATERMARK_BYTES,
|
||||
} as unknown as RequestOptions,
|
||||
(r) => resolve(r),
|
||||
),
|
||||
);
|
||||
|
||||
if (response.statusCode !== 200) {
|
||||
|
||||
@@ -312,18 +312,6 @@ export async function getLanguagesSupportingCaching(
|
||||
return result;
|
||||
}
|
||||
|
||||
export async function getTotalCacheSize(
|
||||
trapCaches: Partial<Record<Language, string>>,
|
||||
logger: Logger,
|
||||
): Promise<number> {
|
||||
const sizes = await Promise.all(
|
||||
Object.values(trapCaches).map((cacheDir) =>
|
||||
tryGetFolderBytes(cacheDir, logger),
|
||||
),
|
||||
);
|
||||
return sizes.map((a) => a || 0).reduce((a, b) => a + b, 0);
|
||||
}
|
||||
|
||||
async function cacheKey(
|
||||
codeql: CodeQL,
|
||||
language: Language,
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
*/
|
||||
import * as core from "@actions/core";
|
||||
|
||||
import * as actionsUtil from "./actions-util";
|
||||
import { getTemporaryDirectory } from "./actions-util";
|
||||
import { getGitHubVersion } from "./api-client";
|
||||
import * as debugArtifacts from "./debug-artifacts";
|
||||
@@ -20,6 +21,8 @@ import {
|
||||
|
||||
async function runWrapper() {
|
||||
try {
|
||||
// Restore inputs from `upload-sarif` Action.
|
||||
actionsUtil.restoreInputs();
|
||||
const logger = getActionsLogger();
|
||||
const gitHubVersion = await getGitHubVersion();
|
||||
checkGitHubVersionInRange(gitHubVersion, logger);
|
||||
|
||||
@@ -60,6 +60,9 @@ async function run() {
|
||||
const gitHubVersion = await getGitHubVersion();
|
||||
checkActionVersion(getActionVersion(), gitHubVersion);
|
||||
|
||||
// Make inputs accessible in the `post` step.
|
||||
actionsUtil.persistInputs();
|
||||
|
||||
const repositoryNwo = parseRepositoryNwo(
|
||||
getRequiredEnvParam("GITHUB_REPOSITORY"),
|
||||
);
|
||||
|
||||
Reference in New Issue
Block a user