github/codeql-action
1
0
Fork 0
mirror of https://github.com/github/codeql-action.git synced 2025-12-27 17:50:07 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2475 from github/angelapwen/refactor-debug-artifacts-upload

Browse Source 
Refactor: prepare debug artifacts for `artifact` upgrades
This commit is contained in:
Angela P Wen
2024-09-13 09:47:26 -07:00
committed by GitHub
parent c101242d73 bc660fcf8c
commit fe22310da9
33 changed files with 228 additions and 494 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
src/analyze-action-post-helper.test.ts
View File

@@ -1,57 +0,0 @@
import test from "ava";
import * as sinon from "sinon";
import * as actionsUtil from "./actions-util";
import * as analyzeActionPostHelper from "./analyze-action-post-helper";
import * as configUtils from "./config-utils";
import { setupTests } from "./testing-utils";
import * as util from "./util";
setupTests(test);
test("post: analyze action with debug mode off", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
process.env["RUNNER_TEMP"] = tmpDir;
const gitHubVersion: util.GitHubVersion = {
type: util.GitHubVariant.DOTCOM,
};
sinon.stub(configUtils, "getConfig").resolves({
debugMode: false,
gitHubVersion,
languages: [],
packs: [],
} as unknown as configUtils.Config);
const uploadSarifSpy = sinon.spy();
await analyzeActionPostHelper.run(uploadSarifSpy);
t.assert(uploadSarifSpy.notCalled);
});
});
test("post: analyze action with debug mode on", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
process.env["RUNNER_TEMP"] = tmpDir;
const gitHubVersion: util.GitHubVersion = {
type: util.GitHubVariant.DOTCOM,
};
sinon.stub(configUtils, "getConfig").resolves({
debugMode: true,
gitHubVersion,
languages: [],
packs: [],
} as unknown as configUtils.Config);
const requiredInputStub = sinon.stub(actionsUtil, "getRequiredInput");
requiredInputStub.withArgs("output").returns("fake-output-dir");
const uploadSarifSpy = sinon.spy();
await analyzeActionPostHelper.run(uploadSarifSpy);
t.assert(uploadSarifSpy.called);
});
});

30
src/analyze-action-post-helper.ts
View File

@@ -1,30 +0,0 @@
import * as core from "@actions/core";
import * as actionsUtil from "./actions-util";
import { Config, getConfig } from "./config-utils";
import { getActionsLogger } from "./logging";
export async function run(
uploadSarifDebugArtifact: (
config: Config,
outputDir: string,
) => Promise<void>,
) {
const logger = getActionsLogger();
const config = await getConfig(actionsUtil.getTemporaryDirectory(), logger);
if (config === undefined) {
throw new Error(
"Config file could not be found at expected location. Did the 'init' action fail to start?",
);
}
// Upload Actions SARIF artifacts for debugging
if (config?.debugMode) {
core.info(
"Debug mode is on. Uploading available SARIF files as Actions debugging artifact...",
);
const outputDir = actionsUtil.getRequiredInput("output");
await uploadSarifDebugArtifact(config, outputDir);
}
}

15
src/analyze-action-post.ts
View File

@@ -5,20 +5,17 @@
*/
import * as core from "@actions/core";
import * as analyzeActionPostHelper from "./analyze-action-post-helper";
import * as debugArtifacts from "./debug-artifacts";
import * as uploadSarifActionPostHelper from "./upload-sarif-action-post-helper";
import { EnvVar } from "./environment";
import { wrapError } from "./util";
async function runWrapper() {
try {
await analyzeActionPostHelper.run(debugArtifacts.uploadSarifDebugArtifact);
// Also run the upload-sarif post action since we're potentially running
// the same steps in the analyze action.
await uploadSarifActionPostHelper.uploadArtifacts(
debugArtifacts.uploadDebugArtifacts,
);
// Upload SARIF artifacts if we determine that this is a first-party analysis run.
// For third-party runs, this artifact will be uploaded in the `upload-sarif-post` step.
if (process.env[EnvVar.INIT_ACTION_HAS_RUN] === "true") {
await debugArtifacts.uploadCombinedSarifArtifacts();
}
} catch (error) {
core.setFailed(
`analyze post-action step failed: ${wrapError(error).message}`,

1
src/analyze-action.ts
View File

@@ -230,6 +230,7 @@ async function run() {
const apiDetails = getApiDetails();
const outputDir = actionsUtil.getRequiredInput("output");
core.exportVariable(EnvVar.SARIF_RESULTS_OUTPUT_DIR, outputDir);
const threads = util.getThreadsFlag(
actionsUtil.getOptionalInput("threads") || process.env["CODEQL_THREADS"],
logger,

175
src/debug-artifacts.ts
View File

@@ -6,10 +6,11 @@ import * as core from "@actions/core";
import AdmZip from "adm-zip";
import del from "del";
import { getRequiredInput } from "./actions-util";
import { getRequiredInput, getTemporaryDirectory } from "./actions-util";
import { dbIsFinalized } from "./analyze";
import { getCodeQL } from "./codeql";
import { Config } from "./config-utils";
import { EnvVar } from "./environment";
import { Language } from "./languages";
import { Logger } from "./logging";
import {
@@ -23,6 +24,106 @@ export function sanitizeArifactName(name: string): string {
return name.replace(/[^a-zA-Z0-9_\\-]+/g, "");
}
/**
* Upload Actions SARIF artifacts for debugging when CODEQL_ACTION_DEBUG_COMBINED_SARIF
* environment variable is set
*/
export async function uploadCombinedSarifArtifacts() {
const tempDir = getTemporaryDirectory();
// Upload Actions SARIF artifacts for debugging when environment variable is set
if (process.env["CODEQL_ACTION_DEBUG_COMBINED_SARIF"] === "true") {
core.info(
"Uploading available combined SARIF files as Actions debugging artifact...",
);
const baseTempDir = path.resolve(tempDir, "combined-sarif");
const toUpload: string[] = [];
if (fs.existsSync(baseTempDir)) {
const outputDirs = fs.readdirSync(baseTempDir);
for (const outputDir of outputDirs) {
const sarifFiles = fs
.readdirSync(path.resolve(baseTempDir, outputDir))
.filter((f) => f.endsWith(".sarif"));
for (const sarifFile of sarifFiles) {
toUpload.push(path.resolve(baseTempDir, outputDir, sarifFile));
}
}
}
if (toUpload.length > 0) {
await uploadDebugArtifacts(
toUpload,
baseTempDir,
"combined-sarif-artifacts",
);
}
}
}
export async function uploadAllAvailableDebugArtifacts(
config: Config,
logger: Logger,
) {
const filesToUpload: string[] = [];
const analyzeActionOutputDir = process.env[EnvVar.SARIF_RESULTS_OUTPUT_DIR];
for (const lang of config.languages) {
// Add any SARIF files, if they exist
if (
analyzeActionOutputDir !== undefined &&
fs.existsSync(analyzeActionOutputDir) &&
fs.lstatSync(analyzeActionOutputDir).isDirectory()
) {
const sarifFile = path.resolve(analyzeActionOutputDir, `${lang}.sarif`);
// Move SARIF to DB location so that they can be uploaded with the same root directory as the other artifacts.
if (fs.existsSync(sarifFile)) {
const sarifInDbLocation = path.resolve(
config.dbLocation,
`${lang}.sarif`,
);
fs.copyFileSync(sarifFile, sarifInDbLocation);
filesToUpload.push(sarifInDbLocation);
}
}
// Add any log files
const databaseDirectory = getCodeQLDatabasePath(config, lang);
const logsDirectory = path.resolve(databaseDirectory, "log");
if (doesDirectoryExist(logsDirectory)) {
filesToUpload.push(...listFolder(logsDirectory));
}
// Multilanguage tracing: there are additional logs in the root of the cluster
const multiLanguageTracingLogsDirectory = path.resolve(
config.dbLocation,
"log",
);
if (doesDirectoryExist(multiLanguageTracingLogsDirectory)) {
filesToUpload.push(...listFolder(multiLanguageTracingLogsDirectory));
}
// Add database bundle
let databaseBundlePath: string;
if (!dbIsFinalized(config, lang, logger)) {
databaseBundlePath = await createPartialDatabaseBundle(config, lang);
} else {
databaseBundlePath = await createDatabaseBundleCli(config, lang);
}
filesToUpload.push(databaseBundlePath);
}
await uploadDebugArtifacts(
filesToUpload,
config.dbLocation,
config.debugArtifactName,
);
}
export async function uploadDebugArtifacts(
toUpload: string[],
rootDir: string,
@@ -63,50 +164,6 @@ export async function uploadDebugArtifacts(
}
}
export async function uploadSarifDebugArtifact(
config: Config,
outputDir: string,
) {
if (!doesDirectoryExist(outputDir)) {
return;
}
let toUpload: string[] = [];
for (const lang of config.languages) {
const sarifFile = path.resolve(outputDir, `${lang}.sarif`);
if (fs.existsSync(sarifFile)) {
toUpload = toUpload.concat(sarifFile);
}
}
await uploadDebugArtifacts(toUpload, outputDir, config.debugArtifactName);
}
export async function uploadLogsDebugArtifact(config: Config) {
let toUpload: string[] = [];
for (const language of config.languages) {
const databaseDirectory = getCodeQLDatabasePath(config, language);
const logsDirectory = path.resolve(databaseDirectory, "log");
if (doesDirectoryExist(logsDirectory)) {
toUpload = toUpload.concat(listFolder(logsDirectory));
}
}
// Multilanguage tracing: there are additional logs in the root of the cluster
const multiLanguageTracingLogsDirectory = path.resolve(
config.dbLocation,
"log",
);
if (doesDirectoryExist(multiLanguageTracingLogsDirectory)) {
toUpload = toUpload.concat(listFolder(multiLanguageTracingLogsDirectory));
}
await uploadDebugArtifacts(
toUpload,
config.dbLocation,
config.debugArtifactName,
);
}
/**
* If a database has not been finalized, we cannot run the `codeql database bundle`
* command in the CLI because it will return an error. Instead we directly zip
@@ -150,31 +207,3 @@ async function createDatabaseBundleCli(
);
return databaseBundlePath;
}
export async function uploadDatabaseBundleDebugArtifact(
config: Config,
logger: Logger,
) {
for (const language of config.languages) {
try {
let databaseBundlePath: string;
if (!dbIsFinalized(config, language, logger)) {
databaseBundlePath = await createPartialDatabaseBundle(
config,
language,
);
} else {
databaseBundlePath = await createDatabaseBundleCli(config, language);
}
await uploadDebugArtifacts(
[databaseBundlePath],
config.dbLocation,
config.debugArtifactName,
);
} catch (error) {
core.info(
`Failed to upload database debug bundle for ${config.debugDatabaseName}-${language}: ${error}`,
);
}
}
}

3
src/environment.ts
View File

@@ -64,6 +64,9 @@ export enum EnvVar {
ODASA_TRACER_CONFIGURATION = "ODASA_TRACER_CONFIGURATION",
/** The value of the `output` input for the analyze action. */
SARIF_RESULTS_OUTPUT_DIR = "CODEQL_ACTION_SARIF_RESULTS_OUTPUT_DIR",
/**
* What percentage of the total amount of RAM over 8 GB that the Action should reserve for the
* system.

18
src/init-action-post-helper.test.ts
View File

@@ -35,13 +35,11 @@ test("post: init action with debug mode off", async (t) => {
packs: [],
} as unknown as configUtils.Config);
const uploadDatabaseBundleSpy = sinon.spy();
const uploadLogsSpy = sinon.spy();
const uploadAllAvailableDebugArtifactsSpy = sinon.spy();
const printDebugLogsSpy = sinon.spy();
await initActionPostHelper.run(
uploadDatabaseBundleSpy,
uploadLogsSpy,
uploadAllAvailableDebugArtifactsSpy,
printDebugLogsSpy,
createTestConfig({ debugMode: false }),
parseRepositoryNwo("github/codeql-action"),
@@ -49,8 +47,7 @@ test("post: init action with debug mode off", async (t) => {
getRunnerLogger(true),
);
t.assert(uploadDatabaseBundleSpy.notCalled);
t.assert(uploadLogsSpy.notCalled);
t.assert(uploadAllAvailableDebugArtifactsSpy.notCalled);
t.assert(printDebugLogsSpy.notCalled);
});
});
@@ -60,13 +57,11 @@ test("post: init action with debug mode on", async (t) => {
process.env["GITHUB_REPOSITORY"] = "github/codeql-action-fake-repository";
process.env["RUNNER_TEMP"] = tmpDir;
const uploadDatabaseBundleSpy = sinon.spy();
const uploadLogsSpy = sinon.spy();
const uploadAllAvailableDebugArtifactsSpy = sinon.spy();
const printDebugLogsSpy = sinon.spy();
await initActionPostHelper.run(
uploadDatabaseBundleSpy,
uploadLogsSpy,
uploadAllAvailableDebugArtifactsSpy,
printDebugLogsSpy,
createTestConfig({ debugMode: true }),
parseRepositoryNwo("github/codeql-action"),
@@ -74,8 +69,7 @@ test("post: init action with debug mode on", async (t) => {
getRunnerLogger(true),
);
t.assert(uploadDatabaseBundleSpy.called);
t.assert(uploadLogsSpy.called);
t.assert(uploadAllAvailableDebugArtifactsSpy.called);
t.assert(printDebugLogsSpy.called);
});
});

7
src/init-action-post-helper.ts
View File

@@ -158,11 +158,10 @@ export async function tryUploadSarifIfRunFailed(
}
export async function run(
uploadDatabaseBundleDebugArtifact: (
uploadAllAvailableDebugArtifacts: (
config: Config,
logger: Logger,
) => Promise<void>,
uploadLogsDebugArtifact: (config: Config) => Promise<void>,
printDebugLogs: (config: Config) => Promise<void>,
config: Config,
repositoryNwo: RepositoryNwo,
@@ -211,9 +210,7 @@ export async function run(
logger.info(
"Debug mode is on. Uploading available database bundles and logs as Actions debugging artifacts...",
);
await uploadDatabaseBundleDebugArtifact(config, logger);
await uploadLogsDebugArtifact(config);
await uploadAllAvailableDebugArtifacts(config, logger);
await printDebugLogs(config);
}

3
src/init-action-post.ts
View File

@@ -64,8 +64,7 @@ async function runWrapper() {
}
uploadFailedSarifResult = await initActionPostHelper.run(
debugArtifacts.uploadDatabaseBundleDebugArtifact,
debugArtifacts.uploadLogsDebugArtifact,
debugArtifacts.uploadAllAvailableDebugArtifacts,
printDebugLogs,
config,
repositoryNwo,

49
src/upload-sarif-action-post-helper.ts
View File

@@ -1,49 +0,0 @@
import * as fs from "fs";
import * as path from "path";
import * as core from "@actions/core";
import * as actionsUtil from "./actions-util";
export async function uploadArtifacts(
uploadDebugArtifacts: (
toUpload: string[],
rootDir: string,
artifactName: string,
) => Promise<void>,
) {
const tempDir = actionsUtil.getTemporaryDirectory();
// Upload Actions SARIF artifacts for debugging when environment variable is set
if (process.env["CODEQL_ACTION_DEBUG_COMBINED_SARIF"] === "true") {
core.info(
"Uploading available combined SARIF files as Actions debugging artifact...",
);
const baseTempDir = path.resolve(tempDir, "combined-sarif");
const toUpload: string[] = [];
if (fs.existsSync(baseTempDir)) {
const outputDirs = fs.readdirSync(baseTempDir);
for (const outputDir of outputDirs) {
const sarifFiles = fs
.readdirSync(path.resolve(baseTempDir, outputDir))
.filter((f) => f.endsWith(".sarif"));
for (const sarifFile of sarifFiles) {
toUpload.push(path.resolve(baseTempDir, outputDir, sarifFile));
}
}
}
if (toUpload.length > 0) {
await uploadDebugArtifacts(
toUpload,
baseTempDir,
"upload-debug-artifacts",
);
}
}
}

10
src/upload-sarif-action-post.ts
View File

@@ -6,14 +6,16 @@
import * as core from "@actions/core";
import * as debugArtifacts from "./debug-artifacts";
import * as uploadSarifActionPostHelper from "./upload-sarif-action-post-helper";
import { EnvVar } from "./environment";
import { wrapError } from "./util";
async function runWrapper() {
try {
await uploadSarifActionPostHelper.uploadArtifacts(
debugArtifacts.uploadDebugArtifacts,
);
// Upload SARIF artifacts if we determine that this is a third-party analysis run.
// For first-party runs, this artifact will be uploaded in the `analyze-post` step.
if (process.env[EnvVar.INIT_ACTION_HAS_RUN] !== "true") {
await debugArtifacts.uploadCombinedSarifArtifacts();
}
} catch (error) {
core.setFailed(
`upload-sarif post-action step failed: ${wrapError(error).message}`,