github/codeql-action
1
0
Fork 0
mirror of https://github.com/github/codeql-action.git synced 2026-01-04 13:40:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add an option to upload some debugging artifacts

Browse Source
This commit is contained in:
Edoardo Pirovano
2021-10-28 14:15:22 +01:00
parent 4293754ed2
commit bc31f604d3
40 changed files with 374 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/analysis-paths.test.ts
View File

@@ -22,6 +22,7 @@ test("emptyPaths", async (t) => {
gitHubVersion: { type: util.GitHubVariant.DOTCOM } as util.GitHubVersion,
dbLocation: path.resolve(tmpDir, "codeql_databases"),
packs: {},
debugMode: false,
};
analysisPaths.includeAndExcludeAnalysisPaths(config);
t.is(process.env["LGTM_INDEX_INCLUDE"], undefined);
@@ -44,6 +45,7 @@ test("nonEmptyPaths", async (t) => {
gitHubVersion: { type: util.GitHubVariant.DOTCOM } as util.GitHubVersion,
dbLocation: path.resolve(tmpDir, "codeql_databases"),
packs: {},
debugMode: false,
};
analysisPaths.includeAndExcludeAnalysisPaths(config);
t.is(process.env["LGTM_INDEX_INCLUDE"], "path1\npath2");
@@ -70,6 +72,7 @@ test("exclude temp dir", async (t) => {
gitHubVersion: { type: util.GitHubVariant.DOTCOM } as util.GitHubVersion,
dbLocation: path.resolve(tempDir, "codeql_databases"),
packs: {},
debugMode: false,
};
analysisPaths.includeAndExcludeAnalysisPaths(config);
t.is(process.env["LGTM_INDEX_INCLUDE"], undefined);

71
src/analyze-action.ts
View File

@@ -1,6 +1,7 @@
import * as fs from "fs";
import * as path from "path";
import * as artifact from "@actions/artifact";
import * as core from "@actions/core";
import * as actionsUtil from "./actions-util";
@@ -11,7 +12,7 @@ import {
runQueries,
runFinalize,
} from "./analyze";
import { getCodeQL } from "./codeql";
import { CODEQL_VERSION_NEW_TRACING, getCodeQL } from "./codeql";
import { Config, getConfig } from "./config-utils";
import { uploadDatabases } from "./database-upload";
import { getActionsLogger } from "./logging";
@@ -19,6 +20,7 @@ import { parseRepositoryNwo } from "./repository";
import * as upload_lib from "./upload-lib";
import { UploadStatusReport } from "./upload-lib";
import * as util from "./util";
import { bundleDb, codeQlVersionAbove, DEBUG_ARTIFACT_NAME } from "./util";
// eslint-disable-next-line import/no-commonjs
const pkg = require("../package.json");
@@ -108,6 +110,42 @@ async function run() {
config,
logger
);
if (config.debugMode) {
// Upload the SARIF files as an Actions artifact for debugging
await uploadDebugArtifacts(
config.languages.map((lang) =>
path.resolve(outputDir, `${lang}.sarif`)
),
outputDir
);
}
}
const codeql = await getCodeQL(config.codeQLCmd);
if (config.debugMode) {
// Upload the logs as an Actions artifact for debugging
const toUpload: string[] = [];
for (const language of config.languages) {
toUpload.push(
...listFolder(
path.resolve(util.getCodeQLDatabasePath(config, language), "log")
)
);
}
if (await codeQlVersionAbove(codeql, CODEQL_VERSION_NEW_TRACING)) {
// Multilanguage tracing: there are additional logs in the root of the cluster
toUpload.push(...listFolder(path.resolve(config.dbLocation, "log")));
}
await uploadDebugArtifacts(toUpload, config.dbLocation);
if (!(await codeQlVersionAbove(codeql, CODEQL_VERSION_NEW_TRACING))) {
// Before multi-language tracing, we wrote a compound-build-tracer.log in the temp dir
await uploadDebugArtifacts(
[path.resolve(config.tempDir, "compound-build-tracer.log")],
config.tempDir
);
}
}
if (actionsUtil.getOptionalInput("cleanup-level") !== "none") {
@@ -138,7 +176,15 @@ async function run() {
const repositoryNwo = parseRepositoryNwo(
util.getRequiredEnvParam("GITHUB_REPOSITORY")
);
await uploadDatabases(repositoryNwo, config, apiDetails, logger);
await uploadDatabases(repositoryNwo, config, apiDetails, logger); // Possibly upload the database bundles for remote queries
if (config.debugMode) {
// Upload the database bundles as an Actions artifact for debugging
const toUpload: string[] = [];
for (const language of config.languages)
toUpload.push(await bundleDb(config, language, codeql));
await uploadDebugArtifacts(toUpload, config.dbLocation);
}
} catch (origError) {
const error =
origError instanceof Error ? origError : new Error(String(origError));
@@ -190,6 +236,27 @@ async function run() {
}
}
async function uploadDebugArtifacts(toUpload: string[], rootDir: string) {
await artifact.create().uploadArtifact(
DEBUG_ARTIFACT_NAME,
toUpload.map((file) => path.normalize(file)),
path.normalize(rootDir)
);
}
function listFolder(dir: string): string[] {
const entries = fs.readdirSync(dir, { withFileTypes: true });
const files: string[] = [];
for (const entry of entries) {
if (entry.isFile()) {
files.push(path.resolve(dir, entry.name));
} else if (entry.isDirectory()) {
files.push(...listFolder(path.resolve(dir, entry.name)));
}
}
return files;
}
export const runPromise = run();
async function runWrapper() {

1
src/analyze.test.ts
View File

@@ -119,6 +119,7 @@ test("status report fields and search path setting", async (t) => {
} as util.GitHubVersion,
dbLocation: path.resolve(tmpDir, "codeql_databases"),
packs,
debugMode: false,
};
fs.mkdirSync(util.getCodeQLDatabasePath(config, language), {
recursive: true,

23
src/config-utils.test.ts
View File

@@ -85,6 +85,7 @@ test("load empty config", async (t) => {
undefined,
undefined,
undefined,
false,
{ owner: "github", repo: "example " },
tmpDir,
tmpDir,
@@ -102,6 +103,7 @@ test("load empty config", async (t) => {
undefined,
undefined,
undefined,
false,
{ owner: "github", repo: "example " },
tmpDir,
tmpDir,
@@ -144,6 +146,7 @@ test("loading config saves config", async (t) => {
undefined,
undefined,
undefined,
false,
{ owner: "github", repo: "example " },
tmpDir,
tmpDir,
@@ -172,6 +175,7 @@ test("load input outside of workspace", async (t) => {
undefined,
"../input",
undefined,
false,
{ owner: "github", repo: "example " },
tmpDir,
tmpDir,
@@ -207,6 +211,7 @@ test("load non-local input with invalid repo syntax", async (t) => {
undefined,
configFile,
undefined,
false,
{ owner: "github", repo: "example " },
tmpDir,
tmpDir,
@@ -243,6 +248,7 @@ test("load non-existent input", async (t) => {
undefined,
configFile,
undefined,
false,
{ owner: "github", repo: "example " },
tmpDir,
tmpDir,
@@ -326,6 +332,7 @@ test("load non-empty input", async (t) => {
gitHubVersion,
dbLocation: path.resolve(tmpDir, "codeql_databases"),
packs: {} as configUtils.Packs,
debugMode: false,
};
const languages = "javascript";
@@ -337,6 +344,7 @@ test("load non-empty input", async (t) => {
undefined,
configFilePath,
undefined,
false,
{ owner: "github", repo: "example " },
tmpDir,
tmpDir,
@@ -400,6 +408,7 @@ test("Default queries are used", async (t) => {
undefined,
configFilePath,
undefined,
false,
{ owner: "github", repo: "example " },
tmpDir,
tmpDir,
@@ -471,6 +480,7 @@ test("Queries can be specified in config file", async (t) => {
undefined,
configFilePath,
undefined,
false,
{ owner: "github", repo: "example " },
tmpDir,
tmpDir,
@@ -536,6 +546,7 @@ test("Queries from config file can be overridden in workflow file", async (t) =>
undefined,
configFilePath,
undefined,
false,
{ owner: "github", repo: "example " },
tmpDir,
tmpDir,
@@ -599,6 +610,7 @@ test("Queries in workflow file can be used in tandem with the 'disable default q
undefined,
configFilePath,
undefined,
false,
{ owner: "github", repo: "example " },
tmpDir,
tmpDir,
@@ -655,6 +667,7 @@ test("Multiple queries can be specified in workflow file, no config file require
undefined,
undefined,
undefined,
false,
{ owner: "github", repo: "example " },
tmpDir,
tmpDir,
@@ -732,6 +745,7 @@ test("Queries in workflow file can be added to the set of queries without overri
undefined,
configFilePath,
undefined,
false,
{ owner: "github", repo: "example " },
tmpDir,
tmpDir,
@@ -799,6 +813,7 @@ test("Invalid queries in workflow file handled correctly", async (t) => {
undefined,
undefined,
undefined,
false,
{ owner: "github", repo: "example " },
tmpDir,
tmpDir,
@@ -863,6 +878,7 @@ test("API client used when reading remote config", async (t) => {
undefined,
configFile,
undefined,
false,
{ owner: "github", repo: "example " },
tmpDir,
tmpDir,
@@ -889,6 +905,7 @@ test("Remote config handles the case where a directory is provided", async (t) =
undefined,
repoReference,
undefined,
false,
{ owner: "github", repo: "example " },
tmpDir,
tmpDir,
@@ -923,6 +940,7 @@ test("Invalid format of remote config handled correctly", async (t) => {
undefined,
repoReference,
undefined,
false,
{ owner: "github", repo: "example " },
tmpDir,
tmpDir,
@@ -958,6 +976,7 @@ test("No detected languages", async (t) => {
undefined,
undefined,
undefined,
false,
{ owner: "github", repo: "example " },
tmpDir,
tmpDir,
@@ -985,6 +1004,7 @@ test("Unknown languages", async (t) => {
undefined,
undefined,
undefined,
false,
{ owner: "github", repo: "example " },
tmpDir,
tmpDir,
@@ -1034,6 +1054,7 @@ test("Config specifies packages", async (t) => {
undefined,
configFile,
undefined,
false,
{ owner: "github", repo: "example " },
tmpDir,
tmpDir,
@@ -1092,6 +1113,7 @@ test("Config specifies packages for multiple languages", async (t) => {
undefined,
configFile,
undefined,
false,
{ owner: "github", repo: "example" },
tmpDir,
tmpDir,
@@ -1166,6 +1188,7 @@ function doInvalidInputTest(
undefined,
configFile,
undefined,
false,
{ owner: "github", repo: "example " },
tmpDir,
tmpDir,

12
src/config-utils.ts
View File

@@ -125,6 +125,11 @@ export interface Config {
* List of packages, separated by language to download before any analysis.
*/
packs: Packs;
/**
* Specifies whether we are debugging mode and should try to produce extra
* output for debugging purposes when possible.
*/
debugMode: boolean;
}
export type Packs = Partial<Record<Language, PackWithVersion[]>>;
@@ -813,6 +818,7 @@ export async function getDefaultConfig(
queriesInput: string | undefined,
packsInput: string | undefined,
dbLocation: string | undefined,
debugMode: boolean,
repository: RepositoryNwo,
tempDir: string,
toolCacheDir: string,
@@ -864,6 +870,7 @@ export async function getDefaultConfig(
codeQLCmd: codeQL.getPath(),
gitHubVersion,
dbLocation: dbLocationOrDefault(dbLocation, tempDir),
debugMode,
};
}
@@ -876,6 +883,7 @@ async function loadConfig(
packsInput: string | undefined,
configFile: string,
dbLocation: string | undefined,
debugMode: boolean,
repository: RepositoryNwo,
tempDir: string,
toolCacheDir: string,
@@ -1032,6 +1040,7 @@ async function loadConfig(
codeQLCmd: codeQL.getPath(),
gitHubVersion,
dbLocation: dbLocationOrDefault(dbLocation, tempDir),
debugMode,
};
}
@@ -1201,6 +1210,7 @@ export async function initConfig(
packsInput: string | undefined,
configFile: string | undefined,
dbLocation: string | undefined,
debugMode: boolean,
repository: RepositoryNwo,
tempDir: string,
toolCacheDir: string,
@@ -1220,6 +1230,7 @@ export async function initConfig(
queriesInput,
packsInput,
dbLocation,
debugMode,
repository,
tempDir,
toolCacheDir,
@@ -1236,6 +1247,7 @@ export async function initConfig(
packsInput,
configFile,
dbLocation,
debugMode,
repository,
tempDir,
toolCacheDir,

1
src/database-upload.test.ts
View File

@@ -47,6 +47,7 @@ function getTestConfig(tmpDir: string): Config {
gitHubVersion: { type: GitHubVariant.DOTCOM },
dbLocation: tmpDir,
packs: {},
debugMode: false,
};
}

8
src/database-upload.ts
View File

@@ -7,6 +7,7 @@ import { Config } from "./config-utils";
import { Logger } from "./logging";
import { RepositoryNwo } from "./repository";
import * as util from "./util";
import { bundleDb } from "./util";
export async function uploadDatabases(
repositoryNwo: RepositoryNwo,
@@ -54,13 +55,8 @@ export async function uploadDatabases(
const codeql = await getCodeQL(config.codeQLCmd);
for (const language of config.languages) {
// Bundle the database up into a single zip file
const databasePath = util.getCodeQLDatabasePath(config, language);
const databaseBundlePath = `${databasePath}.zip`;
await codeql.databaseBundle(databasePath, databaseBundlePath);
// Upload the database bundle
const payload = fs.readFileSync(databaseBundlePath);
const payload = fs.readFileSync(await bundleDb(config, language, codeql));
try {
await client.request(
`PUT /repos/:owner/:repo/code-scanning/codeql/databases/:language`,

1
src/init-action.ts
View File

@@ -163,6 +163,7 @@ async function run() {
getOptionalInput("packs"),
getOptionalInput("config-file"),
getOptionalInput("db-location"),
getOptionalInput("debug") === "true",
parseRepositoryNwo(getRequiredEnvParam("GITHUB_REPOSITORY")),
getTemporaryDirectory(),
getRequiredEnvParam("RUNNER_TOOL_CACHE"),

2
src/init.ts
View File

@@ -43,6 +43,7 @@ export async function initConfig(
packsInput: string | undefined,
configFile: string | undefined,
dbLocation: string | undefined,
debugMode: boolean,
repository: RepositoryNwo,
tempDir: string,
toolCacheDir: string,
@@ -59,6 +60,7 @@ export async function initConfig(
packsInput,
configFile,
dbLocation,
debugMode,
repository,
tempDir,
toolCacheDir,

1
src/runner.ts
View File

@@ -249,6 +249,7 @@ program
cmd.packs,
cmd.configFile,
undefined,
false,
parseRepositoryNwo(cmd.repository),
tempDir,
toolsDir,

1
src/tracer-config.test.ts
View File

@@ -29,6 +29,7 @@ function getTestConfig(tmpDir: string): configUtils.Config {
gitHubVersion: { type: util.GitHubVariant.DOTCOM } as util.GitHubVersion,
dbLocation: path.resolve(tmpDir, "codeql_databases"),
packs: {},
debugMode: false,
};
}

22
src/util.ts
View File

@@ -18,6 +18,11 @@ import { Logger } from "./logging";
*/
export const GITHUB_DOTCOM_URL = "https://github.com";
/**
* Name of the debugging artifact.
*/
export const DEBUG_ARTIFACT_NAME = "debug-artifacts";
/**
* Get the extra options for the codeql commands.
*/
@@ -542,3 +547,20 @@ export async function codeQlVersionAbove(
): Promise<boolean> {
return semver.gte(await codeql.getVersion(), requiredVersion);
}
// Create a bundle for the given DB, if it doesn't already exist
export async function bundleDb(
config: Config,
language: Language,
codeql: CodeQL
) {
const databasePath = getCodeQLDatabasePath(config, language);
const databaseBundlePath = path.resolve(
config.dbLocation,
`${databasePath}.zip`
);
if (!fs.existsSync(databaseBundlePath)) {
await codeql.databaseBundle(databasePath, databaseBundlePath);
}
return databaseBundlePath;
}