github/codeql-action
1
0
Fork 0
mirror of https://github.com/github/codeql-action.git synced 2025-12-24 08:10:06 +08:00
Code Issues Packages Projects Releases Wiki Activity

Clean up the database if it will be uploaded

Browse Source
This commit is contained in:
Henry Mercer
2025-08-07 11:47:45 +01:00
parent b1228d060c
commit c7884c6fd8
21 changed files with 123 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
lib/analyze-action.js generated
View File

@@ -186,6 +186,9 @@ async function run() {
if (hasBadExpectErrorInput()) {
throw new util.ConfigurationError("`expect-error` input parameter is for internal use only. It should only be set by codeql-action or a fork.");
}
if (actionsUtil.getOptionalInput("cleanup-level") !== "") {
logger.info("The 'cleanup-level' input is ignored since the CodeQL Action no longer writes intermediate results to the database. This input can safely be removed from your workflow.");
}
const apiDetails = (0, api_client_1.getApiDetails)();
const outputDir = actionsUtil.getRequiredInput("output");
core.exportVariable(environment_1.EnvVar.SARIF_RESULTS_OUTPUT_DIR, outputDir);
@@ -202,24 +205,8 @@ async function run() {
await (0, analyze_1.warnIfGoInstalledAfterInit)(config, logger);
await runAutobuildIfLegacyGoWorkflow(config, logger);
dbCreationTimings = await (0, analyze_1.runFinalize)(outputDir, threads, memory, codeql, config, logger);
if (actionsUtil.getOptionalInput("cleanup-level") !== "") {
logger.info("The 'cleanup-level' input is ignored since the CodeQL Action no longer writes intermediate results to the database. This input can safely be removed from your workflow.");
}
// An overlay-base database should always use the 'overlay' cleanup level
// to preserve the cached intermediate results.
//
// Otherwise, use cleanup level 'none'. We are already discarding
// intermediate results during evaluation with '--expect-discarded-cache',
// so there is nothing to clean up.
const cleanupLevel = config.augmentationProperties.overlayDatabaseMode ===
overlay_database_utils_1.OverlayDatabaseMode.OverlayBase
? "overlay"
: "none";
if (actionsUtil.getRequiredInput("skip-queries") !== "true") {
runStats = await (0, analyze_1.runQueries)(outputDir, memory, util.getAddSnippetsFlag(actionsUtil.getRequiredInput("add-snippets")), threads, cleanupLevel, diffRangePackDir, actionsUtil.getOptionalInput("category"), config, logger, features);
}
if (cleanupLevel !== "none") {
await (0, analyze_1.runCleanup)(config, cleanupLevel, logger);
runStats = await (0, analyze_1.runQueries)(outputDir, memory, util.getAddSnippetsFlag(actionsUtil.getRequiredInput("add-snippets")), threads, diffRangePackDir, actionsUtil.getOptionalInput("category"), config, logger, features);
}
const dbLocations = {};
for (const language of config.languages) {
@@ -239,10 +226,12 @@ async function run() {
else {
logger.info("Not uploading results");
}
// Possibly upload the database bundles for remote queries
await (0, database_upload_1.uploadDatabases)(repositoryNwo, config, apiDetails, logger);
// Possibly upload the overlay-base database to actions cache
// Possibly upload the overlay-base database to actions cache.
// If databases are to be uploaded, they will first be cleaned up at the overlay level.
await (0, overlay_database_utils_1.uploadOverlayBaseDatabaseToCache)(codeql, config, logger);
// Possibly upload the database bundles for remote queries.
// If databases are to be uploaded, they will first be cleaned up at the clear level.
await (0, database_upload_1.uploadDatabases)(repositoryNwo, codeql, config, apiDetails, logger);
// Possibly upload the TRAP caches for later re-use
const trapCacheUploadStartTime = perf_hooks_1.performance.now();
didUploadTrapCaches = await (0, trap_caching_1.uploadTrapCaches)(codeql, config, logger);

2
lib/analyze-action.js.map
View File

File diff suppressed because one or more lines are too long

16
lib/analyze.js generated
View File

@@ -44,7 +44,6 @@ exports.resolveQuerySuiteAlias = resolveQuerySuiteAlias;
exports.runQueries = runQueries;
exports.runFinalize = runFinalize;
exports.warnIfGoInstalledAfterInit = warnIfGoInstalledAfterInit;
exports.runCleanup = runCleanup;
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const perf_hooks_1 = require("perf_hooks");
@@ -411,11 +410,13 @@ function resolveQuerySuiteAlias(language, maybeSuite) {
return maybeSuite;
}
// Runs queries and creates sarif files in the given folder
async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag, cleanupLevel, diffRangePackDir, automationDetailsId, config, logger, features) {
async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag, diffRangePackDir, automationDetailsId, config, logger, features) {
const statusReport = {};
const queryFlags = [memoryFlag, threadsFlag];
const incrementalMode = [];
if (cleanupLevel !== "overlay") {
// Preserve cached intermediate results for overlay-base databases.
if (config.augmentationProperties.overlayDatabaseMode !==
overlay_database_utils_1.OverlayDatabaseMode.OverlayBase) {
queryFlags.push("--expect-discarded-cache");
}
statusReport.analysis_is_diff_informed = diffRangePackDir !== undefined;
@@ -567,15 +568,6 @@ async function warnIfGoInstalledAfterInit(config, logger) {
}
}
}
async function runCleanup(config, cleanupLevel, logger) {
logger.startGroup("Cleaning up databases");
for (const language of config.languages) {
const codeql = await (0, codeql_1.getCodeQL)(config.codeQLCmd);
const databasePath = util.getCodeQLDatabasePath(config, language);
await codeql.databaseCleanup(databasePath, cleanupLevel);
}
logger.endGroup();
}
exports.exportedForTesting = {
getDiffRanges,
};

2
lib/analyze.js.map
View File

File diff suppressed because one or more lines are too long

2
lib/analyze.test.js generated
View File

@@ -114,7 +114,7 @@ const util = __importStar(require("./util"));
fs.mkdirSync(util.getCodeQLDatabasePath(config, language), {
recursive: true,
});
const statusReport = await (0, analyze_1.runQueries)(tmpDir, memoryFlag, addSnippetsFlag, threadsFlag, "brutal", undefined, undefined, config, (0, logging_1.getRunnerLogger)(true), (0, testing_utils_1.createFeatures)([feature_flags_1.Feature.QaTelemetryEnabled]));
const statusReport = await (0, analyze_1.runQueries)(tmpDir, memoryFlag, addSnippetsFlag, threadsFlag, undefined, undefined, config, (0, logging_1.getRunnerLogger)(true), (0, testing_utils_1.createFeatures)([feature_flags_1.Feature.QaTelemetryEnabled]));
t.deepEqual(Object.keys(statusReport).sort(), [
"analysis_builds_overlay_base_database",
"analysis_is_diff_informed",

2
lib/analyze.test.js.map
View File

File diff suppressed because one or more lines are too long

7
lib/codeql.js generated
View File

@@ -190,6 +190,7 @@ function setCodeQL(partialCodeql) {
resolveBuildEnvironment: resolveFunction(partialCodeql, "resolveBuildEnvironment"),
packDownload: resolveFunction(partialCodeql, "packDownload"),
databaseCleanup: resolveFunction(partialCodeql, "databaseCleanup"),
databaseCleanupCluster: resolveFunction(partialCodeql, "databaseCleanupCluster"),
databaseBundle: resolveFunction(partialCodeql, "databaseBundle"),
databaseRunQueries: resolveFunction(partialCodeql, "databaseRunQueries"),
databaseInterpretResults: resolveFunction(partialCodeql, "databaseInterpretResults"),
@@ -589,6 +590,12 @@ async function getCodeQLForCmd(cmd, checkVersion) {
];
await runCli(cmd, codeqlArgs);
},
async databaseCleanupCluster(config, cleanupLevel) {
for (const language of config.languages) {
const databasePath = util.getCodeQLDatabasePath(config, language);
await codeql.databaseCleanup(databasePath, cleanupLevel);
}
},
async databaseBundle(databasePath, outputFilePath, databaseName) {
const args = [
"database",

2
lib/codeql.js.map
View File

File diff suppressed because one or more lines are too long

10
lib/database-upload.js generated
View File

@@ -37,11 +37,11 @@ exports.uploadDatabases = uploadDatabases;
const fs = __importStar(require("fs"));
const actionsUtil = __importStar(require("./actions-util"));
const api_client_1 = require("./api-client");
const codeql_1 = require("./codeql");
const gitUtils = __importStar(require("./git-utils"));
const logging_1 = require("./logging");
const util = __importStar(require("./util"));
const util_1 = require("./util");
async function uploadDatabases(repositoryNwo, config, apiDetails, logger) {
async function uploadDatabases(repositoryNwo, codeql, config, apiDetails, logger) {
if (actionsUtil.getRequiredInput("upload-database") !== "true") {
logger.debug("Database upload disabled in workflow. Skipping upload.");
return;
@@ -61,8 +61,12 @@ async function uploadDatabases(repositoryNwo, config, apiDetails, logger) {
logger.debug("Not analyzing default branch. Skipping upload.");
return;
}
// Clean up the database, since intermediate results may still be written to the
// database if there is high RAM pressure.
await (0, logging_1.withGroupAsync)("Cleaning up databases", async () => {
await codeql.databaseCleanupCluster(config, "clear");
});
const client = (0, api_client_1.getApiClient)();
const codeql = await (0, codeql_1.getCodeQL)(config.codeQLCmd);
const uploadsUrl = new URL((0, util_1.parseGitHubUrl)(apiDetails.url));
uploadsUrl.hostname = `uploads.${uploadsUrl.hostname}`;
// Octokit expects the baseUrl to not have a trailing slash,

2
lib/database-upload.js.map
View File

@@ -1 +1 @@
{"version":3,"file":"database-upload.js","sourceRoot":"","sources":["../src/database-upload.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAYA,0CAmFC;AA/FD,uCAAyB;AAEzB,4DAA8C;AAC9C,6CAA8D;AAC9D,qCAAqC;AAErC,sDAAwC;AAGxC,6CAA+B;AAC/B,iCAAkD;AAE3C,KAAK,UAAU,eAAe,CACnC,aAA4B,EAC5B,MAAc,EACd,UAA4B,EAC5B,MAAc;IAEd,IAAI,WAAW,CAAC,gBAAgB,CAAC,iBAAiB,CAAC,KAAK,MAAM,EAAE,CAAC;QAC/D,MAAM,CAAC,KAAK,CAAC,wDAAwD,CAAC,CAAC;QACvE,OAAO;IACT,CAAC;IAED,IAAI,IAAI,CAAC,YAAY,EAAE,EAAE,CAAC;QACxB,MAAM,CAAC,KAAK,CAAC,yCAAyC,CAAC,CAAC;QACxD,OAAO;IACT,CAAC;IAED,iDAAiD;IACjD,IACE,MAAM,CAAC,aAAa,CAAC,IAAI,KAAK,IAAI,CAAC,aAAa,CAAC,MAAM;QACvD,MAAM,CAAC,aAAa,CAAC,IAAI,KAAK,IAAI,CAAC,aAAa,CAAC,UAAU,EAC3D,CAAC;QACD,MAAM,CAAC,KAAK,CAAC,6DAA6D,CAAC,CAAC;QAC5E,OAAO;IACT,CAAC;IAED,IAAI,CAAC,CAAC,MAAM,QAAQ,CAAC,wBAAwB,EAAE,CAAC,EAAE,CAAC;QACjD,4EAA4E;QAC5E,MAAM,CAAC,KAAK,CAAC,gDAAgD,CAAC,CAAC;QAC/D,OAAO;IACT,CAAC;IAED,MAAM,MAAM,GAAG,IAAA,yBAAY,GAAE,CAAC;IAC9B,MAAM,MAAM,GAAG,MAAM,IAAA,kBAAS,EAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAEjD,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,IAAA,qBAAc,EAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;IAC3D,UAAU,CAAC,QAAQ,GAAG,WAAW,UAAU,CAAC,QAAQ,EAAE,CAAC;IAEvD,4DAA4D;IAC5D,0CAA0C;IAC1C,IAAI,cAAc,GAAG,UAAU,CAAC,QAAQ,EAAE,CAAC;IAC3C,IAAI,cAAc,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACjC,cAAc,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC/C,CAAC;IAED,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;QACxC,IAAI,CAAC;YACH,8BAA8B;YAC9B,2EAA2E;YAC3E,8EAA8E;YAC9E,wEAAwE;YACxE,MAAM,SAAS,GAAG,MAAM,IAAA,eAAQ,EAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;YACrE,MAAM,aAAa,GAAG,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC;YAClD,MAAM,mBAAmB,GAAG,EAAE,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;YAC3D,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,YAAY,CAC3C,WAAW,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAC9C,CAAC;YACF,IAAI,CAAC;gBACH,MAAM,MAAM,CAAC,OAAO,CAClB,qGAAqG,EACrG;oBACE,OAAO,EAAE,cAAc;oBACvB,KAAK,EAAE,aAAa,CAAC,KAAK;oBAC1B,IAAI,EAAE,aAAa,CAAC,IAAI;oBACxB,QAAQ;oBACR,IAAI,EAAE,GAAG,QAAQ,WAAW;oBAC5B,UAAU,EAAE,SAAS;oBACrB,IAAI,EAAE,mBAAmB;oBACzB,OAAO,EAAE;wBACP,aAAa,EAAE,SAAS,UAAU,CAAC,IAAI,EAAE;wBACzC,cAAc,EAAE,iBAAiB;wBACjC,gBAAgB,EAAE,aAAa;qBAChC;iBACF,CACF,CAAC;gBACF,MAAM,CAAC,KAAK,CAAC,sCAAsC,QAAQ,EAAE,CAAC,CAAC;YACjE,CAAC;oBAAS,CAAC;gBACT,mBAAmB,CAAC,KAAK,EAAE,CAAC;YAC9B,CAAC;QACH,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,4CAA4C;YAC5C,MAAM,CAAC,OAAO,CAAC,iCAAiC,QAAQ,KAAK,CAAC,EAAE,CAAC,CAAC;QACpE,CAAC;IACH,CAAC;AACH,CAAC"}
{"version":3,"file":"database-upload.js","sourceRoot":"","sources":["../src/database-upload.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAYA,0CAyFC;AArGD,uCAAyB;AAEzB,4DAA8C;AAC9C,6CAA8D;AAG9D,sDAAwC;AACxC,uCAAmD;AAEnD,6CAA+B;AAC/B,iCAAkD;AAE3C,KAAK,UAAU,eAAe,CACnC,aAA4B,EAC5B,MAAc,EACd,MAAc,EACd,UAA4B,EAC5B,MAAc;IAEd,IAAI,WAAW,CAAC,gBAAgB,CAAC,iBAAiB,CAAC,KAAK,MAAM,EAAE,CAAC;QAC/D,MAAM,CAAC,KAAK,CAAC,wDAAwD,CAAC,CAAC;QACvE,OAAO;IACT,CAAC;IAED,IAAI,IAAI,CAAC,YAAY,EAAE,EAAE,CAAC;QACxB,MAAM,CAAC,KAAK,CAAC,yCAAyC,CAAC,CAAC;QACxD,OAAO;IACT,CAAC;IAED,iDAAiD;IACjD,IACE,MAAM,CAAC,aAAa,CAAC,IAAI,KAAK,IAAI,CAAC,aAAa,CAAC,MAAM;QACvD,MAAM,CAAC,aAAa,CAAC,IAAI,KAAK,IAAI,CAAC,aAAa,CAAC,UAAU,EAC3D,CAAC;QACD,MAAM,CAAC,KAAK,CAAC,6DAA6D,CAAC,CAAC;QAC5E,OAAO;IACT,CAAC;IAED,IAAI,CAAC,CAAC,MAAM,QAAQ,CAAC,wBAAwB,EAAE,CAAC,EAAE,CAAC;QACjD,4EAA4E;QAC5E,MAAM,CAAC,KAAK,CAAC,gDAAgD,CAAC,CAAC;QAC/D,OAAO;IACT,CAAC;IAED,gFAAgF;IAChF,0CAA0C;IAC1C,MAAM,IAAA,wBAAc,EAAC,uBAAuB,EAAE,KAAK,IAAI,EAAE;QACvD,MAAM,MAAM,CAAC,sBAAsB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACvD,CAAC,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,IAAA,yBAAY,GAAE,CAAC;IAE9B,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,IAAA,qBAAc,EAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;IAC3D,UAAU,CAAC,QAAQ,GAAG,WAAW,UAAU,CAAC,QAAQ,EAAE,CAAC;IAEvD,4DAA4D;IAC5D,0CAA0C;IAC1C,IAAI,cAAc,GAAG,UAAU,CAAC,QAAQ,EAAE,CAAC;IAC3C,IAAI,cAAc,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACjC,cAAc,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC/C,CAAC;IAED,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;QACxC,IAAI,CAAC;YACH,8BAA8B;YAC9B,2EAA2E;YAC3E,8EAA8E;YAC9E,wEAAwE;YACxE,MAAM,SAAS,GAAG,MAAM,IAAA,eAAQ,EAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;YACrE,MAAM,aAAa,GAAG,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC;YAClD,MAAM,mBAAmB,GAAG,EAAE,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;YAC3D,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,YAAY,CAC3C,WAAW,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAC9C,CAAC;YACF,IAAI,CAAC;gBACH,MAAM,MAAM,CAAC,OAAO,CAClB,qGAAqG,EACrG;oBACE,OAAO,EAAE,cAAc;oBACvB,KAAK,EAAE,aAAa,CAAC,KAAK;oBAC1B,IAAI,EAAE,aAAa,CAAC,IAAI;oBACxB,QAAQ;oBACR,IAAI,EAAE,GAAG,QAAQ,WAAW;oBAC5B,UAAU,EAAE,SAAS;oBACrB,IAAI,EAAE,mBAAmB;oBACzB,OAAO,EAAE;wBACP,aAAa,EAAE,SAAS,UAAU,CAAC,IAAI,EAAE;wBACzC,cAAc,EAAE,iBAAiB;wBACjC,gBAAgB,EAAE,aAAa;qBAChC;iBACF,CACF,CAAC;gBACF,MAAM,CAAC,KAAK,CAAC,sCAAsC,QAAQ,EAAE,CAAC,CAAC;YACjE,CAAC;oBAAS,CAAC;gBACT,mBAAmB,CAAC,KAAK,EAAE,CAAC;YAC9B,CAAC;QACH,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,4CAA4C;YAC5C,MAAM,CAAC,OAAO,CAAC,iCAAiC,QAAQ,KAAK,CAAC,EAAE,CAAC,CAAC;QACpE,CAAC;IACH,CAAC;AACH,CAAC"}

37
lib/database-upload.test.js generated
View File

@@ -79,6 +79,16 @@ async function mockHttpRequests(databaseUploadStatusCode) {
sinon.stub(apiClient, "getApiClient").value(() => client);
return databaseUploadSpy;
}
function getCodeQL() {
return (0, codeql_1.setCodeQL)({
async databaseBundle(_, outputFilePath) {
fs.writeFileSync(outputFilePath, "");
},
async databaseCleanupCluster() {
// Do nothing, as we are not testing cleanup here.
},
});
}
(0, ava_1.default)("Abort database upload if 'upload-database' input set to false", async (t) => {
await (0, util_1.withTmpDir)(async (tmpDir) => {
(0, testing_utils_1.setupActionsVars)(tmpDir, tmpDir);
@@ -88,7 +98,7 @@ async function mockHttpRequests(databaseUploadStatusCode) {
.returns("false");
sinon.stub(gitUtils, "isAnalyzingDefaultBranch").resolves(true);
const loggedMessages = [];
await (0, database_upload_1.uploadDatabases)(testRepoName, getTestConfig(tmpDir), testApiDetails, (0, testing_utils_1.getRecordingLogger)(loggedMessages));
await (0, database_upload_1.uploadDatabases)(testRepoName, getCodeQL(), getTestConfig(tmpDir), testApiDetails, (0, testing_utils_1.getRecordingLogger)(loggedMessages));
t.assert(loggedMessages.find((v) => v.type === "debug" &&
v.message ===
"Database upload disabled in workflow. Skipping upload.") !== undefined);
@@ -105,7 +115,7 @@ async function mockHttpRequests(databaseUploadStatusCode) {
const config = getTestConfig(tmpDir);
config.gitHubVersion = { type: util_1.GitHubVariant.GHES, version: "3.0" };
const loggedMessages = [];
await (0, database_upload_1.uploadDatabases)(testRepoName, config, testApiDetails, (0, testing_utils_1.getRecordingLogger)(loggedMessages));
await (0, database_upload_1.uploadDatabases)(testRepoName, getCodeQL(), config, testApiDetails, (0, testing_utils_1.getRecordingLogger)(loggedMessages));
t.assert(loggedMessages.find((v) => v.type === "debug" &&
v.message ===
"Not running against github.com or GHEC-DR. Skipping upload.") !== undefined);
@@ -120,7 +130,7 @@ async function mockHttpRequests(databaseUploadStatusCode) {
.returns("true");
sinon.stub(gitUtils, "isAnalyzingDefaultBranch").resolves(false);
const loggedMessages = [];
await (0, database_upload_1.uploadDatabases)(testRepoName, getTestConfig(tmpDir), testApiDetails, (0, testing_utils_1.getRecordingLogger)(loggedMessages));
await (0, database_upload_1.uploadDatabases)(testRepoName, getCodeQL(), getTestConfig(tmpDir), testApiDetails, (0, testing_utils_1.getRecordingLogger)(loggedMessages));
t.assert(loggedMessages.find((v) => v.type === "debug" &&
v.message === "Not analyzing default branch. Skipping upload.") !== undefined);
});
@@ -134,13 +144,8 @@ async function mockHttpRequests(databaseUploadStatusCode) {
.returns("true");
sinon.stub(gitUtils, "isAnalyzingDefaultBranch").resolves(true);
await mockHttpRequests(500);
(0, codeql_1.setCodeQL)({
async databaseBundle(_, outputFilePath) {
fs.writeFileSync(outputFilePath, "");
},
});
const loggedMessages = [];
await (0, database_upload_1.uploadDatabases)(testRepoName, getTestConfig(tmpDir), testApiDetails, (0, testing_utils_1.getRecordingLogger)(loggedMessages));
await (0, database_upload_1.uploadDatabases)(testRepoName, getCodeQL(), getTestConfig(tmpDir), testApiDetails, (0, testing_utils_1.getRecordingLogger)(loggedMessages));
t.assert(loggedMessages.find((v) => v.type === "warning" &&
v.message ===
"Failed to upload database for javascript: Error: some error message") !== undefined);
@@ -155,13 +160,8 @@ async function mockHttpRequests(databaseUploadStatusCode) {
.returns("true");
sinon.stub(gitUtils, "isAnalyzingDefaultBranch").resolves(true);
await mockHttpRequests(201);
(0, codeql_1.setCodeQL)({
async databaseBundle(_, outputFilePath) {
fs.writeFileSync(outputFilePath, "");
},
});
const loggedMessages = [];
await (0, database_upload_1.uploadDatabases)(testRepoName, getTestConfig(tmpDir), testApiDetails, (0, testing_utils_1.getRecordingLogger)(loggedMessages));
await (0, database_upload_1.uploadDatabases)(testRepoName, getCodeQL(), getTestConfig(tmpDir), testApiDetails, (0, testing_utils_1.getRecordingLogger)(loggedMessages));
t.assert(loggedMessages.find((v) => v.type === "debug" &&
v.message === "Successfully uploaded database for javascript") !== undefined);
});
@@ -175,13 +175,8 @@ async function mockHttpRequests(databaseUploadStatusCode) {
.returns("true");
sinon.stub(gitUtils, "isAnalyzingDefaultBranch").resolves(true);
const databaseUploadSpy = await mockHttpRequests(201);
(0, codeql_1.setCodeQL)({
async databaseBundle(_, outputFilePath) {
fs.writeFileSync(outputFilePath, "");
},
});
const loggedMessages = [];
await (0, database_upload_1.uploadDatabases)(testRepoName, getTestConfig(tmpDir), {
await (0, database_upload_1.uploadDatabases)(testRepoName, getCodeQL(), getTestConfig(tmpDir), {
auth: "1234",
url: "https://tenant.ghe.com",
apiURL: undefined,

2
lib/database-upload.test.js.map
View File

File diff suppressed because one or more lines are too long

5
lib/overlay-database-utils.js generated
View File

@@ -44,6 +44,7 @@ const path = __importStar(require("path"));
const actionsCache = __importStar(require("@actions/cache"));
const actions_util_1 = require("./actions-util");
const git_utils_1 = require("./git-utils");
const logging_1 = require("./logging");
const util_1 = require("./util");
var OverlayDatabaseMode;
(function (OverlayDatabaseMode) {
@@ -186,6 +187,10 @@ async function uploadOverlayBaseDatabaseToCache(codeql, config, logger) {
if (!databaseIsValid) {
return false;
}
// Clean up the database using the overlay cleanup level.
await (0, logging_1.withGroupAsync)("Cleaning up databases", async () => {
await codeql.databaseCleanupCluster(config, "overlay");
});
const dbLocation = config.dbLocation;
const codeQlVersion = (await codeql.getVersion()).version;
const checkoutPath = (0, actions_util_1.getRequiredInput)("checkout_path");

2
lib/overlay-database-utils.js.map
View File

File diff suppressed because one or more lines are too long

45
src/analyze-action.ts
View File

@@ -9,7 +9,6 @@ import {
CodeQLAnalysisError,
dbIsFinalized,
QueriesStatusReport,
runCleanup,
runFinalize,
runQueries,
setupDiffInformedQueryRun,
@@ -27,10 +26,7 @@ import { EnvVar } from "./environment";
import { Features } from "./feature-flags";
import { KnownLanguage } from "./languages";
import { getActionsLogger, Logger } from "./logging";
import {
OverlayDatabaseMode,
uploadOverlayBaseDatabaseToCache,
} from "./overlay-database-utils";
import { uploadOverlayBaseDatabaseToCache } from "./overlay-database-utils";
import { getRepositoryNwo } from "./repository";
import * as statusReport from "./status-report";
import {
@@ -251,6 +247,12 @@ async function run() {
);
}
if (actionsUtil.getOptionalInput("cleanup-level") !== "") {
logger.info(
"The 'cleanup-level' input is ignored since the CodeQL Action no longer writes intermediate results to the database. This input can safely be removed from your workflow.",
);
}
const apiDetails = getApiDetails();
const outputDir = actionsUtil.getRequiredInput("output");
core.exportVariable(EnvVar.SARIF_RESULTS_OUTPUT_DIR, outputDir);
@@ -298,31 +300,12 @@ async function run() {
logger,
);
if (actionsUtil.getOptionalInput("cleanup-level") !== "") {
logger.info(
"The 'cleanup-level' input is ignored since the CodeQL Action no longer writes intermediate results to the database. This input can safely be removed from your workflow.",
);
}
// An overlay-base database should always use the 'overlay' cleanup level
// to preserve the cached intermediate results.
//
// Otherwise, use cleanup level 'none'. We are already discarding
// intermediate results during evaluation with '--expect-discarded-cache',
// so there is nothing to clean up.
const cleanupLevel =
config.augmentationProperties.overlayDatabaseMode ===
OverlayDatabaseMode.OverlayBase
? "overlay"
: "none";
if (actionsUtil.getRequiredInput("skip-queries") !== "true") {
runStats = await runQueries(
outputDir,
memory,
util.getAddSnippetsFlag(actionsUtil.getRequiredInput("add-snippets")),
threads,
cleanupLevel,
diffRangePackDir,
actionsUtil.getOptionalInput("category"),
config,
@@ -331,10 +314,6 @@ async function run() {
);
}
if (cleanupLevel !== "none") {
await runCleanup(config, cleanupLevel, logger);
}
const dbLocations: { [lang: string]: string } = {};
for (const language of config.languages) {
dbLocations[language] = util.getCodeQLDatabasePath(config, language);
@@ -368,12 +347,14 @@ async function run() {
logger.info("Not uploading results");
}
// Possibly upload the database bundles for remote queries
await uploadDatabases(repositoryNwo, config, apiDetails, logger);
// Possibly upload the overlay-base database to actions cache
// Possibly upload the overlay-base database to actions cache.
// If databases are to be uploaded, they will first be cleaned up at the overlay level.
await uploadOverlayBaseDatabaseToCache(codeql, config, logger);
// Possibly upload the database bundles for remote queries.
// If databases are to be uploaded, they will first be cleaned up at the clear level.
await uploadDatabases(repositoryNwo, codeql, config, apiDetails, logger);
// Possibly upload the TRAP caches for later re-use
const trapCacheUploadStartTime = performance.now();
didUploadTrapCaches = await uploadTrapCaches(codeql, config, logger);

1
src/analyze.test.ts
View File

@@ -106,7 +106,6 @@ test("status report fields", async (t) => {
memoryFlag,
addSnippetsFlag,
threadsFlag,
"brutal",
undefined,
undefined,
config,

21
src/analyze.ts
View File

@@ -612,7 +612,6 @@ export async function runQueries(
memoryFlag: string,
addSnippetsFlag: string,
threadsFlag: string,
cleanupLevel: string,
diffRangePackDir: string | undefined,
automationDetailsId: string | undefined,
config: configUtils.Config,
@@ -623,7 +622,11 @@ export async function runQueries(
const queryFlags = [memoryFlag, threadsFlag];
const incrementalMode: string[] = [];
if (cleanupLevel !== "overlay") {
// Preserve cached intermediate results for overlay-base databases.
if (
config.augmentationProperties.overlayDatabaseMode !==
OverlayDatabaseMode.OverlayBase
) {
queryFlags.push("--expect-discarded-cache");
}
@@ -874,20 +877,6 @@ export async function warnIfGoInstalledAfterInit(
}
}
export async function runCleanup(
config: configUtils.Config,
cleanupLevel: string,
logger: Logger,
): Promise<void> {
logger.startGroup("Cleaning up databases");
for (const language of config.languages) {
const codeql = await getCodeQL(config.codeQLCmd);
const databasePath = util.getCodeQLDatabasePath(config, language);
await codeql.databaseCleanup(databasePath, cleanupLevel);
}
logger.endGroup();
}
export const exportedForTesting = {
getDiffRanges,
};

17
src/codeql.ts
View File

@@ -155,6 +155,10 @@ export interface CodeQL {
* Run 'codeql database cleanup'.
*/
databaseCleanup(databasePath: string, cleanupLevel: string): Promise<void>;
/**
* Clean up all the databases within a database cluster.
*/
databaseCleanupCluster(config: Config, cleanupLevel: string): Promise<void>;
/**
* Run 'codeql database bundle'.
*/
@@ -483,6 +487,10 @@ export function setCodeQL(partialCodeql: Partial<CodeQL>): CodeQL {
),
packDownload: resolveFunction(partialCodeql, "packDownload"),
databaseCleanup: resolveFunction(partialCodeql, "databaseCleanup"),
databaseCleanupCluster: resolveFunction(
partialCodeql,
"databaseCleanupCluster",
),
databaseBundle: resolveFunction(partialCodeql, "databaseBundle"),
databaseRunQueries: resolveFunction(partialCodeql, "databaseRunQueries"),
databaseInterpretResults: resolveFunction(
@@ -997,6 +1005,15 @@ export async function getCodeQLForCmd(
];
await runCli(cmd, codeqlArgs);
},
async databaseCleanupCluster(
config: Config,
cleanupLevel: string,
): Promise<void> {
for (const language of config.languages) {
const databasePath = util.getCodeQLDatabasePath(config, language);
await codeql.databaseCleanup(databasePath, cleanupLevel);
}
},
async databaseBundle(
databasePath: string,
outputFilePath: string,

35
src/database-upload.test.ts
View File

@@ -69,6 +69,17 @@ async function mockHttpRequests(databaseUploadStatusCode: number) {
return databaseUploadSpy;
}
function getCodeQL() {
return setCodeQL({
async databaseBundle(_: string, outputFilePath: string) {
fs.writeFileSync(outputFilePath, "");
},
async databaseCleanupCluster() {
// Do nothing, as we are not testing cleanup here.
},
});
}
test("Abort database upload if 'upload-database' input set to false", async (t) => {
await withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);
@@ -81,6 +92,7 @@ test("Abort database upload if 'upload-database' input set to false", async (t)
const loggedMessages = [];
await uploadDatabases(
testRepoName,
getCodeQL(),
getTestConfig(tmpDir),
testApiDetails,
getRecordingLogger(loggedMessages),
@@ -111,6 +123,7 @@ test("Abort database upload if running against GHES", async (t) => {
const loggedMessages = [];
await uploadDatabases(
testRepoName,
getCodeQL(),
config,
testApiDetails,
getRecordingLogger(loggedMessages),
@@ -138,6 +151,7 @@ test("Abort database upload if not analyzing default branch", async (t) => {
const loggedMessages = [];
await uploadDatabases(
testRepoName,
getCodeQL(),
getTestConfig(tmpDir),
testApiDetails,
getRecordingLogger(loggedMessages),
@@ -163,15 +177,10 @@ test("Don't crash if uploading a database fails", async (t) => {
await mockHttpRequests(500);
setCodeQL({
async databaseBundle(_: string, outputFilePath: string) {
fs.writeFileSync(outputFilePath, "");
},
});
const loggedMessages = [] as LoggedMessage[];
await uploadDatabases(
testRepoName,
getCodeQL(),
getTestConfig(tmpDir),
testApiDetails,
getRecordingLogger(loggedMessages),
@@ -199,15 +208,10 @@ test("Successfully uploading a database to github.com", async (t) => {
await mockHttpRequests(201);
setCodeQL({
async databaseBundle(_: string, outputFilePath: string) {
fs.writeFileSync(outputFilePath, "");
},
});
const loggedMessages = [] as LoggedMessage[];
await uploadDatabases(
testRepoName,
getCodeQL(),
getTestConfig(tmpDir),
testApiDetails,
getRecordingLogger(loggedMessages),
@@ -233,15 +237,10 @@ test("Successfully uploading a database to GHEC-DR", async (t) => {
const databaseUploadSpy = await mockHttpRequests(201);
setCodeQL({
async databaseBundle(_: string, outputFilePath: string) {
fs.writeFileSync(outputFilePath, "");
},
});
const loggedMessages = [] as LoggedMessage[];
await uploadDatabases(
testRepoName,
getCodeQL(),
getTestConfig(tmpDir),
{
auth: "1234",

12
src/database-upload.ts
View File

@@ -2,16 +2,17 @@ import * as fs from "fs";
import * as actionsUtil from "./actions-util";
import { getApiClient, GitHubApiDetails } from "./api-client";
import { getCodeQL } from "./codeql";
import { type CodeQL } from "./codeql";
import { Config } from "./config-utils";
import * as gitUtils from "./git-utils";
import { Logger } from "./logging";
import { Logger, withGroupAsync } from "./logging";
import { RepositoryNwo } from "./repository";
import * as util from "./util";
import { bundleDb, parseGitHubUrl } from "./util";
export async function uploadDatabases(
repositoryNwo: RepositoryNwo,
codeql: CodeQL,
config: Config,
apiDetails: GitHubApiDetails,
logger: Logger,
@@ -41,8 +42,13 @@ export async function uploadDatabases(
return;
}
// Clean up the database, since intermediate results may still be written to the
// database if there is high RAM pressure.
await withGroupAsync("Cleaning up databases", async () => {
await codeql.databaseCleanupCluster(config, "clear");
});
const client = getApiClient();
const codeql = await getCodeQL(config.codeQLCmd);
const uploadsUrl = new URL(parseGitHubUrl(apiDetails.url));
uploadsUrl.hostname = `uploads.${uploadsUrl.hostname}`;

7
src/overlay-database-utils.ts
View File

@@ -7,7 +7,7 @@ import { getRequiredInput, getTemporaryDirectory } from "./actions-util";
import { type CodeQL } from "./codeql";
import { type Config } from "./config-utils";
import { getCommitOid, getFileOidsUnderPath } from "./git-utils";
import { Logger } from "./logging";
import { Logger, withGroupAsync } from "./logging";
import { isInTestMode, tryGetFolderBytes, withTimeout } from "./util";
export enum OverlayDatabaseMode {
@@ -206,6 +206,11 @@ export async function uploadOverlayBaseDatabaseToCache(
return false;
}
// Clean up the database using the overlay cleanup level.
await withGroupAsync("Cleaning up databases", async () => {
await codeql.databaseCleanupCluster(config, "overlay");
});
const dbLocation = config.dbLocation;
const codeQlVersion = (await codeql.getVersion()).version;
const checkoutPath = getRequiredInput("checkout_path");