diff --git a/src/analyze-action.ts b/src/analyze-action.ts index 70d7d3939..49c96a5ab 100644 --- a/src/analyze-action.ts +++ b/src/analyze-action.ts @@ -27,6 +27,7 @@ import { EnvVar } from "./environment"; import { Features } from "./feature-flags"; import { Language } from "./languages"; import { getActionsLogger, Logger } from "./logging"; +import { uploadOverlayBaseDatabaseToCache } from "./overlay-database-utils"; import { getRepositoryNwo } from "./repository"; import * as statusReport from "./status-report"; import { @@ -349,6 +350,9 @@ async function run() { // Possibly upload the database bundles for remote queries await uploadDatabases(repositoryNwo, config, apiDetails, logger); + // Possibly upload the overlay-base database to actions cache + await uploadOverlayBaseDatabaseToCache(codeql, config, logger); + // Possibly upload the TRAP caches for later re-use const trapCacheUploadStartTime = performance.now(); didUploadTrapCaches = await uploadTrapCaches(codeql, config, logger); diff --git a/src/overlay-database-utils.ts b/src/overlay-database-utils.ts index f6b811e00..c82504793 100644 --- a/src/overlay-database-utils.ts +++ b/src/overlay-database-utils.ts @@ -1,10 +1,14 @@ import * as fs from "fs"; import * as path from "path"; -import { getTemporaryDirectory } from "./actions-util"; +import * as actionsCache from "@actions/cache"; + +import { getRequiredInput, getTemporaryDirectory } from "./actions-util"; +import { type CodeQL } from "./codeql"; import { type Config } from "./config-utils"; -import { getFileOidsUnderPath } from "./git-utils"; +import { getCommitOid, getFileOidsUnderPath } from "./git-utils"; import { Logger } from "./logging"; +import { isInTestMode, withTimeout } from "./util"; export enum OverlayDatabaseMode { Overlay = "overlay", @@ -122,3 +126,111 @@ function computeChangedFiles( } return changes; } + +// Constants for database caching +const CACHE_VERSION = 1; +const CACHE_PREFIX = "codeql-overlay-base-database"; +const MAX_CACHE_OPERATION_MS = 120_000; // Two minutes + +/** + * Uploads the overlay-base database to the GitHub Actions cache. If conditions + * for uploading are not met, the function does nothing and returns false. + * + * This function uses the `checkout_path` input to determine the repository path + * and works only when called from `analyze` or `upload-sarif`. + * + * @param codeql The CodeQL instance + * @param config The configuration object + * @param logger The logger instance + * @returns A promise that resolves to true if the upload was performed and + * successfully completed, or false otherwise + */ +export async function uploadOverlayBaseDatabaseToCache( + codeql: CodeQL, + config: Config, + logger: Logger, +): Promise { + const overlayDatabaseMode = config.augmentationProperties.overlayDatabaseMode; + if (overlayDatabaseMode !== OverlayDatabaseMode.OverlayBase) { + logger.debug( + `Overlay database mode is ${overlayDatabaseMode}. ` + + "Skip uploading overlay-base database to cache.", + ); + return false; + } + if (!config.augmentationProperties.useOverlayDatabaseCaching) { + logger.debug( + "Overlay database caching is disabled. " + + "Skip uploading overlay-base database to cache.", + ); + return false; + } + if (isInTestMode()) { + logger.debug( + "In test mode. Skip uploading overlay-base database to cache.", + ); + return false; + } + + // An overlay-base database should contain the base database OIDs file. + // Verifying that the file exists serves as a sanity check. + const baseDatabaseOidsFilePath = getBaseDatabaseOidsFilePath(config); + if (!fs.existsSync(baseDatabaseOidsFilePath)) { + logger.warning( + "Cannot upload overlay-base database to cache: " + + `${baseDatabaseOidsFilePath} does not exist`, + ); + return false; + } + + const dbLocation = config.dbLocation; + const codeQlVersion = (await codeql.getVersion()).version; + const checkoutPath = getRequiredInput("checkout_path"); + const cacheKey = await generateCacheKey(config, codeQlVersion, checkoutPath); + logger.info( + `Uploading overlay-base database to Actions cache with key ${cacheKey}`, + ); + + try { + const cacheId = await withTimeout( + MAX_CACHE_OPERATION_MS, + actionsCache.saveCache([dbLocation], cacheKey), + () => {}, + ); + if (cacheId === undefined) { + logger.warning("Timed out while uploading overlay-base database"); + return false; + } + } catch (error) { + logger.warning( + "Failed to upload overlay-base database to cache: " + + `${error instanceof Error ? error.message : String(error)}`, + ); + return false; + } + logger.info(`Successfully uploaded overlay-base database from ${dbLocation}`); + return true; +} + +async function generateCacheKey( + config: Config, + codeQlVersion: string, + checkoutPath: string, +): Promise { + const sha = await getCommitOid(checkoutPath); + return `${getCacheRestoreKey(config, codeQlVersion)}${sha}`; +} + +function getCacheRestoreKey(config: Config, codeQlVersion: string): string { + // The restore key (prefix) specifies which cached overlay-base databases are + // compatible with the current analysis: the cached database must have the + // same cache version and the same CodeQL bundle version. + // + // Actions cache supports using multiple restore keys to indicate preference. + // Technically we prefer a cached overlay-base database with the same SHA as + // we are analyzing. However, since overlay-base databases are built from the + // default branch and used in PR analysis, it is exceedingly unlikely that + // the commit SHA will ever be the same, so we can just leave it out. + const languages = [...config.languages].sort().join("_"); + return `${CACHE_PREFIX}-${CACHE_VERSION}-${languages}-${codeQlVersion}-`; +}