import * as fs from "fs"; import * as path from "path"; import * as cache from "@actions/cache"; import * as actionsUtil from "./actions-util"; import { CodeQL } from "./codeql"; import type { Config } from "./config-utils"; import { Language } from "./languages"; import { Logger } from "./logging"; import { tryGetFolderBytes, withTimeout } from "./util"; // This constant should be bumped if we make a breaking change // to how the CodeQL Action stores or retrieves the TRAP cache, // and will invalidate previous caches. We don't need to bump // this for CLI/extractor changes, since the CLI version also // goes into the cache key. const CACHE_VERSION = 1; // This constant sets the minimum size in megabytes of a TRAP // cache for us to consider it worth uploading. const MINIMUM_CACHE_MB_TO_UPLOAD = 10; // The maximum number of milliseconds to wait for TRAP cache // uploads or downloads to complete before continuing. Note // this timeout is per operation, so will be run as many // times as there are languages with TRAP caching enabled. const MAX_CACHE_OPERATION_MS = 120_000; // Two minutes /** * Download TRAP caches from the Actions cache. * @param codeql The CodeQL instance to use. * @param languages The languages being analyzed. * @param logger A logger to record some informational messages to. * @returns A partial map from languages to TRAP cache paths on disk, with * languages for which we shouldn't use TRAP caching omitted. */ export async function downloadTrapCaches( codeql: CodeQL, languages: Language[], logger: Logger, ): Promise>> { const result: Partial> = {}; const languagesSupportingCaching = await getLanguagesSupportingCaching( codeql, languages, logger, ); logger.info( `Found ${languagesSupportingCaching.length} languages that support TRAP caching`, ); if (languagesSupportingCaching.length === 0) return result; const cachesDir = path.join( actionsUtil.getTemporaryDirectory(), "trapCaches", ); for (const language of languagesSupportingCaching) { const cacheDir = path.join(cachesDir, language); fs.mkdirSync(cacheDir, { recursive: true }); result[language] = cacheDir; } if (await actionsUtil.isAnalyzingDefaultBranch()) { logger.info( "Analyzing default branch. Skipping downloading of TRAP caches.", ); return result; } let baseSha = "unknown"; const eventPath = process.env.GITHUB_EVENT_PATH; if ( actionsUtil.getWorkflowEventName() === "pull_request" && eventPath !== undefined ) { const event = JSON.parse(fs.readFileSync(path.resolve(eventPath), "utf-8")); baseSha = event.pull_request?.base?.sha || baseSha; } for (const language of languages) { const cacheDir = result[language]; if (cacheDir === undefined) continue; // The SHA from the base of the PR is the most similar commit we might have a cache for const preferredKey = await cacheKey(codeql, language, baseSha); logger.info( `Looking in Actions cache for TRAP cache with key ${preferredKey}`, ); const found = await withTimeout( MAX_CACHE_OPERATION_MS, cache.restoreCache([cacheDir], preferredKey, [ // Fall back to any cache with the right key prefix await cachePrefix(codeql, language), ]), () => { logger.info( `Timed out downloading cache for ${language}, will continue without it`, ); }, ); if (found === undefined) { // We didn't find a TRAP cache in the Actions cache, so the directory on disk is // still just an empty directory. There's no reason to tell the extractor to use it, // so let's unset the entry in the map so we don't set any extractor options. logger.info(`No TRAP cache found in Actions cache for ${language}`); delete result[language]; } } return result; } /** * Possibly upload TRAP caches to the Actions cache. * @param codeql The CodeQL instance to use. * @param config The configuration for this workflow. * @param logger A logger to record some informational messages to. * @returns Whether the TRAP caches were uploaded. */ export async function uploadTrapCaches( codeql: CodeQL, config: Config, logger: Logger, ): Promise { if (!(await actionsUtil.isAnalyzingDefaultBranch())) return false; // Only upload caches from the default branch for (const language of config.languages) { const cacheDir = config.trapCaches[language]; if (cacheDir === undefined) continue; const trapFolderSize = await tryGetFolderBytes(cacheDir, logger); if (trapFolderSize === undefined) { logger.info( `Skipping upload of TRAP cache for ${language} as we couldn't determine its size`, ); continue; } if (trapFolderSize < MINIMUM_CACHE_MB_TO_UPLOAD * 1_048_576) { logger.info( `Skipping upload of TRAP cache for ${language} as it is too small`, ); continue; } const key = await cacheKey( codeql, language, process.env.GITHUB_SHA || "unknown", ); logger.info(`Uploading TRAP cache to Actions cache with key ${key}`); await withTimeout( MAX_CACHE_OPERATION_MS, cache.saveCache([cacheDir], key), () => { logger.info( `Timed out waiting for TRAP cache for ${language} to upload, will continue without uploading`, ); }, ); } return true; } export async function getLanguagesSupportingCaching( codeql: CodeQL, languages: Language[], logger: Logger, ): Promise { const result: Language[] = []; const resolveResult = await codeql.betterResolveLanguages(); outer: for (const lang of languages) { const extractorsForLanguage = resolveResult.extractors[lang]; if (extractorsForLanguage === undefined) { logger.info( `${lang} does not support TRAP caching (couldn't find an extractor)`, ); continue; } if (extractorsForLanguage.length !== 1) { logger.info( `${lang} does not support TRAP caching (found multiple extractors)`, ); continue; } const extractor = extractorsForLanguage[0]; const trapCacheOptions = extractor.extractor_options?.trap?.properties?.cache?.properties; if (trapCacheOptions === undefined) { logger.info( `${lang} does not support TRAP caching (missing option group)`, ); continue; } for (const requiredOpt of ["dir", "bound", "write"]) { if (!(requiredOpt in trapCacheOptions)) { logger.info( `${lang} does not support TRAP caching (missing ${requiredOpt} option)`, ); continue outer; } } result.push(lang); } return result; } export async function getTotalCacheSize( trapCaches: Partial>, logger: Logger, ): Promise { const sizes = await Promise.all( Object.values(trapCaches).map((cacheDir) => tryGetFolderBytes(cacheDir, logger), ), ); return sizes.map((a) => a || 0).reduce((a, b) => a + b, 0); } async function cacheKey( codeql: CodeQL, language: Language, baseSha: string, ): Promise { return `${await cachePrefix(codeql, language)}${baseSha}`; } async function cachePrefix( codeql: CodeQL, language: Language, ): Promise { return `codeql-trap-${CACHE_VERSION}-${ (await codeql.getVersion()).version }-${language}-`; }