import * as fs from "fs"; import * as path from "path"; import * as cache from "@actions/cache"; import * as actionsUtil from "./actions-util"; import { CodeQL, CODEQL_VERSION_BETTER_RESOLVE_LANGUAGES } from "./codeql"; import { Config } from "./config-utils"; import { Language } from "./languages"; import { Logger } from "./logging"; import { codeQlVersionAbove, tryGetFolderBytes, withTimeout } from "./util"; // This constant should be bumped if we make a breaking change // to how the CodeQL Action stores or retrieves the TRAP cache, // and will invalidate previous caches. We don't need to bump // this for CLI/extractor changes, since the CLI version also // goes into the cache key. const CACHE_VERSION = 1; // This constant sets the size of each TRAP cache in megabytes. const CACHE_SIZE_MB = 1024; // This constant sets the minimum size in megabytes of a TRAP // cache for us to consider it worth uploading. const MINIMUM_CACHE_MB_TO_UPLOAD = 10; // The maximum number of milliseconds to wait for TRAP cache // uploads or downloads to complete before continuing. Note // this timeout is per operation, so will be run as many // times as there are languages with TRAP caching enabled. const MAX_CACHE_OPERATION_MS = 120_000; // Two minutes export async function getTrapCachingExtractorConfigArgs( config: Config ): Promise { const result: string[][] = []; for (const language of config.languages) result.push( await getTrapCachingExtractorConfigArgsForLang(config, language) ); return result.flat(); } export async function getTrapCachingExtractorConfigArgsForLang( config: Config, language: Language ): Promise { const cacheDir = config.trapCaches[language]; if (cacheDir === undefined) return []; const write = await actionsUtil.isAnalyzingDefaultBranch(); return [ `-O=${language}.trap.cache.dir=${cacheDir}`, `-O=${language}.trap.cache.bound=${CACHE_SIZE_MB}`, `-O=${language}.trap.cache.write=${write}`, ]; } /** * Download TRAP caches from the Actions cache. * @param codeql The CodeQL instance to use. * @param languages The languages being analyzed. * @param logger A logger to record some informational messages to. * @returns A partial map from languages to TRAP cache paths on disk, with * languages for which we shouldn't use TRAP caching omitted. */ export async function downloadTrapCaches( codeql: CodeQL, languages: Language[], logger: Logger ): Promise>> { const result: Partial> = {}; const languagesSupportingCaching = await getLanguagesSupportingCaching( codeql, languages, logger ); logger.info( `Found ${languagesSupportingCaching.length} languages that support TRAP caching` ); if (languagesSupportingCaching.length === 0) return result; const cachesDir = path.join( actionsUtil.getTemporaryDirectory(), "trapCaches" ); for (const language of languagesSupportingCaching) { const cacheDir = path.join(cachesDir, language); fs.mkdirSync(cacheDir, { recursive: true }); result[language] = cacheDir; } if (await actionsUtil.isAnalyzingDefaultBranch()) { logger.info( "Analyzing default branch. Skipping downloading of TRAP caches." ); return result; } let baseSha = "unknown"; const eventPath = process.env.GITHUB_EVENT_PATH; if ( actionsUtil.getWorkflowEventName() === "pull_request" && eventPath !== undefined ) { const event = JSON.parse(fs.readFileSync(path.resolve(eventPath), "utf-8")); baseSha = event.pull_request?.base?.sha || baseSha; } for (const language of languages) { const cacheDir = result[language]; if (cacheDir === undefined) continue; // The SHA from the base of the PR is the most similar commit we might have a cache for const preferredKey = await cacheKey(codeql, language, baseSha); logger.info( `Looking in Actions cache for TRAP cache with key ${preferredKey}` ); const found = await withTimeout( MAX_CACHE_OPERATION_MS, cache.restoreCache([cacheDir], preferredKey, [ // Fall back to any cache with the right key prefix await cachePrefix(codeql, language), ]), () => { logger.info( `Timed out downloading cache for ${language}, will continue without it` ); } ); if (found === undefined) { // We didn't find a TRAP cache in the Actions cache, so the directory on disk is // still just an empty directory. There's no reason to tell the extractor to use it, // so let's unset the entry in the map so we don't set any extractor options. logger.info(`No TRAP cache found in Actions cache for ${language}`); delete result[language]; } } return result; } /** * Possibly upload TRAP caches to the Actions cache. * @param codeql The CodeQL instance to use. * @param config The configuration for this workflow. * @param logger A logger to record some informational messages to. * @returns Whether the TRAP caches were uploaded. */ export async function uploadTrapCaches( codeql: CodeQL, config: Config, logger: Logger ): Promise { if (!(await actionsUtil.isAnalyzingDefaultBranch())) return false; // Only upload caches from the default branch for (const language of config.languages) { const cacheDir = config.trapCaches[language]; if (cacheDir === undefined) continue; const trapFolderSize = await tryGetFolderBytes(cacheDir, logger); if (trapFolderSize === undefined) { logger.info( `Skipping upload of TRAP cache for ${language} as we couldn't determine its size` ); continue; } if (trapFolderSize < MINIMUM_CACHE_MB_TO_UPLOAD * 1_048_576) { logger.info( `Skipping upload of TRAP cache for ${language} as it is too small` ); continue; } const key = await cacheKey( codeql, language, process.env.GITHUB_SHA || "unknown" ); logger.info(`Uploading TRAP cache to Actions cache with key ${key}`); await withTimeout( MAX_CACHE_OPERATION_MS, cache.saveCache([cacheDir], key), () => { logger.info( `Timed out waiting for TRAP cache for ${language} to upload, will continue without uploading` ); } ); } return true; } export async function getLanguagesSupportingCaching( codeql: CodeQL, languages: Language[], logger: Logger ): Promise { const result: Language[] = []; if ( !(await codeQlVersionAbove(codeql, CODEQL_VERSION_BETTER_RESOLVE_LANGUAGES)) ) return result; const resolveResult = await codeql.betterResolveLanguages(); outer: for (const lang of languages) { const extractorsForLanguage = resolveResult.extractors[lang]; if (extractorsForLanguage === undefined) { logger.info( `${lang} does not support TRAP caching (couldn't find an extractor)` ); continue; } if (extractorsForLanguage.length !== 1) { logger.info( `${lang} does not support TRAP caching (found multiple extractors)` ); continue; } const extractor = extractorsForLanguage[0]; const trapCacheOptions = extractor.extractor_options?.trap?.properties?.cache?.properties; if (trapCacheOptions === undefined) { logger.info( `${lang} does not support TRAP caching (missing option group)` ); continue; } for (const requiredOpt of ["dir", "bound", "write"]) { if (!(requiredOpt in trapCacheOptions)) { logger.info( `${lang} does not support TRAP caching (missing ${requiredOpt} option)` ); continue outer; } } result.push(lang); } return result; } export async function getTotalCacheSize( trapCaches: Partial>, logger: Logger ): Promise { const sizes = await Promise.all( Object.values(trapCaches).map((cacheDir) => tryGetFolderBytes(cacheDir, logger) ) ); return sizes.map((a) => a || 0).reduce((a, b) => a + b, 0); } async function cacheKey( codeql: CodeQL, language: Language, baseSha: string ): Promise { return `${await cachePrefix(codeql, language)}${baseSha}`; } async function cachePrefix( codeql: CodeQL, language: Language ): Promise { return `codeql-trap-${CACHE_VERSION}-${await codeql.getVersion()}-${language}-`; }