diff --git a/src/init-action.ts b/src/init-action.ts index 79eb73ba4..ed579018c 100644 --- a/src/init-action.ts +++ b/src/init-action.ts @@ -44,6 +44,7 @@ import { Language } from "./languages"; import { getActionsLogger, Logger } from "./logging"; import { downloadOverlayBaseDatabaseFromCache, + OverlayBaseDatabaseDownloadStats, OverlayDatabaseMode, } from "./overlay-database-utils"; import { getRepositoryNwo } from "./repository"; @@ -107,6 +108,10 @@ interface InitWithConfigStatusReport extends InitStatusReport { trap_cache_download_size_bytes: number; /** Time taken to download TRAP caches, in milliseconds. */ trap_cache_download_duration_ms: number; + /** Size of the overlay-base database that we downloaded, in bytes. */ + overlay_base_database_download_size_bytes?: number; + /** Time taken to download the overlay-base database, in milliseconds. */ + overlay_base_database_download_duration_ms?: number; /** Stringified JSON array of registry configuration objects, from the 'registries' config field or workflow input. **/ registries: string; @@ -134,6 +139,7 @@ async function sendCompletedStatusReport( toolsFeatureFlagsValid: boolean | undefined, toolsSource: ToolsSource, toolsVersion: string, + overlayBaseDatabaseStats: OverlayBaseDatabaseDownloadStats | undefined, logger: Logger, error?: Error, ) { @@ -237,6 +243,10 @@ async function sendCompletedStatusReport( await getTotalCacheSize(Object.values(config.trapCaches), logger), ), trap_cache_download_duration_ms: Math.round(config.trapCacheDownloadTime), + overlay_base_database_download_size_bytes: + overlayBaseDatabaseStats?.databaseSizeBytes, + overlay_base_database_download_duration_ms: + overlayBaseDatabaseStats?.databaseDownloadDurationMs, query_filters: JSON.stringify( config.originalUserInput["query-filters"] ?? [], ), @@ -400,6 +410,7 @@ async function run() { return; } + let overlayBaseDatabaseStats: OverlayBaseDatabaseDownloadStats | undefined; try { if ( config.augmentationProperties.overlayDatabaseMode === @@ -417,9 +428,12 @@ async function run() { // necessary preparations. So, in that mode, we would assume that // everything is in order and let the analysis fail if that turns out not // to be the case. - const overlayDatabaseDownloaded = - await downloadOverlayBaseDatabaseFromCache(codeql, config, logger); - if (!overlayDatabaseDownloaded) { + overlayBaseDatabaseStats = await downloadOverlayBaseDatabaseFromCache( + codeql, + config, + logger, + ); + if (!overlayBaseDatabaseStats) { config.augmentationProperties.overlayDatabaseMode = OverlayDatabaseMode.None; logger.info( @@ -729,6 +743,7 @@ async function run() { toolsFeatureFlagsValid, toolsSource, toolsVersion, + overlayBaseDatabaseStats, logger, error, ); @@ -744,6 +759,7 @@ async function run() { toolsFeatureFlagsValid, toolsSource, toolsVersion, + overlayBaseDatabaseStats, logger, ); } diff --git a/src/overlay-database-utils.ts b/src/overlay-database-utils.ts index c88cea261..bd7d12107 100644 --- a/src/overlay-database-utils.ts +++ b/src/overlay-database-utils.ts @@ -8,7 +8,7 @@ import { type CodeQL } from "./codeql"; import { type Config } from "./config-utils"; import { getCommitOid, getFileOidsUnderPath } from "./git-utils"; import { Logger } from "./logging"; -import { isInTestMode, withTimeout } from "./util"; +import { isInTestMode, tryGetFolderBytes, withTimeout } from "./util"; export enum OverlayDatabaseMode { Overlay = "overlay", @@ -235,6 +235,11 @@ export async function uploadOverlayBaseDatabaseToCache( return true; } +export interface OverlayBaseDatabaseDownloadStats { + databaseSizeBytes: number; + databaseDownloadDurationMs: number; +} + /** * Downloads the overlay-base database from the GitHub Actions cache. If conditions * for downloading are not met, the function does nothing and returns false. @@ -242,34 +247,35 @@ export async function uploadOverlayBaseDatabaseToCache( * @param codeql The CodeQL instance * @param config The configuration object * @param logger The logger instance - * @returns A promise that resolves to true if the download was performed and - * successfully completed, or false otherwise + * @returns A promise that resolves to download statistics if an overlay-base + * database was successfully downloaded, or undefined if the download was + * either not performed or failed. */ export async function downloadOverlayBaseDatabaseFromCache( codeql: CodeQL, config: Config, logger: Logger, -): Promise { +): Promise { const overlayDatabaseMode = config.augmentationProperties.overlayDatabaseMode; if (overlayDatabaseMode !== OverlayDatabaseMode.Overlay) { logger.debug( `Overlay database mode is ${overlayDatabaseMode}. ` + "Skip downloading overlay-base database from cache.", ); - return false; + return undefined; } if (!config.augmentationProperties.useOverlayDatabaseCaching) { logger.debug( "Overlay database caching is disabled. " + "Skip downloading overlay-base database from cache.", ); - return false; + return undefined; } if (isInTestMode()) { logger.debug( "In test mode. Skip downloading overlay-base database from cache.", ); - return false; + return undefined; } const dbLocation = config.dbLocation; @@ -280,7 +286,9 @@ export async function downloadOverlayBaseDatabaseFromCache( `Looking in Actions cache for overlay-base database with restore key ${restoreKey}`, ); + let databaseDownloadDurationMs = 0; try { + const databaseDownloadStart = performance.now(); const foundKey = await withTimeout( MAX_CACHE_OPERATION_MS, actionsCache.restoreCache([dbLocation], restoreKey), @@ -288,10 +296,13 @@ export async function downloadOverlayBaseDatabaseFromCache( logger.info("Timed out downloading overlay-base database from cache"); }, ); + databaseDownloadDurationMs = Math.round( + performance.now() - databaseDownloadStart, + ); if (foundKey === undefined) { logger.info("No overlay-base database found in Actions cache"); - return false; + return undefined; } logger.info( @@ -302,7 +313,7 @@ export async function downloadOverlayBaseDatabaseFromCache( "Failed to download overlay-base database from cache: " + `${error instanceof Error ? error.message : String(error)}`, ); - return false; + return undefined; } const databaseIsValid = checkOverlayBaseDatabase( @@ -312,11 +323,26 @@ export async function downloadOverlayBaseDatabaseFromCache( ); if (!databaseIsValid) { logger.warning("Downloaded overlay-base database failed validation"); - return false; + return undefined; + } + + const databaseSizeBytes = await tryGetFolderBytes(dbLocation, logger); + if (databaseSizeBytes === undefined) { + logger.info( + "Filesystem error while accessing downloaded overlay-base database", + ); + // The problem that warrants reporting download failure is not that we are + // unable to determine the size of the database. Rather, it is that we + // encountered a filesystem error while accessing the database, which + // indicates that an overlay analysis will likely fail. + return undefined; } logger.info(`Successfully downloaded overlay-base database to ${dbLocation}`); - return true; + return { + databaseSizeBytes: Math.round(databaseSizeBytes), + databaseDownloadDurationMs, + }; } async function generateCacheKey(