Move overlay db settings from AugmentationProperties to Config

This commit is contained in:
Michael B. Gale
2025-09-02 12:45:17 +01:00
parent c4d2772a75
commit ac9b91e745
18 changed files with 79 additions and 183 deletions

View File

@@ -626,10 +626,7 @@ export async function runQueries(
const incrementalMode: string[] = [];
// Preserve cached intermediate results for overlay-base databases.
if (
config.augmentationProperties.overlayDatabaseMode !==
OverlayDatabaseMode.OverlayBase
) {
if (config.overlayDatabaseMode !== OverlayDatabaseMode.OverlayBase) {
queryFlags.push("--expect-discarded-cache");
}
@@ -641,15 +638,10 @@ export async function runQueries(
}
statusReport.analysis_is_overlay =
config.augmentationProperties.overlayDatabaseMode ===
OverlayDatabaseMode.Overlay;
config.overlayDatabaseMode === OverlayDatabaseMode.Overlay;
statusReport.analysis_builds_overlay_base_database =
config.augmentationProperties.overlayDatabaseMode ===
OverlayDatabaseMode.OverlayBase;
if (
config.augmentationProperties.overlayDatabaseMode ===
OverlayDatabaseMode.Overlay
) {
config.overlayDatabaseMode === OverlayDatabaseMode.OverlayBase;
if (config.overlayDatabaseMode === OverlayDatabaseMode.Overlay) {
incrementalMode.push("overlay");
}

View File

@@ -593,8 +593,7 @@ export async function getCodeQLForCmd(
? "--force-overwrite"
: "--overwrite";
const overlayDatabaseMode =
config.augmentationProperties.overlayDatabaseMode;
const overlayDatabaseMode = config.overlayDatabaseMode;
if (overlayDatabaseMode === OverlayDatabaseMode.Overlay) {
const overlayChangesFile = await writeOverlayChangesFile(
config,

View File

@@ -348,6 +348,8 @@ test("load non-empty input", async (t) => {
trapCaches: {},
trapCacheDownloadTime: 0,
dependencyCachingEnabled: CachingKind.None,
overlayDatabaseMode: OverlayDatabaseMode.None,
useOverlayDatabaseCaching: false,
};
const languagesInput = "javascript";

View File

@@ -169,6 +169,23 @@ export interface Config {
/** A value indicating how dependency caching should be used. */
dependencyCachingEnabled: CachingKind;
/**
* The overlay database mode to use.
*/
overlayDatabaseMode: OverlayDatabaseMode;
/**
* Whether to use caching for overlay databases. If it is true, the action
* will upload the created overlay-base database to the actions cache, and
* download an overlay-base database from the actions cache before it creates
* a new overlay database. If it is false, the action assumes that the
* workflow will be responsible for managing database storage and retrieval.
*
* This property has no effect unless `overlayDatabaseMode` is `Overlay` or
* `OverlayBase`.
*/
useOverlayDatabaseCaching: boolean;
}
/**
@@ -206,23 +223,6 @@ export interface AugmentationProperties {
* Extra query exclusions to append to the config.
*/
extraQueryExclusions: ExcludeQueryFilter[];
/**
* The overlay database mode to use.
*/
overlayDatabaseMode: OverlayDatabaseMode;
/**
* Whether to use caching for overlay databases. If it is true, the action
* will upload the created overlay-base database to the actions cache, and
* download an overlay-base database from the actions cache before it creates
* a new overlay database. If it is false, the action assumes that the
* workflow will be responsible for managing database storage and retrieval.
*
* This property has no effect unless `overlayDatabaseMode` is `Overlay` or
* `OverlayBase`.
*/
useOverlayDatabaseCaching: boolean;
}
/**
@@ -235,8 +235,6 @@ export const defaultAugmentationProperties: AugmentationProperties = {
packsInput: undefined,
queriesInput: undefined,
extraQueryExclusions: [],
overlayDatabaseMode: OverlayDatabaseMode.None,
useOverlayDatabaseCaching: false,
};
export type Packs = Partial<Record<Language, string[]>>;
@@ -597,6 +595,8 @@ export async function getDefaultConfig({
trapCaches,
trapCacheDownloadTime,
dependencyCachingEnabled: getCachingKind(dependencyCachingEnabled),
overlayDatabaseMode: OverlayDatabaseMode.None,
useOverlayDatabaseCaching: false,
};
}
@@ -684,8 +684,6 @@ export async function calculateAugmentation(
queriesInput,
queriesInputCombines,
extraQueryExclusions: [],
overlayDatabaseMode: OverlayDatabaseMode.None,
useOverlayDatabaseCaching: false,
};
}
@@ -1108,7 +1106,6 @@ export async function initConfig(inputs: InitConfigInputs): Promise<Config> {
}
const config = await getDefaultConfig(inputs);
const augmentationProperties = config.augmentationProperties;
config.originalUserInput = userConfig;
// Compute the full Code Scanning configuration that combines the configuration from the
@@ -1137,8 +1134,8 @@ export async function initConfig(inputs: InitConfigInputs): Promise<Config> {
`Using overlay database mode: ${overlayDatabaseMode} ` +
`${useOverlayDatabaseCaching ? "with" : "without"} caching.`,
);
augmentationProperties.overlayDatabaseMode = overlayDatabaseMode;
augmentationProperties.useOverlayDatabaseCaching = useOverlayDatabaseCaching;
config.overlayDatabaseMode = overlayDatabaseMode;
config.useOverlayDatabaseCaching = useOverlayDatabaseCaching;
if (
overlayDatabaseMode === OverlayDatabaseMode.Overlay ||
@@ -1148,7 +1145,10 @@ export async function initConfig(inputs: InitConfigInputs): Promise<Config> {
logger,
))
) {
augmentationProperties.extraQueryExclusions.push({
if (config.computedConfig["query-filters"] === undefined) {
config.computedConfig["query-filters"] = [];
}
config.computedConfig["query-filters"].push({
exclude: { tags: "exclude-from-incremental" },
});
}

View File

@@ -449,9 +449,8 @@ async function run() {
let overlayBaseDatabaseStats: OverlayBaseDatabaseDownloadStats | undefined;
try {
if (
config.augmentationProperties.overlayDatabaseMode ===
OverlayDatabaseMode.Overlay &&
config.augmentationProperties.useOverlayDatabaseCaching
config.overlayDatabaseMode === OverlayDatabaseMode.Overlay &&
config.useOverlayDatabaseCaching
) {
// OverlayDatabaseMode.Overlay comes in two flavors: with database
// caching, or without. The flavor with database caching is intended to be
@@ -470,8 +469,7 @@ async function run() {
logger,
);
if (!overlayBaseDatabaseStats) {
config.augmentationProperties.overlayDatabaseMode =
OverlayDatabaseMode.None;
config.overlayDatabaseMode = OverlayDatabaseMode.None;
logger.info(
"No overlay-base database found in cache, " +
`reverting overlay database mode to ${OverlayDatabaseMode.None}.`,
@@ -479,10 +477,7 @@ async function run() {
}
}
if (
config.augmentationProperties.overlayDatabaseMode !==
OverlayDatabaseMode.Overlay
) {
if (config.overlayDatabaseMode !== OverlayDatabaseMode.Overlay) {
cleanupDatabaseClusterDirectory(config, logger);
}
@@ -739,15 +734,13 @@ async function run() {
// revert to `OverlayDatabaseMode.None`, re-initialize the database cluster
// with the new overlay database mode.
if (
config.augmentationProperties.overlayDatabaseMode !==
OverlayDatabaseMode.None &&
config.overlayDatabaseMode !== OverlayDatabaseMode.None &&
!(await checkPacksForOverlayCompatibility(codeql, config, logger))
) {
logger.info(
"Reverting overlay database mode to None due to incompatible packs.",
);
config.augmentationProperties.overlayDatabaseMode =
OverlayDatabaseMode.None;
config.overlayDatabaseMode = OverlayDatabaseMode.None;
cleanupDatabaseClusterDirectory(config, logger, {
disableExistingDirectoryWarning: true,
});

View File

@@ -120,10 +120,8 @@ const testDownloadOverlayBaseDatabaseFromCache = test.macro({
const testCase = { ...defaultDownloadTestCase, ...partialTestCase };
config.augmentationProperties.overlayDatabaseMode =
testCase.overlayDatabaseMode;
config.augmentationProperties.useOverlayDatabaseCaching =
testCase.useOverlayDatabaseCaching;
config.overlayDatabaseMode = testCase.overlayDatabaseMode;
config.useOverlayDatabaseCaching = testCase.useOverlayDatabaseCaching;
if (testCase.hasBaseDatabaseOidsFile) {
const baseDatabaseOidsFile = path.join(

View File

@@ -192,7 +192,7 @@ export async function uploadOverlayBaseDatabaseToCache(
config: Config,
logger: Logger,
): Promise<boolean> {
const overlayDatabaseMode = config.augmentationProperties.overlayDatabaseMode;
const overlayDatabaseMode = config.overlayDatabaseMode;
if (overlayDatabaseMode !== OverlayDatabaseMode.OverlayBase) {
logger.debug(
`Overlay database mode is ${overlayDatabaseMode}. ` +
@@ -200,7 +200,7 @@ export async function uploadOverlayBaseDatabaseToCache(
);
return false;
}
if (!config.augmentationProperties.useOverlayDatabaseCaching) {
if (!config.useOverlayDatabaseCaching) {
logger.debug(
"Overlay database caching is disabled. " +
"Skip uploading overlay-base database to cache.",
@@ -298,7 +298,7 @@ export async function downloadOverlayBaseDatabaseFromCache(
config: Config,
logger: Logger,
): Promise<OverlayBaseDatabaseDownloadStats | undefined> {
const overlayDatabaseMode = config.augmentationProperties.overlayDatabaseMode;
const overlayDatabaseMode = config.overlayDatabaseMode;
if (overlayDatabaseMode !== OverlayDatabaseMode.Overlay) {
logger.debug(
`Overlay database mode is ${overlayDatabaseMode}. ` +
@@ -306,7 +306,7 @@ export async function downloadOverlayBaseDatabaseFromCache(
);
return undefined;
}
if (!config.augmentationProperties.useOverlayDatabaseCaching) {
if (!config.useOverlayDatabaseCaching) {
logger.debug(
"Overlay database caching is disabled. " +
"Skip downloading overlay-base database from cache.",

View File

@@ -11,7 +11,7 @@ import * as apiClient from "./api-client";
import { GitHubApiDetails } from "./api-client";
import { CachingKind } from "./caching-utils";
import * as codeql from "./codeql";
import { Config } from "./config-utils";
import { AugmentationProperties, Config } from "./config-utils";
import * as defaults from "./defaults.json";
import {
CodeQLDefaultVersionInfo,
@@ -374,12 +374,12 @@ export function createTestConfig(overrides: Partial<Config>): Config {
packsInputCombines: false,
queriesInputCombines: false,
extraQueryExclusions: [],
overlayDatabaseMode: OverlayDatabaseMode.None,
useOverlayDatabaseCaching: false,
},
} satisfies AugmentationProperties,
trapCaches: {},
trapCacheDownloadTime: 0,
dependencyCachingEnabled: CachingKind.None,
overlayDatabaseMode: OverlayDatabaseMode.None,
useOverlayDatabaseCaching: false,
} satisfies Config,
overrides,
);