From 0abf548bb364e0852a4bf445eb1aa80755288fd4 Mon Sep 17 00:00:00 2001 From: Nick Rolfe Date: Wed, 10 Sep 2025 17:26:55 +0100 Subject: [PATCH] Add feature flag to roll out JAR minimization in the Java extractor --- lib/analyze-action-post.js | 5 ++++ lib/analyze-action.js | 24 +++++++++++++----- lib/autobuild-action.js | 5 ++++ lib/init-action-post.js | 5 ++++ lib/init-action.js | 42 +++++++++++++++++++++++++------ lib/resolve-environment-action.js | 5 ++++ lib/start-proxy-action-post.js | 5 ++++ lib/upload-lib.js | 5 ++++ lib/upload-sarif-action-post.js | 5 ++++ lib/upload-sarif-action.js | 5 ++++ src/analyze-action.ts | 8 ++++-- src/dependency-caching.ts | 34 +++++++++++++++++++------ src/environment.ts | 3 +++ src/feature-flags.ts | 6 +++++ src/init-action.ts | 29 ++++++++++++++++++++- 15 files changed, 163 insertions(+), 23 deletions(-) diff --git a/lib/analyze-action-post.js b/lib/analyze-action-post.js index 6f5aecb52..e07116d45 100644 --- a/lib/analyze-action-post.js +++ b/lib/analyze-action-post.js @@ -117918,6 +117918,11 @@ var featureConfig = { envVar: "CODEQL_ACTION_QA_TELEMETRY", legacyApi: true, minimumVersion: void 0 + }, + ["java_minimize_dependency_jars" /* JavaMinimizeDependencyJars */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_JAVA_MINIMIZE_DEPENDENCY_JARS", + minimumVersion: "2.23.0" } }; diff --git a/lib/analyze-action.js b/lib/analyze-action.js index b9613ab30..ac5d1425e 100644 --- a/lib/analyze-action.js +++ b/lib/analyze-action.js @@ -91152,6 +91152,11 @@ var featureConfig = { envVar: "CODEQL_ACTION_QA_TELEMETRY", legacyApi: true, minimumVersion: void 0 + }, + ["java_minimize_dependency_jars" /* JavaMinimizeDependencyJars */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_JAVA_MINIMIZE_DEPENDENCY_JARS", + minimumVersion: "2.23.0" } }; var FEATURE_FLAGS_FILE_NAME = "cached-feature-flags.json"; @@ -93241,7 +93246,7 @@ function getDefaultCacheConfig() { async function makeGlobber(patterns) { return glob.create(patterns.join("\n")); } -async function uploadDependencyCaches(config, logger) { +async function uploadDependencyCaches(config, logger, minimizeJavaJars) { for (const language of config.languages) { const cacheConfig = getDefaultCacheConfig()[language]; if (cacheConfig === void 0) { @@ -93264,7 +93269,7 @@ async function uploadDependencyCaches(config, logger) { ); continue; } - const key = await cacheKey2(language, cacheConfig); + const key = await cacheKey2(language, cacheConfig, minimizeJavaJars); logger.info( `Uploading cache of size ${size} for ${language} with key ${key}...` ); @@ -93282,17 +93287,20 @@ async function uploadDependencyCaches(config, logger) { } } } -async function cacheKey2(language, cacheConfig) { +async function cacheKey2(language, cacheConfig, minimizeJavaJars = false) { const hash2 = await glob.hashFiles(cacheConfig.hash.join("\n")); - return `${await cachePrefix2(language)}${hash2}`; + return `${await cachePrefix2(language, minimizeJavaJars)}${hash2}`; } -async function cachePrefix2(language) { +async function cachePrefix2(language, minimizeJavaJars) { const runnerOs = getRequiredEnvParam("RUNNER_OS"); const customPrefix = process.env["CODEQL_ACTION_DEPENDENCY_CACHE_PREFIX" /* DEPENDENCY_CACHING_PREFIX */]; let prefix = CODEQL_DEPENDENCY_CACHE_PREFIX; if (customPrefix !== void 0 && customPrefix.length > 0) { prefix = `${prefix}-${customPrefix}`; } + if (language === "java" /* java */ && minimizeJavaJars) { + prefix = `minify-${prefix}`; + } return `${prefix}-${CODEQL_DEPENDENCY_CACHE_VERSION}-${runnerOs}-${language}-`; } @@ -96074,7 +96082,11 @@ async function run() { logger ); if (shouldStoreCache(config.dependencyCachingEnabled)) { - await uploadDependencyCaches(config, logger); + const minimizeJavaJars = await features.getValue( + "java_minimize_dependency_jars" /* JavaMinimizeDependencyJars */, + codeql + ); + await uploadDependencyCaches(config, logger, minimizeJavaJars); } if (isInTestMode()) { logger.debug("In test mode. Waiting for processing is disabled."); diff --git a/lib/autobuild-action.js b/lib/autobuild-action.js index bf8b988bb..fd82c6461 100644 --- a/lib/autobuild-action.js +++ b/lib/autobuild-action.js @@ -78656,6 +78656,11 @@ var featureConfig = { envVar: "CODEQL_ACTION_QA_TELEMETRY", legacyApi: true, minimumVersion: void 0 + }, + ["java_minimize_dependency_jars" /* JavaMinimizeDependencyJars */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_JAVA_MINIMIZE_DEPENDENCY_JARS", + minimumVersion: "2.23.0" } }; var FEATURE_FLAGS_FILE_NAME = "cached-feature-flags.json"; diff --git a/lib/init-action-post.js b/lib/init-action-post.js index 2ce7f2c69..dd16dad19 100644 --- a/lib/init-action-post.js +++ b/lib/init-action-post.js @@ -129251,6 +129251,11 @@ var featureConfig = { envVar: "CODEQL_ACTION_QA_TELEMETRY", legacyApi: true, minimumVersion: void 0 + }, + ["java_minimize_dependency_jars" /* JavaMinimizeDependencyJars */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_JAVA_MINIMIZE_DEPENDENCY_JARS", + minimumVersion: "2.23.0" } }; var FEATURE_FLAGS_FILE_NAME = "cached-feature-flags.json"; diff --git a/lib/init-action.js b/lib/init-action.js index f790d4749..1c2814a6e 100644 --- a/lib/init-action.js +++ b/lib/init-action.js @@ -86751,6 +86751,11 @@ var featureConfig = { envVar: "CODEQL_ACTION_QA_TELEMETRY", legacyApi: true, minimumVersion: void 0 + }, + ["java_minimize_dependency_jars" /* JavaMinimizeDependencyJars */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_JAVA_MINIMIZE_DEPENDENCY_JARS", + minimumVersion: "2.23.0" } }; var FEATURE_FLAGS_FILE_NAME = "cached-feature-flags.json"; @@ -87968,7 +87973,7 @@ function getDefaultCacheConfig() { async function makeGlobber(patterns) { return glob.create(patterns.join("\n")); } -async function downloadDependencyCaches(languages, logger) { +async function downloadDependencyCaches(languages, logger, minimizeJavaJars) { const restoredCaches = []; for (const language of languages) { const cacheConfig = getDefaultCacheConfig()[language]; @@ -87985,8 +87990,10 @@ async function downloadDependencyCaches(languages, logger) { ); continue; } - const primaryKey = await cacheKey2(language, cacheConfig); - const restoreKeys = [await cachePrefix2(language)]; + const primaryKey = await cacheKey2(language, cacheConfig, minimizeJavaJars); + const restoreKeys = [ + await cachePrefix2(language, minimizeJavaJars) + ]; logger.info( `Downloading cache for ${language} with key ${primaryKey} and restore keys ${restoreKeys.join( ", " @@ -88006,17 +88013,20 @@ async function downloadDependencyCaches(languages, logger) { } return restoredCaches; } -async function cacheKey2(language, cacheConfig) { +async function cacheKey2(language, cacheConfig, minimizeJavaJars = false) { const hash = await glob.hashFiles(cacheConfig.hash.join("\n")); - return `${await cachePrefix2(language)}${hash}`; + return `${await cachePrefix2(language, minimizeJavaJars)}${hash}`; } -async function cachePrefix2(language) { +async function cachePrefix2(language, minimizeJavaJars) { const runnerOs = getRequiredEnvParam("RUNNER_OS"); const customPrefix = process.env["CODEQL_ACTION_DEPENDENCY_CACHE_PREFIX" /* DEPENDENCY_CACHING_PREFIX */]; let prefix = CODEQL_DEPENDENCY_CACHE_PREFIX; if (customPrefix !== void 0 && customPrefix.length > 0) { prefix = `${prefix}-${customPrefix}`; } + if (language === "java" /* java */ && minimizeJavaJars) { + prefix = `minify-${prefix}`; + } return `${prefix}-${CODEQL_DEPENDENCY_CACHE_VERSION}-${runnerOs}-${language}-`; } @@ -90622,8 +90632,16 @@ exec ${goBinaryPath} "$@"` core13.exportVariable(envVar, "false"); } } + const minimizeJavaJars = await features.getValue( + "java_minimize_dependency_jars" /* JavaMinimizeDependencyJars */, + codeql + ); if (shouldRestoreCache(config.dependencyCachingEnabled)) { - await downloadDependencyCaches(config.languages, logger); + await downloadDependencyCaches( + config.languages, + logger, + minimizeJavaJars + ); } if (await codeQlVersionAtLeast(codeql, "2.17.1")) { } else { @@ -90656,6 +90674,16 @@ exec ${goBinaryPath} "$@"` core13.exportVariable("CODEQL_EXTRACTOR_PYTHON_EXTRACT_STDLIB", "true"); } } + if (process.env["CODEQL_EXTRACTOR_JAVA_OPTION_MINIMIZE_DEPENDENCY_JARS" /* JAVA_EXTRACTOR_MINIMIZE_DEPENDENCY_JARS */]) { + logger.debug( + `${"CODEQL_EXTRACTOR_JAVA_OPTION_MINIMIZE_DEPENDENCY_JARS" /* JAVA_EXTRACTOR_MINIMIZE_DEPENDENCY_JARS */} is already set to '${process.env["CODEQL_EXTRACTOR_JAVA_OPTION_MINIMIZE_DEPENDENCY_JARS" /* JAVA_EXTRACTOR_MINIMIZE_DEPENDENCY_JARS */]}', so the Action will not override it.` + ); + } else if (minimizeJavaJars && config.buildMode === "none" /* None */ && config.languages.includes("java" /* java */)) { + core13.exportVariable( + "CODEQL_EXTRACTOR_JAVA_OPTION_MINIMIZE_DEPENDENCY_JARS" /* JAVA_EXTRACTOR_MINIMIZE_DEPENDENCY_JARS */, + "true" + ); + } const { registriesAuthTokens, qlconfigFile } = await generateRegistries( getOptionalInput("registries"), config.tempDir, diff --git a/lib/resolve-environment-action.js b/lib/resolve-environment-action.js index a71808c9b..f83993d01 100644 --- a/lib/resolve-environment-action.js +++ b/lib/resolve-environment-action.js @@ -78647,6 +78647,11 @@ var featureConfig = { envVar: "CODEQL_ACTION_QA_TELEMETRY", legacyApi: true, minimumVersion: void 0 + }, + ["java_minimize_dependency_jars" /* JavaMinimizeDependencyJars */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_JAVA_MINIMIZE_DEPENDENCY_JARS", + minimumVersion: "2.23.0" } }; diff --git a/lib/start-proxy-action-post.js b/lib/start-proxy-action-post.js index 92b143742..9e74768f1 100644 --- a/lib/start-proxy-action-post.js +++ b/lib/start-proxy-action-post.js @@ -117327,6 +117327,11 @@ var featureConfig = { envVar: "CODEQL_ACTION_QA_TELEMETRY", legacyApi: true, minimumVersion: void 0 + }, + ["java_minimize_dependency_jars" /* JavaMinimizeDependencyJars */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_JAVA_MINIMIZE_DEPENDENCY_JARS", + minimumVersion: "2.23.0" } }; diff --git a/lib/upload-lib.js b/lib/upload-lib.js index fd8be1bd6..7c0b6aa23 100644 --- a/lib/upload-lib.js +++ b/lib/upload-lib.js @@ -89343,6 +89343,11 @@ var featureConfig = { envVar: "CODEQL_ACTION_QA_TELEMETRY", legacyApi: true, minimumVersion: void 0 + }, + ["java_minimize_dependency_jars" /* JavaMinimizeDependencyJars */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_JAVA_MINIMIZE_DEPENDENCY_JARS", + minimumVersion: "2.23.0" } }; diff --git a/lib/upload-sarif-action-post.js b/lib/upload-sarif-action-post.js index 1a87ff27c..aa56d1f55 100644 --- a/lib/upload-sarif-action-post.js +++ b/lib/upload-sarif-action-post.js @@ -117492,6 +117492,11 @@ var featureConfig = { envVar: "CODEQL_ACTION_QA_TELEMETRY", legacyApi: true, minimumVersion: void 0 + }, + ["java_minimize_dependency_jars" /* JavaMinimizeDependencyJars */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_JAVA_MINIMIZE_DEPENDENCY_JARS", + minimumVersion: "2.23.0" } }; diff --git a/lib/upload-sarif-action.js b/lib/upload-sarif-action.js index edf8de1d3..9dbcd66bd 100644 --- a/lib/upload-sarif-action.js +++ b/lib/upload-sarif-action.js @@ -89339,6 +89339,11 @@ var featureConfig = { envVar: "CODEQL_ACTION_QA_TELEMETRY", legacyApi: true, minimumVersion: void 0 + }, + ["java_minimize_dependency_jars" /* JavaMinimizeDependencyJars */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_JAVA_MINIMIZE_DEPENDENCY_JARS", + minimumVersion: "2.23.0" } }; var FEATURE_FLAGS_FILE_NAME = "cached-feature-flags.json"; diff --git a/src/analyze-action.ts b/src/analyze-action.ts index f93072d72..31251be48 100644 --- a/src/analyze-action.ts +++ b/src/analyze-action.ts @@ -29,7 +29,7 @@ import { uploadDatabases } from "./database-upload"; import { uploadDependencyCaches } from "./dependency-caching"; import { getDiffInformedAnalysisBranches } from "./diff-informed-analysis-utils"; import { EnvVar } from "./environment"; -import { Features } from "./feature-flags"; +import { Feature, Features } from "./feature-flags"; import { KnownLanguage } from "./languages"; import { getActionsLogger, Logger } from "./logging"; import { uploadOverlayBaseDatabaseToCache } from "./overlay-database-utils"; @@ -384,7 +384,11 @@ async function run() { // Store dependency cache(s) if dependency caching is enabled. if (shouldStoreCache(config.dependencyCachingEnabled)) { - await uploadDependencyCaches(config, logger); + const minimizeJavaJars = await features.getValue( + Feature.JavaMinimizeDependencyJars, + codeql, + ); + await uploadDependencyCaches(config, logger, minimizeJavaJars); } // We don't upload results in test mode, so don't wait for processing diff --git a/src/dependency-caching.ts b/src/dependency-caching.ts index ae6f21539..6289ca2f6 100644 --- a/src/dependency-caching.ts +++ b/src/dependency-caching.ts @@ -8,7 +8,7 @@ import { getTemporaryDirectory } from "./actions-util"; import { getTotalCacheSize } from "./caching-utils"; import { Config } from "./config-utils"; import { EnvVar } from "./environment"; -import { Language } from "./languages"; +import { KnownLanguage, Language } from "./languages"; import { Logger } from "./logging"; import { getRequiredEnvParam } from "./util"; @@ -89,11 +89,13 @@ async function makeGlobber(patterns: string[]): Promise { * * @param languages The languages being analyzed. * @param logger A logger to record some informational messages to. + * @param minimizeJavaJars Whether the Java extractor should rewrite downloaded JARs to minimize their size. * @returns A list of languages for which dependency caches were restored. */ export async function downloadDependencyCaches( languages: Language[], logger: Logger, + minimizeJavaJars: boolean, ): Promise { const restoredCaches: Language[] = []; @@ -118,8 +120,10 @@ export async function downloadDependencyCaches( continue; } - const primaryKey = await cacheKey(language, cacheConfig); - const restoreKeys: string[] = [await cachePrefix(language)]; + const primaryKey = await cacheKey(language, cacheConfig, minimizeJavaJars); + const restoreKeys: string[] = [ + await cachePrefix(language, minimizeJavaJars), + ]; logger.info( `Downloading cache for ${language} with key ${primaryKey} and restore keys ${restoreKeys.join( @@ -149,8 +153,13 @@ export async function downloadDependencyCaches( * * @param config The configuration for this workflow. * @param logger A logger to record some informational messages to. + * @param minimizeJavaJars Whether the Java extractor should rewrite downloaded JARs to minimize their size. */ -export async function uploadDependencyCaches(config: Config, logger: Logger) { +export async function uploadDependencyCaches( + config: Config, + logger: Logger, + minimizeJavaJars: boolean, +): Promise { for (const language of config.languages) { const cacheConfig = getDefaultCacheConfig()[language]; @@ -192,7 +201,7 @@ export async function uploadDependencyCaches(config: Config, logger: Logger) { continue; } - const key = await cacheKey(language, cacheConfig); + const key = await cacheKey(language, cacheConfig, minimizeJavaJars); logger.info( `Uploading cache of size ${size} for ${language} with key ${key}...`, @@ -222,14 +231,16 @@ export async function uploadDependencyCaches(config: Config, logger: Logger) { * * @param language The language being analyzed. * @param cacheConfig The cache configuration for the language. + * @param minimizeJavaJars Whether the Java extractor should rewrite downloaded JARs to minimize their size. * @returns A cache key capturing information about the project(s) being analyzed in the specified language. */ async function cacheKey( language: Language, cacheConfig: CacheConfig, + minimizeJavaJars: boolean = false, ): Promise { const hash = await glob.hashFiles(cacheConfig.hash.join("\n")); - return `${await cachePrefix(language)}${hash}`; + return `${await cachePrefix(language, minimizeJavaJars)}${hash}`; } /** @@ -237,9 +248,13 @@ async function cacheKey( * can be changed to invalidate old caches, the runner's operating system, and the specified language name. * * @param language The language being analyzed. + * @param minimizeJavaJars Whether the Java extractor should rewrite downloaded JARs to minimize their size. * @returns The prefix that identifies what a cache is for. */ -async function cachePrefix(language: Language): Promise { +async function cachePrefix( + language: Language, + minimizeJavaJars: boolean, +): Promise { const runnerOs = getRequiredEnvParam("RUNNER_OS"); const customPrefix = process.env[EnvVar.DEPENDENCY_CACHING_PREFIX]; let prefix = CODEQL_DEPENDENCY_CACHE_PREFIX; @@ -248,5 +263,10 @@ async function cachePrefix(language: Language): Promise { prefix = `${prefix}-${customPrefix}`; } + // To ensure a safe rollout of JAR minimization, we change the key when the feature is enabled. + if (language === KnownLanguage.java && minimizeJavaJars) { + prefix = `minify-${prefix}`; + } + return `${prefix}-${CODEQL_DEPENDENCY_CACHE_VERSION}-${runnerOs}-${language}-`; } diff --git a/src/environment.ts b/src/environment.ts index e78c36724..e1daeedc2 100644 --- a/src/environment.ts +++ b/src/environment.ts @@ -115,6 +115,9 @@ export enum EnvVar { */ DEPENDENCY_CACHING_PREFIX = "CODEQL_ACTION_DEPENDENCY_CACHE_PREFIX", + /** Used by the Java extractor option to enable minimizing dependency JARs. */ + JAVA_EXTRACTOR_MINIMIZE_DEPENDENCY_JARS = "CODEQL_EXTRACTOR_JAVA_OPTION_MINIMIZE_DEPENDENCY_JARS", + /** * Whether to enable experimental extractors for CodeQL. */ diff --git a/src/feature-flags.ts b/src/feature-flags.ts index ef6747d6d..b7946d62f 100644 --- a/src/feature-flags.ts +++ b/src/feature-flags.ts @@ -50,6 +50,7 @@ export enum Feature { DisableJavaBuildlessEnabled = "disable_java_buildless_enabled", DisableKotlinAnalysisEnabled = "disable_kotlin_analysis_enabled", ExportDiagnosticsEnabled = "export_diagnostics_enabled", + JavaMinimizeDependencyJars = "java_minimize_dependency_jars", OverlayAnalysis = "overlay_analysis", OverlayAnalysisActions = "overlay_analysis_actions", OverlayAnalysisCodeScanningActions = "overlay_analysis_code_scanning_actions", @@ -269,6 +270,11 @@ export const featureConfig: Record< legacyApi: true, minimumVersion: undefined, }, + [Feature.JavaMinimizeDependencyJars]: { + defaultValue: false, + envVar: "CODEQL_ACTION_JAVA_MINIMIZE_DEPENDENCY_JARS", + minimumVersion: "2.23.0", + }, }; /** diff --git a/src/init-action.ts b/src/init-action.ts index aa97b65fc..c28b1e72c 100644 --- a/src/init-action.ts +++ b/src/init-action.ts @@ -78,6 +78,7 @@ import { wrapError, checkActionVersion, getErrorMessage, + BuildMode, } from "./util"; import { validateWorkflow } from "./workflow"; @@ -546,8 +547,16 @@ async function run() { } // Restore dependency cache(s), if they exist. + const minimizeJavaJars = await features.getValue( + Feature.JavaMinimizeDependencyJars, + codeql, + ); if (shouldRestoreCache(config.dependencyCachingEnabled)) { - await downloadDependencyCaches(config.languages, logger); + await downloadDependencyCaches( + config.languages, + logger, + minimizeJavaJars, + ); } // Suppress warnings about disabled Python library extraction. @@ -597,6 +606,24 @@ async function run() { } } + // If the feature flag to minimize Java dependency jars is enabled, and we are doing a Java + // `build-mode: none` analysis (i.e. the flag is relevant), then set the environment variable + // that enables the corresponding option in the Java extractor. + if (process.env[EnvVar.JAVA_EXTRACTOR_MINIMIZE_DEPENDENCY_JARS]) { + logger.debug( + `${EnvVar.JAVA_EXTRACTOR_MINIMIZE_DEPENDENCY_JARS} is already set to '${process.env[EnvVar.JAVA_EXTRACTOR_MINIMIZE_DEPENDENCY_JARS]}', so the Action will not override it.`, + ); + } else if ( + minimizeJavaJars && + config.buildMode === BuildMode.None && + config.languages.includes(KnownLanguage.java) + ) { + core.exportVariable( + EnvVar.JAVA_EXTRACTOR_MINIMIZE_DEPENDENCY_JARS, + "true", + ); + } + const { registriesAuthTokens, qlconfigFile } = await configUtils.generateRegistries( getOptionalInput("registries"),