Compare commits

..

23 Commits

Author SHA1 Message Date
Michael B. Gale
1cf30a546e Add global.json 2025-11-13 19:18:54 +00:00
Michael B. Gale
ba454b8ab4 Merge pull request #3284 from github/mbg/ci/fix-enterprise-workflow
Fix `update-supported-enterprise-server-versions.yml` workflow
2025-11-12 15:35:56 +00:00
Michael B. Gale
7a7cd8565c Don't push for PR event 2025-11-12 15:09:25 +00:00
Michael B. Gale
fd830db27b Trigger on PR for relevant changes 2025-11-12 15:05:11 +00:00
Michael B. Gale
a7e52b690b Perform sparse checkout 2025-11-12 15:04:21 +00:00
Michael B. Gale
71c3720f43 Run npm ci in update-supported-enterprise-server-versions.yml 2025-11-12 14:57:05 +00:00
Michael B. Gale
534824ea1b Merge pull request #3117 from github/mbg/csharp/new-cache-key-calculation
Support non-lock files for C# cache key computation
2025-11-12 11:03:09 +00:00
Michael B. Gale
48a56f6b93 Add some tests for downloadDependencyCaches related to feature prefixes 2025-11-09 12:03:18 +00:00
Michael B. Gale
4885eb2ad9 Insert new featurePrefix after general cache key prefix 2025-11-09 11:17:02 +00:00
Michael B. Gale
a47d5507cf Restore earlier log messages for checkHashPatterns 2025-11-09 11:11:10 +00:00
Michael B. Gale
b0e9dfce55 Restore missing status.push resulting from a bad merge 2025-11-09 11:10:27 +00:00
Michael B. Gale
35c91ef0af Add tests for getCsharpHashPatterns
- Make the function more easily testable by allowing `makePatternCheck` to be stubbed.
- Use `makePatternCheck` for base patterns as well.
2025-11-09 11:03:15 +00:00
Michael B. Gale
71abac76d2 Fix comment in getCsharpHashPatterns 2025-11-09 10:32:10 +00:00
Michael B. Gale
46e03b48bc Fix JSDoc param name 2025-11-09 10:05:18 +00:00
Michael B. Gale
26804552e4 Use undefined instead of NoMatchingFilesError
Add tests for `makePatternCheck` and `checkHashPatterns`
2025-11-05 17:23:22 +00:00
Michael B. Gale
03b2dc2a3f Add and use getFeaturePrefix for dependency caching 2025-11-05 16:33:21 +00:00
Michael B. Gale
0cbd930deb Move createCacheKeyHash to caching-utils 2025-11-05 16:15:26 +00:00
Michael B. Gale
0324490286 Use additional files for C# key hashes if Feature.CsharpNewCacheKey is enabled 2025-11-05 16:03:41 +00:00
Michael B. Gale
6b48207907 Move check whether there are files for hashing into getHashPatterns 2025-11-05 16:03:39 +00:00
Michael B. Gale
ab1c84236a Change hash to be a function that can use Features 2025-11-05 15:57:57 +00:00
Michael B. Gale
2a7680fca6 Change getDefaultCacheConfig to be a const by turning paths into a function
Changing `paths` to be a function is necessary to allow `getTemporaryDirectory` to be called
2025-11-05 15:57:55 +00:00
Michael B. Gale
2aa1f55f3d Propagate features into cachePrefix function 2025-11-05 15:54:28 +00:00
Michael B. Gale
1ca20ab026 Add CsharpNewCacheKey FF 2025-11-05 15:48:04 +00:00
29 changed files with 1190 additions and 623 deletions

View File

@@ -4,6 +4,12 @@ on:
schedule:
- cron: "0 0 * * *"
workflow_dispatch:
pull_request:
branches:
- main
paths:
- .github/workflows/update-supported-enterprise-server-versions.yml
- .github/workflows/update-supported-enterprise-server-versions/update.py
jobs:
update-supported-enterprise-server-versions:
@@ -28,6 +34,7 @@ jobs:
repository: github/enterprise-releases
token: ${{ secrets.ENTERPRISE_RELEASE_TOKEN }}
path: ${{ github.workspace }}/enterprise-releases/
sparse-checkout: releases.json
- name: Update Supported Enterprise Server Versions
run: |
cd ./.github/workflows/update-supported-enterprise-server-versions/
@@ -35,6 +42,7 @@ jobs:
pipenv install
pipenv run ./update.py
rm --recursive "$ENTERPRISE_RELEASES_PATH"
npm ci
npm run build
env:
ENTERPRISE_RELEASES_PATH: ${{ github.workspace }}/enterprise-releases/
@@ -44,25 +52,33 @@ jobs:
git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
git config --global user.name "github-actions[bot]"
- name: Commit changes and open PR
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Commit changes
id: prepare-commit
run: |
if [[ -z $(git status --porcelain) ]]; then
echo "No changes to commit"
echo "committed=false" >> $GITHUB_OUTPUT
else
git checkout -b update-supported-enterprise-server-versions
git add .
git commit --message "Update supported GitHub Enterprise Server versions"
git push origin update-supported-enterprise-server-versions
body="This PR updates the list of supported GitHub Enterprise Server versions, either because a new "
body+="version is about to be feature frozen, or because an old release has been deprecated."
body+=$'\n\n'
body+="If an old release has been deprecated, please follow the instructions in CONTRIBUTING.md to "
body+="deprecate the corresponding version of CodeQL."
gh pr create --draft \
--title "Update supported GitHub Enterprise Server versions" \
--body "$body"
echo "committed=true" >> $GITHUB_OUTPUT
fi
- name: Open PR
if: github.event_name != 'pull_request' && steps.prepare-commit.outputs.committed == 'true'
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
git push origin update-supported-enterprise-server-versions
body="This PR updates the list of supported GitHub Enterprise Server versions, either because a new "
body+="version is about to be feature frozen, or because an old release has been deprecated."
body+=$'\n\n'
body+="If an old release has been deprecated, please follow the instructions in CONTRIBUTING.md to "
body+="deprecate the corresponding version of CodeQL."
gh pr create --draft \
--title "Update supported GitHub Enterprise Server versions" \
--body "$body"

View File

@@ -119928,6 +119928,11 @@ var featureConfig = {
legacyApi: true,
minimumVersion: "2.15.0"
},
["csharp_new_cache_key" /* CsharpNewCacheKey */]: {
defaultValue: false,
envVar: "CODEQL_ACTION_CSHARP_NEW_CACHE_KEY",
minimumVersion: void 0
},
["diff_informed_queries" /* DiffInformedQueries */]: {
defaultValue: true,
envVar: "CODEQL_ACTION_DIFF_INFORMED_QUERIES",

252
lib/analyze-action.js generated
View File

@@ -88173,6 +88173,7 @@ var fs6 = __toESM(require("fs"));
var path6 = __toESM(require("path"));
// src/caching-utils.ts
var crypto = __toESM(require("crypto"));
var core6 = __toESM(require_core());
async function getTotalCacheSize(paths, logger, quiet = false) {
const sizes = await Promise.all(
@@ -88183,6 +88184,11 @@ async function getTotalCacheSize(paths, logger, quiet = false) {
function shouldStoreCache(kind) {
return kind === "full" /* Full */ || kind === "store" /* Store */;
}
var cacheKeyHashLength = 16;
function createCacheKeyHash(components) {
const componentsJson = JSON.stringify(components);
return crypto.createHash("sha256").update(componentsJson).digest("hex").substring(0, cacheKeyHashLength);
}
// src/config/db-config.ts
var jsonschema = __toESM(require_lib3());
@@ -88208,7 +88214,6 @@ var bundleVersion = "codeql-bundle-v2.23.3";
var cliVersion = "2.23.3";
// src/overlay-database-utils.ts
var crypto = __toESM(require("crypto"));
var fs3 = __toESM(require("fs"));
var path3 = __toESM(require("path"));
var actionsCache = __toESM(require_cache3());
@@ -88620,10 +88625,6 @@ async function getCacheRestoreKeyPrefix(config, codeQlVersion) {
const componentsHash = createCacheKeyHash(cacheKeyComponents);
return `${CACHE_PREFIX}-${CACHE_VERSION}-${componentsHash}-${languages}-${codeQlVersion}-`;
}
function createCacheKeyHash(components) {
const componentsJson = JSON.stringify(components);
return crypto.createHash("sha256").update(componentsJson).digest("hex").substring(0, 16);
}
// src/tools-features.ts
var semver3 = __toESM(require_semver2());
@@ -88657,6 +88658,11 @@ var featureConfig = {
legacyApi: true,
minimumVersion: "2.15.0"
},
["csharp_new_cache_key" /* CsharpNewCacheKey */]: {
defaultValue: false,
envVar: "CODEQL_ACTION_CSHARP_NEW_CACHE_KEY",
minimumVersion: void 0
},
["diff_informed_queries" /* DiffInformedQueries */]: {
defaultValue: true,
envVar: "CODEQL_ACTION_DIFF_INFORMED_QUERIES",
@@ -89182,13 +89188,14 @@ Error Response: ${JSON.stringify(error4.response, null, 2)}`
}
}
function getDiffRanges(fileDiff, logger) {
const filename = path5.join(getRequiredInput("checkout_path"), fileDiff.filename).replaceAll(path5.sep, "/");
if (fileDiff.patch === void 0) {
if (fileDiff.changes === 0) {
return [];
}
return [
{
path: fileDiff.filename,
path: filename,
startLine: 0,
endLine: 0
}
@@ -89212,7 +89219,7 @@ function getDiffRanges(fileDiff, logger) {
}
if (additionRangeStartLine !== void 0) {
diffRanges.push({
path: fileDiff.filename,
path: filename,
startLine: additionRangeStartLine,
endLine: currentLine - 1
});
@@ -91052,66 +91059,107 @@ var CODEQL_DEPENDENCY_CACHE_VERSION = 1;
function getJavaTempDependencyDir() {
return (0, import_path.join)(getTemporaryDirectory(), "codeql_java", "repository");
}
function getDefaultCacheConfig() {
return {
java: {
paths: [
// Maven
(0, import_path.join)(os3.homedir(), ".m2", "repository"),
// Gradle
(0, import_path.join)(os3.homedir(), ".gradle", "caches"),
// CodeQL Java build-mode: none
getJavaTempDependencyDir()
],
hash: [
// Maven
"**/pom.xml",
// Gradle
"**/*.gradle*",
"**/gradle-wrapper.properties",
"buildSrc/**/Versions.kt",
"buildSrc/**/Dependencies.kt",
"gradle/*.versions.toml",
"**/versions.properties"
]
},
csharp: {
paths: [(0, import_path.join)(os3.homedir(), ".nuget", "packages")],
hash: [
// NuGet
"**/packages.lock.json",
// Paket
"**/paket.lock"
]
},
go: {
paths: [(0, import_path.join)(os3.homedir(), "go", "pkg", "mod")],
hash: ["**/go.sum"]
}
};
function getJavaDependencyDirs() {
return [
// Maven
(0, import_path.join)(os3.homedir(), ".m2", "repository"),
// Gradle
(0, import_path.join)(os3.homedir(), ".gradle", "caches"),
// CodeQL Java build-mode: none
getJavaTempDependencyDir()
];
}
async function makePatternCheck(patterns) {
const globber = await makeGlobber(patterns);
if ((await globber.glob()).length === 0) {
return void 0;
}
return patterns;
}
var CSHARP_BASE_PATTERNS = [
// NuGet
"**/packages.lock.json",
// Paket
"**/paket.lock"
];
var CSHARP_EXTRA_PATTERNS = [
"**/*.csproj",
"**/packages.config",
"**/nuget.config"
];
async function getCsharpHashPatterns(codeql, features) {
const basePatterns = await internal.makePatternCheck(CSHARP_BASE_PATTERNS);
if (basePatterns !== void 0) {
return basePatterns;
}
if (await features.getValue("csharp_new_cache_key" /* CsharpNewCacheKey */, codeql)) {
return internal.makePatternCheck(CSHARP_EXTRA_PATTERNS);
}
return void 0;
}
var defaultCacheConfigs = {
java: {
getDependencyPaths: getJavaDependencyDirs,
getHashPatterns: async () => internal.makePatternCheck([
// Maven
"**/pom.xml",
// Gradle
"**/*.gradle*",
"**/gradle-wrapper.properties",
"buildSrc/**/Versions.kt",
"buildSrc/**/Dependencies.kt",
"gradle/*.versions.toml",
"**/versions.properties"
])
},
csharp: {
getDependencyPaths: () => [(0, import_path.join)(os3.homedir(), ".nuget", "packages")],
getHashPatterns: getCsharpHashPatterns
},
go: {
getDependencyPaths: () => [(0, import_path.join)(os3.homedir(), "go", "pkg", "mod")],
getHashPatterns: async () => internal.makePatternCheck(["**/go.sum"])
}
};
async function makeGlobber(patterns) {
return glob.create(patterns.join("\n"));
}
async function uploadDependencyCaches(config, logger, minimizeJavaJars) {
async function checkHashPatterns(codeql, features, language, cacheConfig, checkType, logger) {
const patterns = await cacheConfig.getHashPatterns(codeql, features);
if (patterns === void 0) {
logger.info(
`Skipping ${checkType} of dependency cache for ${language} as we cannot calculate a hash for the cache key.`
);
}
return patterns;
}
async function uploadDependencyCaches(codeql, features, config, logger) {
const status = [];
for (const language of config.languages) {
const cacheConfig = getDefaultCacheConfig()[language];
const cacheConfig = defaultCacheConfigs[language];
if (cacheConfig === void 0) {
logger.info(
`Skipping upload of dependency cache for ${language} as we have no caching configuration for it.`
);
continue;
}
const globber = await makeGlobber(cacheConfig.hash);
if ((await globber.glob()).length === 0) {
const patterns = await checkHashPatterns(
codeql,
features,
language,
cacheConfig,
"upload",
logger
);
if (patterns === void 0) {
status.push({ language, result: "no-hash" /* NoHash */ });
logger.info(
`Skipping upload of dependency cache for ${language} as we cannot calculate a hash for the cache key.`
);
continue;
}
const size = await getTotalCacheSize(cacheConfig.paths, logger, true);
const size = await getTotalCacheSize(
cacheConfig.getDependencyPaths(),
logger,
true
);
if (size === 0) {
status.push({ language, result: "empty" /* Empty */ });
logger.info(
@@ -91119,13 +91167,13 @@ async function uploadDependencyCaches(config, logger, minimizeJavaJars) {
);
continue;
}
const key = await cacheKey2(language, cacheConfig, minimizeJavaJars);
const key = await cacheKey2(codeql, features, language, patterns);
logger.info(
`Uploading cache of size ${size} for ${language} with key ${key}...`
);
try {
const start = performance.now();
await actionsCache3.saveCache(cacheConfig.paths, key);
await actionsCache3.saveCache(cacheConfig.getDependencyPaths(), key);
const upload_duration_ms = Math.round(performance.now() - start);
status.push({
language,
@@ -91147,22 +91195,50 @@ async function uploadDependencyCaches(config, logger, minimizeJavaJars) {
}
return status;
}
async function cacheKey2(language, cacheConfig, minimizeJavaJars = false) {
const hash2 = await glob.hashFiles(cacheConfig.hash.join("\n"));
return `${await cachePrefix2(language, minimizeJavaJars)}${hash2}`;
async function cacheKey2(codeql, features, language, patterns) {
const hash2 = await glob.hashFiles(patterns.join("\n"));
return `${await cachePrefix2(codeql, features, language)}${hash2}`;
}
async function cachePrefix2(language, minimizeJavaJars) {
async function getFeaturePrefix(codeql, features, language) {
const enabledFeatures = [];
const addFeatureIfEnabled = async (feature) => {
if (await features.getValue(feature, codeql)) {
enabledFeatures.push(feature);
}
};
if (language === "java" /* java */) {
const minimizeJavaJars = await features.getValue(
"java_minimize_dependency_jars" /* JavaMinimizeDependencyJars */,
codeql
);
if (minimizeJavaJars) {
return "minify-";
}
} else if (language === "csharp" /* csharp */) {
await addFeatureIfEnabled("csharp_new_cache_key" /* CsharpNewCacheKey */);
}
if (enabledFeatures.length > 0) {
return `${createCacheKeyHash(enabledFeatures)}-`;
}
return "";
}
async function cachePrefix2(codeql, features, language) {
const runnerOs = getRequiredEnvParam("RUNNER_OS");
const customPrefix = process.env["CODEQL_ACTION_DEPENDENCY_CACHE_PREFIX" /* DEPENDENCY_CACHING_PREFIX */];
let prefix = CODEQL_DEPENDENCY_CACHE_PREFIX;
if (customPrefix !== void 0 && customPrefix.length > 0) {
prefix = `${prefix}-${customPrefix}`;
}
if (language === "java" /* java */ && minimizeJavaJars) {
prefix = `minify-${prefix}`;
const featurePrefix = await getFeaturePrefix(codeql, features, language);
if (featurePrefix === "minify-") {
return `${featurePrefix}${prefix}-${CODEQL_DEPENDENCY_CACHE_VERSION}-${runnerOs}-${language}-`;
} else {
return `${prefix}-${featurePrefix}${CODEQL_DEPENDENCY_CACHE_VERSION}-${runnerOs}-${language}-`;
}
return `${prefix}-${CODEQL_DEPENDENCY_CACHE_VERSION}-${runnerOs}-${language}-`;
}
var internal = {
makePatternCheck
};
// src/diagnostics.ts
var import_fs = require("fs");
@@ -91323,25 +91399,6 @@ async function setupDiffInformedQueryRun(branches, logger) {
}
);
}
function diffRangeExtensionPackContents(ranges) {
const header = `
extensions:
- addsTo:
pack: codeql/util
extensible: restrictAlertsTo
checkPresence: false
data:
`;
let data = ranges.map((range) => {
const filename = path12.join(getRequiredInput("checkout_path"), range.path).replaceAll(path12.sep, "/");
return ` - [${dump(filename, { forceQuotes: true }).trim()}, ${range.startLine}, ${range.endLine}]
`;
}).join("");
if (!data) {
data = ' - ["", 0, 0]\n';
}
return header + data;
}
function writeDiffRangeDataExtensionPack(logger, ranges) {
if (ranges === void 0) {
return void 0;
@@ -91363,7 +91420,27 @@ dataExtensions:
- pr-diff-range.yml
`
);
const extensionContents = diffRangeExtensionPackContents(ranges);
const header = `
extensions:
- addsTo:
pack: codeql/util
extensible: restrictAlertsTo
checkPresence: false
data:
`;
let data = ranges.map(
(range) => (
// Using yaml.dump() with `forceQuotes: true` ensures that all special
// characters are escaped, and that the path is always rendered as a
// quoted string on a single line.
` - [${dump(range.path, { forceQuotes: true }).trim()}, ${range.startLine}, ${range.endLine}]
`
)
).join("");
if (!data) {
data = ' - ["", 0, 0]\n';
}
const extensionContents = header + data;
const extensionFilePath = path12.join(diffRangeDir, "pr-diff-range.yml");
fs12.writeFileSync(extensionFilePath, extensionContents);
logger.debug(
@@ -93646,6 +93723,7 @@ function filterAlertsByDiffRange(logger, sarif) {
if (!diffRanges?.length) {
return sarif;
}
const checkoutPath = getRequiredInput("checkout_path");
for (const run2 of sarif.runs) {
if (run2.results) {
run2.results = run2.results.filter((result) => {
@@ -93659,8 +93737,9 @@ function filterAlertsByDiffRange(logger, sarif) {
if (!locationUri || locationStartLine === void 0) {
return false;
}
const locationPath = path14.join(checkoutPath, locationUri).replaceAll(path14.sep, "/");
return diffRanges.some(
(range) => range.path === locationUri && (range.startLine <= locationStartLine && range.endLine >= locationStartLine || range.startLine === 0 && range.endLine === 0)
(range) => range.path === locationPath && (range.startLine <= locationStartLine && range.endLine >= locationStartLine || range.startLine === 0 && range.endLine === 0)
);
});
});
@@ -93972,14 +94051,11 @@ async function run() {
logger
);
if (shouldStoreCache(config.dependencyCachingEnabled)) {
const minimizeJavaJars = await features.getValue(
"java_minimize_dependency_jars" /* JavaMinimizeDependencyJars */,
codeql
);
dependencyCacheResults = await uploadDependencyCaches(
codeql,
features,
config,
logger,
minimizeJavaJars
logger
);
}
if (isInTestMode()) {

View File

@@ -83977,6 +83977,11 @@ var featureConfig = {
legacyApi: true,
minimumVersion: "2.15.0"
},
["csharp_new_cache_key" /* CsharpNewCacheKey */]: {
defaultValue: false,
envVar: "CODEQL_ACTION_CSHARP_NEW_CACHE_KEY",
minimumVersion: void 0
},
["diff_informed_queries" /* DiffInformedQueries */]: {
defaultValue: true,
envVar: "CODEQL_ACTION_DIFF_INFORMED_QUERIES",

View File

@@ -90452,8 +90452,8 @@ var require_primordials = __commonJS({
ArrayPrototypeIndexOf(self2, el) {
return self2.indexOf(el);
},
ArrayPrototypeJoin(self2, sep3) {
return self2.join(sep3);
ArrayPrototypeJoin(self2, sep4) {
return self2.join(sep4);
},
ArrayPrototypeMap(self2, fn) {
return self2.map(fn);
@@ -102340,7 +102340,7 @@ var require_commonjs16 = __commonJS({
*
* @internal
*/
constructor(cwd = process.cwd(), pathImpl, sep3, { nocase, childrenCacheSize = 16 * 1024, fs: fs17 = defaultFS } = {}) {
constructor(cwd = process.cwd(), pathImpl, sep4, { nocase, childrenCacheSize = 16 * 1024, fs: fs17 = defaultFS } = {}) {
this.#fs = fsFromOption(fs17);
if (cwd instanceof URL || cwd.startsWith("file://")) {
cwd = (0, node_url_1.fileURLToPath)(cwd);
@@ -102351,7 +102351,7 @@ var require_commonjs16 = __commonJS({
this.#resolveCache = new ResolveCache();
this.#resolvePosixCache = new ResolveCache();
this.#children = new ChildrenCache(childrenCacheSize);
const split = cwdPath.substring(this.rootPath.length).split(sep3);
const split = cwdPath.substring(this.rootPath.length).split(sep4);
if (split.length === 1 && !split[0]) {
split.pop();
}
@@ -123309,6 +123309,11 @@ var featureConfig = {
legacyApi: true,
minimumVersion: "2.15.0"
},
["csharp_new_cache_key" /* CsharpNewCacheKey */]: {
defaultValue: false,
envVar: "CODEQL_ACTION_CSHARP_NEW_CACHE_KEY",
minimumVersion: void 0
},
["diff_informed_queries" /* DiffInformedQueries */]: {
defaultValue: true,
envVar: "CODEQL_ACTION_DIFF_INFORMED_QUERIES",
@@ -127560,6 +127565,7 @@ function filterAlertsByDiffRange(logger, sarif) {
if (!diffRanges?.length) {
return sarif;
}
const checkoutPath = getRequiredInput("checkout_path");
for (const run2 of sarif.runs) {
if (run2.results) {
run2.results = run2.results.filter((result) => {
@@ -127573,8 +127579,9 @@ function filterAlertsByDiffRange(logger, sarif) {
if (!locationUri || locationStartLine === void 0) {
return false;
}
const locationPath = path13.join(checkoutPath, locationUri).replaceAll(path13.sep, "/");
return diffRanges.some(
(range) => range.path === locationUri && (range.startLine <= locationStartLine && range.endLine >= locationStartLine || range.startLine === 0 && range.endLine === 0)
(range) => range.path === locationPath && (range.startLine <= locationStartLine && range.endLine >= locationStartLine || range.startLine === 0 && range.endLine === 0)
);
});
});

204
lib/init-action.js generated
View File

@@ -85234,6 +85234,7 @@ function wrapApiConfigurationError(e) {
}
// src/caching-utils.ts
var crypto = __toESM(require("crypto"));
var core6 = __toESM(require_core());
async function getTotalCacheSize(paths, logger, quiet = false) {
const sizes = await Promise.all(
@@ -85266,6 +85267,11 @@ function getCachingKind(input) {
return "none" /* None */;
}
}
var cacheKeyHashLength = 16;
function createCacheKeyHash(components) {
const componentsJson = JSON.stringify(components);
return crypto.createHash("sha256").update(componentsJson).digest("hex").substring(0, cacheKeyHashLength);
}
function getDependencyCachingEnabled() {
const dependencyCaching = getOptionalInput("dependency-caching") || process.env["CODEQL_ACTION_DEPENDENCY_CACHING" /* DEPENDENCY_CACHING */];
if (dependencyCaching !== void 0) return getCachingKind(dependencyCaching);
@@ -85632,7 +85638,6 @@ var bundleVersion = "codeql-bundle-v2.23.3";
var cliVersion = "2.23.3";
// src/overlay-database-utils.ts
var crypto = __toESM(require("crypto"));
var fs3 = __toESM(require("fs"));
var path4 = __toESM(require("path"));
var actionsCache = __toESM(require_cache3());
@@ -86034,10 +86039,6 @@ async function getCacheRestoreKeyPrefix(config, codeQlVersion) {
const componentsHash = createCacheKeyHash(cacheKeyComponents);
return `${CACHE_PREFIX}-${CACHE_VERSION}-${componentsHash}-${languages}-${codeQlVersion}-`;
}
function createCacheKeyHash(components) {
const componentsJson = JSON.stringify(components);
return crypto.createHash("sha256").update(componentsJson).digest("hex").substring(0, 16);
}
// src/tools-features.ts
var semver3 = __toESM(require_semver2());
@@ -86071,6 +86072,11 @@ var featureConfig = {
legacyApi: true,
minimumVersion: "2.15.0"
},
["csharp_new_cache_key" /* CsharpNewCacheKey */]: {
defaultValue: false,
envVar: "CODEQL_ACTION_CSHARP_NEW_CACHE_KEY",
minimumVersion: void 0
},
["diff_informed_queries" /* DiffInformedQueries */]: {
defaultValue: true,
envVar: "CODEQL_ACTION_DIFF_INFORMED_QUERIES",
@@ -87240,68 +87246,105 @@ var CODEQL_DEPENDENCY_CACHE_VERSION = 1;
function getJavaTempDependencyDir() {
return (0, import_path.join)(getTemporaryDirectory(), "codeql_java", "repository");
}
function getDefaultCacheConfig() {
return {
java: {
paths: [
// Maven
(0, import_path.join)(os2.homedir(), ".m2", "repository"),
// Gradle
(0, import_path.join)(os2.homedir(), ".gradle", "caches"),
// CodeQL Java build-mode: none
getJavaTempDependencyDir()
],
hash: [
// Maven
"**/pom.xml",
// Gradle
"**/*.gradle*",
"**/gradle-wrapper.properties",
"buildSrc/**/Versions.kt",
"buildSrc/**/Dependencies.kt",
"gradle/*.versions.toml",
"**/versions.properties"
]
},
csharp: {
paths: [(0, import_path.join)(os2.homedir(), ".nuget", "packages")],
hash: [
// NuGet
"**/packages.lock.json",
// Paket
"**/paket.lock"
]
},
go: {
paths: [(0, import_path.join)(os2.homedir(), "go", "pkg", "mod")],
hash: ["**/go.sum"]
}
};
function getJavaDependencyDirs() {
return [
// Maven
(0, import_path.join)(os2.homedir(), ".m2", "repository"),
// Gradle
(0, import_path.join)(os2.homedir(), ".gradle", "caches"),
// CodeQL Java build-mode: none
getJavaTempDependencyDir()
];
}
async function makePatternCheck(patterns) {
const globber = await makeGlobber(patterns);
if ((await globber.glob()).length === 0) {
return void 0;
}
return patterns;
}
var CSHARP_BASE_PATTERNS = [
// NuGet
"**/packages.lock.json",
// Paket
"**/paket.lock"
];
var CSHARP_EXTRA_PATTERNS = [
"**/*.csproj",
"**/packages.config",
"**/nuget.config"
];
async function getCsharpHashPatterns(codeql, features) {
const basePatterns = await internal.makePatternCheck(CSHARP_BASE_PATTERNS);
if (basePatterns !== void 0) {
return basePatterns;
}
if (await features.getValue("csharp_new_cache_key" /* CsharpNewCacheKey */, codeql)) {
return internal.makePatternCheck(CSHARP_EXTRA_PATTERNS);
}
return void 0;
}
var defaultCacheConfigs = {
java: {
getDependencyPaths: getJavaDependencyDirs,
getHashPatterns: async () => internal.makePatternCheck([
// Maven
"**/pom.xml",
// Gradle
"**/*.gradle*",
"**/gradle-wrapper.properties",
"buildSrc/**/Versions.kt",
"buildSrc/**/Dependencies.kt",
"gradle/*.versions.toml",
"**/versions.properties"
])
},
csharp: {
getDependencyPaths: () => [(0, import_path.join)(os2.homedir(), ".nuget", "packages")],
getHashPatterns: getCsharpHashPatterns
},
go: {
getDependencyPaths: () => [(0, import_path.join)(os2.homedir(), "go", "pkg", "mod")],
getHashPatterns: async () => internal.makePatternCheck(["**/go.sum"])
}
};
async function makeGlobber(patterns) {
return glob.create(patterns.join("\n"));
}
async function downloadDependencyCaches(languages, logger, minimizeJavaJars) {
async function checkHashPatterns(codeql, features, language, cacheConfig, checkType, logger) {
const patterns = await cacheConfig.getHashPatterns(codeql, features);
if (patterns === void 0) {
logger.info(
`Skipping ${checkType} of dependency cache for ${language} as we cannot calculate a hash for the cache key.`
);
}
return patterns;
}
async function downloadDependencyCaches(codeql, features, languages, logger) {
const status = [];
for (const language of languages) {
const cacheConfig = getDefaultCacheConfig()[language];
const cacheConfig = defaultCacheConfigs[language];
if (cacheConfig === void 0) {
logger.info(
`Skipping download of dependency cache for ${language} as we have no caching configuration for it.`
);
continue;
}
const globber = await makeGlobber(cacheConfig.hash);
if ((await globber.glob()).length === 0) {
const patterns = await checkHashPatterns(
codeql,
features,
language,
cacheConfig,
"download",
logger
);
if (patterns === void 0) {
status.push({ language, hit_kind: "no-hash" /* NoHash */ });
logger.info(
`Skipping download of dependency cache for ${language} as we cannot calculate a hash for the cache key.`
);
continue;
}
const primaryKey = await cacheKey2(language, cacheConfig, minimizeJavaJars);
const primaryKey = await cacheKey2(codeql, features, language, patterns);
const restoreKeys = [
await cachePrefix2(language, minimizeJavaJars)
await cachePrefix2(codeql, features, language)
];
logger.info(
`Downloading cache for ${language} with key ${primaryKey} and restore keys ${restoreKeys.join(
@@ -87310,7 +87353,7 @@ async function downloadDependencyCaches(languages, logger, minimizeJavaJars) {
);
const start = performance.now();
const hitKey = await actionsCache3.restoreCache(
cacheConfig.paths,
cacheConfig.getDependencyPaths(),
primaryKey,
restoreKeys
);
@@ -87326,22 +87369,50 @@ async function downloadDependencyCaches(languages, logger, minimizeJavaJars) {
}
return status;
}
async function cacheKey2(language, cacheConfig, minimizeJavaJars = false) {
const hash = await glob.hashFiles(cacheConfig.hash.join("\n"));
return `${await cachePrefix2(language, minimizeJavaJars)}${hash}`;
async function cacheKey2(codeql, features, language, patterns) {
const hash = await glob.hashFiles(patterns.join("\n"));
return `${await cachePrefix2(codeql, features, language)}${hash}`;
}
async function cachePrefix2(language, minimizeJavaJars) {
async function getFeaturePrefix(codeql, features, language) {
const enabledFeatures = [];
const addFeatureIfEnabled = async (feature) => {
if (await features.getValue(feature, codeql)) {
enabledFeatures.push(feature);
}
};
if (language === "java" /* java */) {
const minimizeJavaJars = await features.getValue(
"java_minimize_dependency_jars" /* JavaMinimizeDependencyJars */,
codeql
);
if (minimizeJavaJars) {
return "minify-";
}
} else if (language === "csharp" /* csharp */) {
await addFeatureIfEnabled("csharp_new_cache_key" /* CsharpNewCacheKey */);
}
if (enabledFeatures.length > 0) {
return `${createCacheKeyHash(enabledFeatures)}-`;
}
return "";
}
async function cachePrefix2(codeql, features, language) {
const runnerOs = getRequiredEnvParam("RUNNER_OS");
const customPrefix = process.env["CODEQL_ACTION_DEPENDENCY_CACHE_PREFIX" /* DEPENDENCY_CACHING_PREFIX */];
let prefix = CODEQL_DEPENDENCY_CACHE_PREFIX;
if (customPrefix !== void 0 && customPrefix.length > 0) {
prefix = `${prefix}-${customPrefix}`;
}
if (language === "java" /* java */ && minimizeJavaJars) {
prefix = `minify-${prefix}`;
const featurePrefix = await getFeaturePrefix(codeql, features, language);
if (featurePrefix === "minify-") {
return `${featurePrefix}${prefix}-${CODEQL_DEPENDENCY_CACHE_VERSION}-${runnerOs}-${language}-`;
} else {
return `${prefix}-${featurePrefix}${CODEQL_DEPENDENCY_CACHE_VERSION}-${runnerOs}-${language}-`;
}
return `${prefix}-${CODEQL_DEPENDENCY_CACHE_VERSION}-${runnerOs}-${language}-`;
}
var internal = {
makePatternCheck
};
// src/diagnostics.ts
var import_fs = require("fs");
@@ -89705,7 +89776,7 @@ async function getWorkflowAbsolutePath(logger) {
async function checkWorkflow(logger, codeql) {
if (!isDynamicWorkflow() && process.env["CODEQL_ACTION_SKIP_WORKFLOW_VALIDATION" /* SKIP_WORKFLOW_VALIDATION */] !== "true") {
core12.startGroup("Validating workflow");
const validateWorkflowResult = await internal.validateWorkflow(
const validateWorkflowResult = await internal2.validateWorkflow(
codeql,
logger
);
@@ -89719,7 +89790,7 @@ async function checkWorkflow(logger, codeql) {
core12.endGroup();
}
}
var internal = {
var internal2 = {
validateWorkflow
};
@@ -90068,15 +90139,12 @@ exec ${goBinaryPath} "$@"`
core13.exportVariable(envVar, "false");
}
}
const minimizeJavaJars = await features.getValue(
"java_minimize_dependency_jars" /* JavaMinimizeDependencyJars */,
codeql
);
if (shouldRestoreCache(config.dependencyCachingEnabled)) {
dependencyCachingResults = await downloadDependencyCaches(
codeql,
features,
config.languages,
logger,
minimizeJavaJars
logger
);
}
if (await codeQlVersionAtLeast(codeql, "2.17.1")) {
@@ -90114,7 +90182,7 @@ exec ${goBinaryPath} "$@"`
logger.debug(
`${"CODEQL_EXTRACTOR_JAVA_OPTION_MINIMIZE_DEPENDENCY_JARS" /* JAVA_EXTRACTOR_MINIMIZE_DEPENDENCY_JARS */} is already set to '${process.env["CODEQL_EXTRACTOR_JAVA_OPTION_MINIMIZE_DEPENDENCY_JARS" /* JAVA_EXTRACTOR_MINIMIZE_DEPENDENCY_JARS */]}', so the Action will not override it.`
);
} else if (minimizeJavaJars && config.dependencyCachingEnabled && config.buildMode === "none" /* None */ && config.languages.includes("java" /* java */)) {
} else if (await features.getValue("java_minimize_dependency_jars" /* JavaMinimizeDependencyJars */, codeql) && config.dependencyCachingEnabled && config.buildMode === "none" /* None */ && config.languages.includes("java" /* java */)) {
core13.exportVariable(
"CODEQL_EXTRACTOR_JAVA_OPTION_MINIMIZE_DEPENDENCY_JARS" /* JAVA_EXTRACTOR_MINIMIZE_DEPENDENCY_JARS */,
"true"

View File

@@ -83968,6 +83968,11 @@ var featureConfig = {
legacyApi: true,
minimumVersion: "2.15.0"
},
["csharp_new_cache_key" /* CsharpNewCacheKey */]: {
defaultValue: false,
envVar: "CODEQL_ACTION_CSHARP_NEW_CACHE_KEY",
minimumVersion: void 0
},
["diff_informed_queries" /* DiffInformedQueries */]: {
defaultValue: true,
envVar: "CODEQL_ACTION_DIFF_INFORMED_QUERIES",

View File

@@ -83596,14 +83596,17 @@ var fs3 = __toESM(require("fs"));
var path3 = __toESM(require("path"));
var actionsCache = __toESM(require_cache3());
// src/git-utils.ts
// src/caching-utils.ts
var core6 = __toESM(require_core());
// src/git-utils.ts
var core7 = __toESM(require_core());
var toolrunner2 = __toESM(require_toolrunner());
var io3 = __toESM(require_io2());
var runGitCommand = async function(workingDirectory, args, customErrorMessage) {
let stdout = "";
let stderr = "";
core6.debug(`Running git command: git ${args.join(" ")}`);
core7.debug(`Running git command: git ${args.join(" ")}`);
try {
await new toolrunner2.ToolRunner(await io3.which("git", true), args, {
silent: true,
@@ -83623,7 +83626,7 @@ var runGitCommand = async function(workingDirectory, args, customErrorMessage) {
if (stderr.includes("not a git repository")) {
reason = "The checkout path provided to the action does not appear to be a git repository.";
}
core6.info(`git call failed. ${customErrorMessage} Error: ${reason}`);
core7.info(`git call failed. ${customErrorMessage} Error: ${reason}`);
throw error4;
}
};
@@ -83734,7 +83737,7 @@ async function getRef() {
) !== head;
if (hasChangedRef) {
const newRef = ref.replace(pull_ref_regex, "refs/pull/$1/head");
core6.debug(
core7.debug(
`No longer on merge commit, rewriting ref from ${ref} to ${newRef}.`
);
return newRef;
@@ -83760,16 +83763,16 @@ async function isAnalyzingDefaultBranch() {
}
// src/logging.ts
var core7 = __toESM(require_core());
var core8 = __toESM(require_core());
function getActionsLogger() {
return {
debug: core7.debug,
info: core7.info,
warning: core7.warning,
error: core7.error,
isDebug: core7.isDebug,
startGroup: core7.startGroup,
endGroup: core7.endGroup
debug: core8.debug,
info: core8.info,
warning: core8.warning,
error: core8.error,
isDebug: core8.isDebug,
startGroup: core8.startGroup,
endGroup: core8.endGroup
};
}
function formatDuration(durationMs) {
@@ -83877,6 +83880,11 @@ var featureConfig = {
legacyApi: true,
minimumVersion: "2.15.0"
},
["csharp_new_cache_key" /* CsharpNewCacheKey */]: {
defaultValue: false,
envVar: "CODEQL_ACTION_CSHARP_NEW_CACHE_KEY",
minimumVersion: void 0
},
["diff_informed_queries" /* DiffInformedQueries */]: {
defaultValue: true,
envVar: "CODEQL_ACTION_DIFF_INFORMED_QUERIES",
@@ -84562,9 +84570,6 @@ var AnalysisKind = /* @__PURE__ */ ((AnalysisKind2) => {
})(AnalysisKind || {});
var supportedAnalysisKinds = new Set(Object.values(AnalysisKind));
// src/caching-utils.ts
var core8 = __toESM(require_core());
// src/config/db-config.ts
var jsonschema = __toESM(require_lib4());
var semver4 = __toESM(require_semver2());

View File

@@ -119334,6 +119334,11 @@ var featureConfig = {
legacyApi: true,
minimumVersion: "2.15.0"
},
["csharp_new_cache_key" /* CsharpNewCacheKey */]: {
defaultValue: false,
envVar: "CODEQL_ACTION_CSHARP_NEW_CACHE_KEY",
minimumVersion: void 0
},
["diff_informed_queries" /* DiffInformedQueries */]: {
defaultValue: true,
envVar: "CODEQL_ACTION_DIFF_INFORMED_QUERIES",

View File

@@ -99996,6 +99996,11 @@ var featureConfig = {
legacyApi: true,
minimumVersion: "2.15.0"
},
["csharp_new_cache_key" /* CsharpNewCacheKey */]: {
defaultValue: false,
envVar: "CODEQL_ACTION_CSHARP_NEW_CACHE_KEY",
minimumVersion: void 0
},
["diff_informed_queries" /* DiffInformedQueries */]: {
defaultValue: true,
envVar: "CODEQL_ACTION_DIFF_INFORMED_QUERIES",

11
lib/upload-lib.js generated
View File

@@ -83220,7 +83220,6 @@ var upload_lib_exports = {};
__export(upload_lib_exports, {
InvalidSarifUploadError: () => InvalidSarifUploadError,
buildPayload: () => buildPayload,
filterAlertsByDiffRange: () => filterAlertsByDiffRange,
findSarifFilesInDir: () => findSarifFilesInDir,
getGroupedSarifFilePaths: () => getGroupedSarifFilePaths,
getSarifFilePaths: () => getSarifFilePaths,
@@ -87035,6 +87034,11 @@ var featureConfig = {
legacyApi: true,
minimumVersion: "2.15.0"
},
["csharp_new_cache_key" /* CsharpNewCacheKey */]: {
defaultValue: false,
envVar: "CODEQL_ACTION_CSHARP_NEW_CACHE_KEY",
minimumVersion: void 0
},
["diff_informed_queries" /* DiffInformedQueries */]: {
defaultValue: true,
envVar: "CODEQL_ACTION_DIFF_INFORMED_QUERIES",
@@ -90616,6 +90620,7 @@ function filterAlertsByDiffRange(logger, sarif) {
if (!diffRanges?.length) {
return sarif;
}
const checkoutPath = getRequiredInput("checkout_path");
for (const run of sarif.runs) {
if (run.results) {
run.results = run.results.filter((result) => {
@@ -90629,8 +90634,9 @@ function filterAlertsByDiffRange(logger, sarif) {
if (!locationUri || locationStartLine === void 0) {
return false;
}
const locationPath = path10.join(checkoutPath, locationUri).replaceAll(path10.sep, "/");
return diffRanges.some(
(range) => range.path === locationUri && (range.startLine <= locationStartLine && range.endLine >= locationStartLine || range.startLine === 0 && range.endLine === 0)
(range) => range.path === locationPath && (range.startLine <= locationStartLine && range.endLine >= locationStartLine || range.startLine === 0 && range.endLine === 0)
);
});
});
@@ -90642,7 +90648,6 @@ function filterAlertsByDiffRange(logger, sarif) {
0 && (module.exports = {
InvalidSarifUploadError,
buildPayload,
filterAlertsByDiffRange,
findSarifFilesInDir,
getGroupedSarifFilePaths,
getSarifFilePaths,

View File

@@ -119500,6 +119500,11 @@ var featureConfig = {
legacyApi: true,
minimumVersion: "2.15.0"
},
["csharp_new_cache_key" /* CsharpNewCacheKey */]: {
defaultValue: false,
envVar: "CODEQL_ACTION_CSHARP_NEW_CACHE_KEY",
minimumVersion: void 0
},
["diff_informed_queries" /* DiffInformedQueries */]: {
defaultValue: true,
envVar: "CODEQL_ACTION_DIFF_INFORMED_QUERIES",

View File

@@ -86512,14 +86512,17 @@ var fs3 = __toESM(require("fs"));
var path3 = __toESM(require("path"));
var actionsCache = __toESM(require_cache3());
// src/git-utils.ts
// src/caching-utils.ts
var core6 = __toESM(require_core());
// src/git-utils.ts
var core7 = __toESM(require_core());
var toolrunner2 = __toESM(require_toolrunner());
var io3 = __toESM(require_io2());
var runGitCommand = async function(workingDirectory, args, customErrorMessage) {
let stdout = "";
let stderr = "";
core6.debug(`Running git command: git ${args.join(" ")}`);
core7.debug(`Running git command: git ${args.join(" ")}`);
try {
await new toolrunner2.ToolRunner(await io3.which("git", true), args, {
silent: true,
@@ -86539,7 +86542,7 @@ var runGitCommand = async function(workingDirectory, args, customErrorMessage) {
if (stderr.includes("not a git repository")) {
reason = "The checkout path provided to the action does not appear to be a git repository.";
}
core6.info(`git call failed. ${customErrorMessage} Error: ${reason}`);
core7.info(`git call failed. ${customErrorMessage} Error: ${reason}`);
throw error4;
}
};
@@ -86684,7 +86687,7 @@ async function getRef() {
) !== head;
if (hasChangedRef) {
const newRef = ref.replace(pull_ref_regex, "refs/pull/$1/head");
core6.debug(
core7.debug(
`No longer on merge commit, rewriting ref from ${ref} to ${newRef}.`
);
return newRef;
@@ -86710,16 +86713,16 @@ async function isAnalyzingDefaultBranch() {
}
// src/logging.ts
var core7 = __toESM(require_core());
var core8 = __toESM(require_core());
function getActionsLogger() {
return {
debug: core7.debug,
info: core7.info,
warning: core7.warning,
error: core7.error,
isDebug: core7.isDebug,
startGroup: core7.startGroup,
endGroup: core7.endGroup
debug: core8.debug,
info: core8.info,
warning: core8.warning,
error: core8.error,
isDebug: core8.isDebug,
startGroup: core8.startGroup,
endGroup: core8.endGroup
};
}
function formatDuration(durationMs) {
@@ -86827,6 +86830,11 @@ var featureConfig = {
legacyApi: true,
minimumVersion: "2.15.0"
},
["csharp_new_cache_key" /* CsharpNewCacheKey */]: {
defaultValue: false,
envVar: "CODEQL_ACTION_CSHARP_NEW_CACHE_KEY",
minimumVersion: void 0
},
["diff_informed_queries" /* DiffInformedQueries */]: {
defaultValue: true,
envVar: "CODEQL_ACTION_DIFF_INFORMED_QUERIES",
@@ -87262,9 +87270,6 @@ var core9 = __toESM(require_core());
var fs6 = __toESM(require("fs"));
var path6 = __toESM(require("path"));
// src/caching-utils.ts
var core8 = __toESM(require_core());
// src/config/db-config.ts
var jsonschema = __toESM(require_lib4());
var semver4 = __toESM(require_semver2());
@@ -91085,6 +91090,7 @@ function filterAlertsByDiffRange(logger, sarif) {
if (!diffRanges?.length) {
return sarif;
}
const checkoutPath = getRequiredInput("checkout_path");
for (const run2 of sarif.runs) {
if (run2.results) {
run2.results = run2.results.filter((result) => {
@@ -91098,8 +91104,9 @@ function filterAlertsByDiffRange(logger, sarif) {
if (!locationUri || locationStartLine === void 0) {
return false;
}
const locationPath = path11.join(checkoutPath, locationUri).replaceAll(path11.sep, "/");
return diffRanges.some(
(range) => range.path === locationUri && (range.startLine <= locationStartLine && range.endLine >= locationStartLine || range.startLine === 0 && range.endLine === 0)
(range) => range.path === locationPath && (range.startLine <= locationStartLine && range.endLine >= locationStartLine || range.startLine === 0 && range.endLine === 0)
);
});
});

View File

@@ -438,14 +438,11 @@ async function run() {
// Store dependency cache(s) if dependency caching is enabled.
if (shouldStoreCache(config.dependencyCachingEnabled)) {
const minimizeJavaJars = await features.getValue(
Feature.JavaMinimizeDependencyJars,
codeql,
);
dependencyCacheResults = await uploadDependencyCaches(
codeql,
features,
config,
logger,
minimizeJavaJars,
);
}

View File

@@ -4,14 +4,12 @@ import * as path from "path";
import test from "ava";
import * as sinon from "sinon";
import * as actionsUtil from "./actions-util";
import { CodeQuality, CodeScanning } from "./analyses";
import {
runQueries,
defaultSuites,
resolveQuerySuiteAlias,
addSarifExtension,
diffRangeExtensionPackContents,
} from "./analyze";
import { createStubCodeQL } from "./codeql";
import { Feature } from "./feature-flags";
@@ -160,23 +158,3 @@ test("addSarifExtension", (t) => {
);
}
});
test("diffRangeExtensionPackContents", (t) => {
sinon
.stub(actionsUtil, "getRequiredInput")
.withArgs("checkout_path")
.returns("/checkout/path");
const output = diffRangeExtensionPackContents([
{
path: "main.js",
startLine: 10,
endLine: 20,
},
]);
const expected = fs.readFileSync(
`${__dirname}/../src/testdata/pr-diff-range.yml`,
"utf8",
);
t.deepEqual(output, expected);
});

View File

@@ -5,11 +5,7 @@ import { performance } from "perf_hooks";
import * as io from "@actions/io";
import * as yaml from "js-yaml";
import {
getTemporaryDirectory,
getRequiredInput,
PullRequestBranches,
} from "./actions-util";
import { getTemporaryDirectory, PullRequestBranches } from "./actions-util";
import * as analyses from "./analyses";
import { setupCppAutobuild } from "./autobuild";
import { type CodeQL } from "./codeql";
@@ -248,45 +244,6 @@ export async function setupDiffInformedQueryRun(
);
}
export function diffRangeExtensionPackContents(
ranges: DiffThunkRange[],
): string {
const header = `
extensions:
- addsTo:
pack: codeql/util
extensible: restrictAlertsTo
checkPresence: false
data:
`;
let data = ranges
.map((range) => {
// Diff-informed queries expect the file path to be absolute. CodeQL always
// uses forward slashes as the path separator, so on Windows we need to
// replace any backslashes with forward slashes.
const filename = path
.join(getRequiredInput("checkout_path"), range.path)
.replaceAll(path.sep, "/");
// Using yaml.dump() with `forceQuotes: true` ensures that all special
// characters are escaped, and that the path is always rendered as a
// quoted string on a single line.
return (
` - [${yaml.dump(filename, { forceQuotes: true }).trim()}, ` +
`${range.startLine}, ${range.endLine}]\n`
);
})
.join("");
if (!data) {
// Ensure that the data extension is not empty, so that a pull request with
// no edited lines would exclude (instead of accepting) all alerts.
data = ' - ["", 0, 0]\n';
}
return header + data;
}
/**
* Create an extension pack in the temporary directory that contains the file
* line ranges that were added or modified in the pull request.
@@ -335,7 +292,32 @@ dataExtensions:
`,
);
const extensionContents = diffRangeExtensionPackContents(ranges);
const header = `
extensions:
- addsTo:
pack: codeql/util
extensible: restrictAlertsTo
checkPresence: false
data:
`;
let data = ranges
.map(
(range) =>
// Using yaml.dump() with `forceQuotes: true` ensures that all special
// characters are escaped, and that the path is always rendered as a
// quoted string on a single line.
` - [${yaml.dump(range.path, { forceQuotes: true }).trim()}, ` +
`${range.startLine}, ${range.endLine}]\n`,
)
.join("");
if (!data) {
// Ensure that the data extension is not empty, so that a pull request with
// no edited lines would exclude (instead of accepting) all alerts.
data = ' - ["", 0, 0]\n';
}
const extensionContents = header + data;
const extensionFilePath = path.join(diffRangeDir, "pr-diff-range.yml");
fs.writeFileSync(extensionFilePath, extensionContents);
logger.debug(

View File

@@ -1,3 +1,5 @@
import * as crypto from "crypto";
import * as core from "@actions/core";
import { getOptionalInput, isDefaultSetup } from "./actions-util";
@@ -71,6 +73,33 @@ export function getCachingKind(input: string | undefined): CachingKind {
}
}
// The length to which `createCacheKeyHash` truncates hash strings.
export const cacheKeyHashLength = 16;
/**
* Creates a SHA-256 hash of the cache key components to ensure uniqueness
* while keeping the cache key length manageable.
*
* @param components Object containing all components that should influence cache key uniqueness
* @returns A short SHA-256 hash (first 16 characters) of the components
*/
export function createCacheKeyHash(components: Record<string, any>): string {
// From https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/JSON/stringify
//
// "Properties are visited using the same algorithm as Object.keys(), which
// has a well-defined order and is stable across implementations. For example,
// JSON.stringify on the same object will always produce the same string, and
// JSON.parse(JSON.stringify(obj)) would produce an object with the same key
// ordering as the original (assuming the object is completely
// JSON-serializable)."
const componentsJson = JSON.stringify(components);
return crypto
.createHash("sha256")
.update(componentsJson)
.digest("hex")
.substring(0, cacheKeyHashLength);
}
/** Determines whether dependency caching is enabled. */
export function getDependencyCachingEnabled(): CachingKind {
// If the workflow specified something always respect that

View File

@@ -0,0 +1,389 @@
import * as fs from "fs";
import path from "path";
import * as actionsCache from "@actions/cache";
import * as glob from "@actions/glob";
import test from "ava";
import * as sinon from "sinon";
import { cacheKeyHashLength } from "./caching-utils";
import { createStubCodeQL } from "./codeql";
import {
CacheConfig,
checkHashPatterns,
getCsharpHashPatterns,
getFeaturePrefix,
makePatternCheck,
internal,
CSHARP_BASE_PATTERNS,
CSHARP_EXTRA_PATTERNS,
downloadDependencyCaches,
CacheHitKind,
cacheKey,
} from "./dependency-caching";
import { Feature } from "./feature-flags";
import { KnownLanguage } from "./languages";
import {
setupTests,
createFeatures,
getRecordingLogger,
checkExpectedLogMessages,
LoggedMessage,
} from "./testing-utils";
import { withTmpDir } from "./util";
setupTests(test);
function makeAbsolutePatterns(tmpDir: string, patterns: string[]): string[] {
return patterns.map((pattern) => path.join(tmpDir, pattern));
}
test("makePatternCheck - returns undefined if no patterns match", async (t) => {
await withTmpDir(async (tmpDir) => {
fs.writeFileSync(path.join(tmpDir, "test.java"), "");
const result = await makePatternCheck(
makeAbsolutePatterns(tmpDir, ["**/*.cs"]),
);
t.is(result, undefined);
});
});
test("makePatternCheck - returns all patterns if any pattern matches", async (t) => {
await withTmpDir(async (tmpDir) => {
fs.writeFileSync(path.join(tmpDir, "test.java"), "");
const patterns = makeAbsolutePatterns(tmpDir, ["**/*.cs", "**/*.java"]);
const result = await makePatternCheck(patterns);
t.deepEqual(result, patterns);
});
});
test("getCsharpHashPatterns - returns base patterns if any pattern matches", async (t) => {
const codeql = createStubCodeQL({});
const features = createFeatures([]);
const makePatternCheckStub = sinon.stub(internal, "makePatternCheck");
makePatternCheckStub
.withArgs(CSHARP_BASE_PATTERNS)
.resolves(CSHARP_BASE_PATTERNS);
makePatternCheckStub.withArgs(CSHARP_EXTRA_PATTERNS).rejects();
await t.notThrowsAsync(async () => {
const result = await getCsharpHashPatterns(codeql, features);
t.deepEqual(result, CSHARP_BASE_PATTERNS);
});
});
test("getCsharpHashPatterns - returns base patterns if any base pattern matches and CsharpNewCacheKey is enabled", async (t) => {
const codeql = createStubCodeQL({});
const features = createFeatures([Feature.CsharpNewCacheKey]);
const makePatternCheckStub = sinon.stub(internal, "makePatternCheck");
makePatternCheckStub
.withArgs(CSHARP_BASE_PATTERNS)
.resolves(CSHARP_BASE_PATTERNS);
makePatternCheckStub
.withArgs(CSHARP_EXTRA_PATTERNS)
.resolves(CSHARP_EXTRA_PATTERNS);
await t.notThrowsAsync(async () => {
const result = await getCsharpHashPatterns(codeql, features);
t.deepEqual(result, CSHARP_BASE_PATTERNS);
});
});
test("getCsharpHashPatterns - returns extra patterns if any extra pattern matches and CsharpNewCacheKey is enabled", async (t) => {
const codeql = createStubCodeQL({});
const features = createFeatures([Feature.CsharpNewCacheKey]);
const makePatternCheckStub = sinon.stub(internal, "makePatternCheck");
makePatternCheckStub.withArgs(CSHARP_BASE_PATTERNS).resolves(undefined);
makePatternCheckStub
.withArgs(CSHARP_EXTRA_PATTERNS)
.resolves(CSHARP_EXTRA_PATTERNS);
await t.notThrowsAsync(async () => {
const result = await getCsharpHashPatterns(codeql, features);
t.deepEqual(result, CSHARP_EXTRA_PATTERNS);
});
});
test("getCsharpHashPatterns - returns undefined if neither base nor extra patterns match", async (t) => {
const codeql = createStubCodeQL({});
const features = createFeatures([Feature.CsharpNewCacheKey]);
const makePatternCheckStub = sinon.stub(internal, "makePatternCheck");
makePatternCheckStub.withArgs(CSHARP_BASE_PATTERNS).resolves(undefined);
makePatternCheckStub.withArgs(CSHARP_EXTRA_PATTERNS).resolves(undefined);
await t.notThrowsAsync(async () => {
const result = await getCsharpHashPatterns(codeql, features);
t.deepEqual(result, undefined);
});
});
test("checkHashPatterns - logs when no patterns match", async (t) => {
const codeql = createStubCodeQL({});
const features = createFeatures([]);
const messages: LoggedMessage[] = [];
const config: CacheConfig = {
getDependencyPaths: () => [],
getHashPatterns: async () => undefined,
};
const result = await checkHashPatterns(
codeql,
features,
KnownLanguage.csharp,
config,
"download",
getRecordingLogger(messages),
);
t.is(result, undefined);
checkExpectedLogMessages(t, messages, [
"Skipping download of dependency cache",
]);
});
test("checkHashPatterns - returns patterns when patterns match", async (t) => {
await withTmpDir(async (tmpDir) => {
const codeql = createStubCodeQL({});
const features = createFeatures([]);
const messages: LoggedMessage[] = [];
const patterns = makeAbsolutePatterns(tmpDir, ["**/*.cs", "**/*.java"]);
fs.writeFileSync(path.join(tmpDir, "test.java"), "");
const config: CacheConfig = {
getDependencyPaths: () => [],
getHashPatterns: async () => makePatternCheck(patterns),
};
const result = await checkHashPatterns(
codeql,
features,
KnownLanguage.csharp,
config,
"upload",
getRecordingLogger(messages),
);
t.deepEqual(result, patterns);
t.deepEqual(messages, []);
});
});
type RestoreCacheFunc = (
paths: string[],
primaryKey: string,
restoreKeys: string[] | undefined,
) => Promise<string | undefined>;
/**
* Constructs a function that `actionsCache.restoreCache` can be stubbed with.
*
* @param mockCacheKeys The keys of caches that we want to exist in the Actions cache.
*
* @returns Returns a function that `actionsCache.restoreCache` can be stubbed with.
*/
function makeMockCacheCheck(mockCacheKeys: string[]): RestoreCacheFunc {
return async (
_paths: string[],
primaryKey: string,
restoreKeys: string[] | undefined,
) => {
// The behaviour here mirrors what the real `restoreCache` would do:
// - Starting with the primary restore key, check all caches for a match:
// even for the primary restore key, this only has to be a prefix match.
// - If the primary restore key doesn't prefix-match any cache, then proceed
// in the same way for each restore key in turn.
for (const restoreKey of [primaryKey, ...(restoreKeys || [])]) {
for (const mockCacheKey of mockCacheKeys) {
if (mockCacheKey.startsWith(restoreKey)) {
return mockCacheKey;
}
}
}
// Only if no restore key matches any cache key prefix, there is no matching
// cache and we return `undefined`.
return undefined;
};
}
test("downloadDependencyCaches - does not restore caches with feature keys if no features are enabled", async (t) => {
process.env["RUNNER_OS"] = "Linux";
const codeql = createStubCodeQL({});
const messages: LoggedMessage[] = [];
const logger = getRecordingLogger(messages);
sinon.stub(glob, "hashFiles").resolves("abcdef");
const keyWithFeature = await cacheKey(
codeql,
createFeatures([Feature.CsharpNewCacheKey]),
KnownLanguage.csharp,
// Patterns don't matter here because we have stubbed `hashFiles` to always return a specific hash above.
[],
);
const restoreCacheStub = sinon
.stub(actionsCache, "restoreCache")
.callsFake(makeMockCacheCheck([keyWithFeature]));
const makePatternCheckStub = sinon.stub(internal, "makePatternCheck");
makePatternCheckStub
.withArgs(CSHARP_BASE_PATTERNS)
.resolves(CSHARP_BASE_PATTERNS);
makePatternCheckStub.withArgs(CSHARP_EXTRA_PATTERNS).resolves(undefined);
const results = await downloadDependencyCaches(
codeql,
createFeatures([]),
[KnownLanguage.csharp],
logger,
);
t.is(results.length, 1);
t.is(results[0].language, KnownLanguage.csharp);
t.is(results[0].hit_kind, CacheHitKind.Miss);
t.assert(restoreCacheStub.calledOnce);
});
test("downloadDependencyCaches - restores caches with feature keys if features are enabled", async (t) => {
process.env["RUNNER_OS"] = "Linux";
const codeql = createStubCodeQL({});
const messages: LoggedMessage[] = [];
const logger = getRecordingLogger(messages);
const features = createFeatures([Feature.CsharpNewCacheKey]);
sinon.stub(glob, "hashFiles").resolves("abcdef");
const keyWithFeature = await cacheKey(
codeql,
features,
KnownLanguage.csharp,
// Patterns don't matter here because we have stubbed `hashFiles` to always return a specific hash above.
[],
);
const restoreCacheStub = sinon
.stub(actionsCache, "restoreCache")
.callsFake(makeMockCacheCheck([keyWithFeature]));
const makePatternCheckStub = sinon.stub(internal, "makePatternCheck");
makePatternCheckStub
.withArgs(CSHARP_BASE_PATTERNS)
.resolves(CSHARP_BASE_PATTERNS);
makePatternCheckStub.withArgs(CSHARP_EXTRA_PATTERNS).resolves(undefined);
const results = await downloadDependencyCaches(
codeql,
features,
[KnownLanguage.csharp],
logger,
);
t.is(results.length, 1);
t.is(results[0].language, KnownLanguage.csharp);
t.is(results[0].hit_kind, CacheHitKind.Exact);
t.assert(restoreCacheStub.calledOnce);
});
test("downloadDependencyCaches - restores caches with feature keys if features are enabled for partial matches", async (t) => {
process.env["RUNNER_OS"] = "Linux";
const codeql = createStubCodeQL({});
const messages: LoggedMessage[] = [];
const logger = getRecordingLogger(messages);
const features = createFeatures([Feature.CsharpNewCacheKey]);
const hashFilesStub = sinon.stub(glob, "hashFiles");
hashFilesStub.onFirstCall().resolves("abcdef");
hashFilesStub.onSecondCall().resolves("123456");
const keyWithFeature = await cacheKey(
codeql,
features,
KnownLanguage.csharp,
// Patterns don't matter here because we have stubbed `hashFiles` to always return a specific hash above.
[],
);
const restoreCacheStub = sinon
.stub(actionsCache, "restoreCache")
.callsFake(makeMockCacheCheck([keyWithFeature]));
const makePatternCheckStub = sinon.stub(internal, "makePatternCheck");
makePatternCheckStub
.withArgs(CSHARP_BASE_PATTERNS)
.resolves(CSHARP_BASE_PATTERNS);
makePatternCheckStub.withArgs(CSHARP_EXTRA_PATTERNS).resolves(undefined);
const results = await downloadDependencyCaches(
codeql,
features,
[KnownLanguage.csharp],
logger,
);
t.is(results.length, 1);
t.is(results[0].language, KnownLanguage.csharp);
t.is(results[0].hit_kind, CacheHitKind.Partial);
t.assert(restoreCacheStub.calledOnce);
});
test("getFeaturePrefix - returns empty string if no features are enabled", async (t) => {
const codeql = createStubCodeQL({});
const features = createFeatures([]);
for (const knownLanguage of Object.values(KnownLanguage)) {
const result = await getFeaturePrefix(codeql, features, knownLanguage);
t.deepEqual(result, "", `Expected no feature prefix for ${knownLanguage}`);
}
});
test("getFeaturePrefix - Java - returns 'minify-' if JavaMinimizeDependencyJars is enabled", async (t) => {
const codeql = createStubCodeQL({});
const features = createFeatures([Feature.JavaMinimizeDependencyJars]);
const result = await getFeaturePrefix(codeql, features, KnownLanguage.java);
t.deepEqual(result, "minify-");
});
test("getFeaturePrefix - non-Java - returns '' if JavaMinimizeDependencyJars is enabled", async (t) => {
const codeql = createStubCodeQL({});
const features = createFeatures([Feature.JavaMinimizeDependencyJars]);
for (const knownLanguage of Object.values(KnownLanguage)) {
// Skip Java since we expect a result for it, which is tested in the previous test.
if (knownLanguage === KnownLanguage.java) {
continue;
}
const result = await getFeaturePrefix(codeql, features, knownLanguage);
t.deepEqual(result, "", `Expected no feature prefix for ${knownLanguage}`);
}
});
test("getFeaturePrefix - C# - returns prefix if CsharpNewCacheKey is enabled", async (t) => {
const codeql = createStubCodeQL({});
const features = createFeatures([Feature.CsharpNewCacheKey]);
const result = await getFeaturePrefix(codeql, features, KnownLanguage.csharp);
t.notDeepEqual(result, "");
t.assert(result.endsWith("-"));
// Check the length of the prefix, which should correspond to `cacheKeyHashLength` + 1 for the trailing `-`.
t.is(result.length, cacheKeyHashLength + 1);
});
test("getFeaturePrefix - non-C# - returns '' if CsharpNewCacheKey is enabled", async (t) => {
const codeql = createStubCodeQL({});
const features = createFeatures([Feature.CsharpNewCacheKey]);
for (const knownLanguage of Object.values(KnownLanguage)) {
// Skip C# since we expect a result for it, which is tested in the previous test.
if (knownLanguage === KnownLanguage.csharp) {
continue;
}
const result = await getFeaturePrefix(codeql, features, knownLanguage);
t.deepEqual(result, "", `Expected no feature prefix for ${knownLanguage}`);
}
});

View File

@@ -6,9 +6,11 @@ import * as glob from "@actions/glob";
import { getTemporaryDirectory } from "./actions-util";
import { listActionsCaches } from "./api-client";
import { getTotalCacheSize } from "./caching-utils";
import { createCacheKeyHash, getTotalCacheSize } from "./caching-utils";
import { CodeQL } from "./codeql";
import { Config } from "./config-utils";
import { EnvVar } from "./environment";
import { Feature, FeatureEnablement } from "./feature-flags";
import { KnownLanguage, Language } from "./languages";
import { Logger } from "./logging";
import { getErrorMessage, getRequiredEnvParam } from "./util";
@@ -16,15 +18,21 @@ import { getErrorMessage, getRequiredEnvParam } from "./util";
/**
* Caching configuration for a particular language.
*/
interface CacheConfig {
/** The paths of directories on the runner that should be included in the cache. */
paths: string[];
export interface CacheConfig {
/** Gets the paths of directories on the runner that should be included in the cache. */
getDependencyPaths: () => string[];
/**
* Patterns for the paths of files whose contents affect which dependencies are used
* by a project. We find all files which match these patterns, calculate a hash for
* their contents, and use that hash as part of the cache key.
* Gets an array of glob patterns for the paths of files whose contents affect which dependencies are used
* by a project. This function also checks whether there are any matching files and returns
* `undefined` if no files match.
*
* The glob patterns are intended to be used for cache keys, where we find all files which match these
* patterns, calculate a hash for their contents, and use that hash as part of the cache key.
*/
hash: string[];
getHashPatterns: (
codeql: CodeQL,
features: FeatureEnablement,
) => Promise<string[] | undefined>;
}
const CODEQL_DEPENDENCY_CACHE_PREFIX = "codeql-dependencies";
@@ -39,21 +47,105 @@ export function getJavaTempDependencyDir(): string {
return join(getTemporaryDirectory(), "codeql_java", "repository");
}
/**
* Returns an array of paths of directories on the runner that should be included in a dependency cache
* for a Java analysis. It is important that this is a function, because we call `getTemporaryDirectory`
* which would otherwise fail in tests if we haven't had a chance to initialise `RUNNER_TEMP`.
*
* @returns The paths of directories on the runner that should be included in a dependency cache
* for a Java analysis.
*/
export function getJavaDependencyDirs(): string[] {
return [
// Maven
join(os.homedir(), ".m2", "repository"),
// Gradle
join(os.homedir(), ".gradle", "caches"),
// CodeQL Java build-mode: none
getJavaTempDependencyDir(),
];
}
/**
* Checks that there are files which match `patterns`. If there are matching files for any of the patterns,
* this function returns all `patterns`. Otherwise, `undefined` is returned.
*
* @param patterns The glob patterns to find matching files for.
* @returns The array of glob patterns if there are matching files, or `undefined` otherwise.
*/
export async function makePatternCheck(
patterns: string[],
): Promise<string[] | undefined> {
const globber = await makeGlobber(patterns);
if ((await globber.glob()).length === 0) {
return undefined;
}
return patterns;
}
/** These files contain accurate information about dependencies, including the exact versions
* that the relevant package manager has determined for the project. Using these gives us
* stable hashes unless the dependencies change.
*/
export const CSHARP_BASE_PATTERNS = [
// NuGet
"**/packages.lock.json",
// Paket
"**/paket.lock",
];
/** These are less accurate for use in cache key calculations, because they:
*
* - Don't contain the exact versions used. They may only contain version ranges or none at all.
* - They contain information unrelated to dependencies, which we don't care about.
*
* As a result, the hash we compute from these files may change, even if
* the dependencies haven't changed.
*/
export const CSHARP_EXTRA_PATTERNS = [
"**/*.csproj",
"**/packages.config",
"**/nuget.config",
];
/**
* Returns the list of glob patterns that should be used to calculate the cache key hash
* for a C# dependency cache. This will try to use `CSHARP_BASE_PATTERNS` whenever possible.
* As a fallback, it will also use `CSHARP_EXTRA_PATTERNS` if the corresponding FF is enabled.
*
* @param codeql The CodeQL instance to use.
* @param features Information about which FFs are enabled.
* @returns A list of glob patterns to use for hashing.
*/
export async function getCsharpHashPatterns(
codeql: CodeQL,
features: FeatureEnablement,
): Promise<string[] | undefined> {
const basePatterns = await internal.makePatternCheck(CSHARP_BASE_PATTERNS);
if (basePatterns !== undefined) {
return basePatterns;
}
if (await features.getValue(Feature.CsharpNewCacheKey, codeql)) {
return internal.makePatternCheck(CSHARP_EXTRA_PATTERNS);
}
// If we get to this point, we didn't find any files with `CSHARP_BASE_PATTERNS`,
// and `Feature.CsharpNewCacheKey` is not enabled.
return undefined;
}
/**
* Default caching configurations per language.
*/
function getDefaultCacheConfig(): { [language: string]: CacheConfig } {
return {
java: {
paths: [
// Maven
join(os.homedir(), ".m2", "repository"),
// Gradle
join(os.homedir(), ".gradle", "caches"),
// CodeQL Java build-mode: none
getJavaTempDependencyDir(),
],
hash: [
const defaultCacheConfigs: { [language: string]: CacheConfig } = {
java: {
getDependencyPaths: getJavaDependencyDirs,
getHashPatterns: async () =>
internal.makePatternCheck([
// Maven
"**/pom.xml",
// Gradle
@@ -63,23 +155,17 @@ function getDefaultCacheConfig(): { [language: string]: CacheConfig } {
"buildSrc/**/Dependencies.kt",
"gradle/*.versions.toml",
"**/versions.properties",
],
},
csharp: {
paths: [join(os.homedir(), ".nuget", "packages")],
hash: [
// NuGet
"**/packages.lock.json",
// Paket
"**/paket.lock",
],
},
go: {
paths: [join(os.homedir(), "go", "pkg", "mod")],
hash: ["**/go.sum"],
},
};
}
]),
},
csharp: {
getDependencyPaths: () => [join(os.homedir(), ".nuget", "packages")],
getHashPatterns: getCsharpHashPatterns,
},
go: {
getDependencyPaths: () => [join(os.homedir(), "go", "pkg", "mod")],
getHashPatterns: async () => internal.makePatternCheck(["**/go.sum"]),
},
};
async function makeGlobber(patterns: string[]): Promise<glob.Globber> {
return glob.create(patterns.join("\n"));
@@ -107,23 +193,57 @@ export interface DependencyCacheRestoreStatus {
/** An array of `DependencyCacheRestoreStatus` objects for each analysed language with a caching configuration. */
export type DependencyCacheRestoreStatusReport = DependencyCacheRestoreStatus[];
/**
* A wrapper around `cacheConfig.getHashPatterns` which logs when there are no files to calculate
* a hash for the cache key from.
*
* @param codeql The CodeQL instance to use.
* @param features Information about which FFs are enabled.
* @param language The language the `CacheConfig` is for. For use in the log message.
* @param cacheConfig The caching configuration to call `getHashPatterns` on.
* @param checkType Whether we are checking the patterns for a download or upload.
* @param logger The logger to write the log message to if there is an error.
* @returns An array of glob patterns to use for hashing files, or `undefined` if there are no matching files.
*/
export async function checkHashPatterns(
codeql: CodeQL,
features: FeatureEnablement,
language: Language,
cacheConfig: CacheConfig,
checkType: "download" | "upload",
logger: Logger,
): Promise<string[] | undefined> {
const patterns = await cacheConfig.getHashPatterns(codeql, features);
if (patterns === undefined) {
logger.info(
`Skipping ${checkType} of dependency cache for ${language} as we cannot calculate a hash for the cache key.`,
);
}
return patterns;
}
/**
* Attempts to restore dependency caches for the languages being analyzed.
*
* @param codeql The CodeQL instance to use.
* @param features Information about which FFs are enabled.
* @param languages The languages being analyzed.
* @param logger A logger to record some informational messages to.
* @param minimizeJavaJars Whether the Java extractor should rewrite downloaded JARs to minimize their size.
*
* @returns An array of `DependencyCacheRestoreStatus` objects for each analysed language with a caching configuration.
*/
export async function downloadDependencyCaches(
codeql: CodeQL,
features: FeatureEnablement,
languages: Language[],
logger: Logger,
minimizeJavaJars: boolean,
): Promise<DependencyCacheRestoreStatusReport> {
const status: DependencyCacheRestoreStatusReport = [];
for (const language of languages) {
const cacheConfig = getDefaultCacheConfig()[language];
const cacheConfig = defaultCacheConfigs[language];
if (cacheConfig === undefined) {
logger.info(
@@ -134,19 +254,22 @@ export async function downloadDependencyCaches(
// Check that we can find files to calculate the hash for the cache key from, so we don't end up
// with an empty string.
const globber = await makeGlobber(cacheConfig.hash);
if ((await globber.glob()).length === 0) {
const patterns = await checkHashPatterns(
codeql,
features,
language,
cacheConfig,
"download",
logger,
);
if (patterns === undefined) {
status.push({ language, hit_kind: CacheHitKind.NoHash });
logger.info(
`Skipping download of dependency cache for ${language} as we cannot calculate a hash for the cache key.`,
);
continue;
}
const primaryKey = await cacheKey(language, cacheConfig, minimizeJavaJars);
const primaryKey = await cacheKey(codeql, features, language, patterns);
const restoreKeys: string[] = [
await cachePrefix(language, minimizeJavaJars),
await cachePrefix(codeql, features, language),
];
logger.info(
@@ -157,7 +280,7 @@ export async function downloadDependencyCaches(
const start = performance.now();
const hitKey = await actionsCache.restoreCache(
cacheConfig.paths,
cacheConfig.getDependencyPaths(),
primaryKey,
restoreKeys,
);
@@ -203,20 +326,22 @@ export type DependencyCacheUploadStatusReport = DependencyCacheUploadStatus[];
/**
* Attempts to store caches for the languages that were analyzed.
*
* @param codeql The CodeQL instance to use.
* @param features Information about which FFs are enabled.
* @param config The configuration for this workflow.
* @param logger A logger to record some informational messages to.
* @param minimizeJavaJars Whether the Java extractor should rewrite downloaded JARs to minimize their size.
*
* @returns An array of `DependencyCacheUploadStatus` objects for each analysed language with a caching configuration.
*/
export async function uploadDependencyCaches(
codeql: CodeQL,
features: FeatureEnablement,
config: Config,
logger: Logger,
minimizeJavaJars: boolean,
): Promise<DependencyCacheUploadStatusReport> {
const status: DependencyCacheUploadStatusReport = [];
for (const language of config.languages) {
const cacheConfig = getDefaultCacheConfig()[language];
const cacheConfig = defaultCacheConfigs[language];
if (cacheConfig === undefined) {
logger.info(
@@ -227,13 +352,16 @@ export async function uploadDependencyCaches(
// Check that we can find files to calculate the hash for the cache key from, so we don't end up
// with an empty string.
const globber = await makeGlobber(cacheConfig.hash);
if ((await globber.glob()).length === 0) {
const patterns = await checkHashPatterns(
codeql,
features,
language,
cacheConfig,
"upload",
logger,
);
if (patterns === undefined) {
status.push({ language, result: CacheStoreResult.NoHash });
logger.info(
`Skipping upload of dependency cache for ${language} as we cannot calculate a hash for the cache key.`,
);
continue;
}
@@ -247,7 +375,11 @@ export async function uploadDependencyCaches(
// use the cache quota that we compete with. In that case, we do not wish to use up all of the quota
// with the dependency caches. For this, we could use the Cache API to check whether other workflows
// are using the quota and how full it is.
const size = await getTotalCacheSize(cacheConfig.paths, logger, true);
const size = await getTotalCacheSize(
cacheConfig.getDependencyPaths(),
logger,
true,
);
// Skip uploading an empty cache.
if (size === 0) {
@@ -258,7 +390,7 @@ export async function uploadDependencyCaches(
continue;
}
const key = await cacheKey(language, cacheConfig, minimizeJavaJars);
const key = await cacheKey(codeql, features, language, patterns);
logger.info(
`Uploading cache of size ${size} for ${language} with key ${key}...`,
@@ -266,7 +398,7 @@ export async function uploadDependencyCaches(
try {
const start = performance.now();
await actionsCache.saveCache(cacheConfig.paths, key);
await actionsCache.saveCache(cacheConfig.getDependencyPaths(), key);
const upload_duration_ms = Math.round(performance.now() - start);
status.push({
@@ -299,31 +431,86 @@ export async function uploadDependencyCaches(
/**
* Computes a cache key for the specified language.
*
* @param codeql The CodeQL instance to use.
* @param features Information about which FFs are enabled.
* @param language The language being analyzed.
* @param cacheConfig The cache configuration for the language.
* @param minimizeJavaJars Whether the Java extractor should rewrite downloaded JARs to minimize their size.
* @param patterns The file patterns to hash.
*
* @returns A cache key capturing information about the project(s) being analyzed in the specified language.
*/
async function cacheKey(
export async function cacheKey(
codeql: CodeQL,
features: FeatureEnablement,
language: Language,
cacheConfig: CacheConfig,
minimizeJavaJars: boolean = false,
patterns: string[],
): Promise<string> {
const hash = await glob.hashFiles(cacheConfig.hash.join("\n"));
return `${await cachePrefix(language, minimizeJavaJars)}${hash}`;
const hash = await glob.hashFiles(patterns.join("\n"));
return `${await cachePrefix(codeql, features, language)}${hash}`;
}
/**
* If experimental features which the cache contents depend on are enabled for the current language,
* this function returns a prefix that uniquely identifies the set of enabled features. The purpose of
* this is to avoid restoring caches whose contents depended on experimental features, if those
* experimental features are later disabled.
*
* @param codeql The CodeQL instance.
* @param features Information about enabled features.
* @param language The language we are creating the key for.
*
* @returns A cache key prefix identifying the enabled, experimental features that the cache depends on.
*/
export async function getFeaturePrefix(
codeql: CodeQL,
features: FeatureEnablement,
language: Language,
): Promise<string> {
const enabledFeatures: Feature[] = [];
const addFeatureIfEnabled = async (feature: Feature) => {
if (await features.getValue(feature, codeql)) {
enabledFeatures.push(feature);
}
};
if (language === KnownLanguage.java) {
// To ensure a safe rollout of JAR minimization, we change the key when the feature is enabled.
const minimizeJavaJars = await features.getValue(
Feature.JavaMinimizeDependencyJars,
codeql,
);
// To maintain backwards compatibility with this, we return "minify-" instead of a hash.
if (minimizeJavaJars) {
return "minify-";
}
} else if (language === KnownLanguage.csharp) {
await addFeatureIfEnabled(Feature.CsharpNewCacheKey);
}
// If any features that affect the cache are enabled, return a feature prefix by
// computing a hash of the feature array.
if (enabledFeatures.length > 0) {
return `${createCacheKeyHash(enabledFeatures)}-`;
}
// No feature prefix.
return "";
}
/**
* Constructs a prefix for the cache key, comprised of a CodeQL-specific prefix, a version number that
* can be changed to invalidate old caches, the runner's operating system, and the specified language name.
*
* @param codeql The CodeQL instance to use.
* @param features Information about which FFs are enabled.
* @param language The language being analyzed.
* @param minimizeJavaJars Whether the Java extractor should rewrite downloaded JARs to minimize their size.
* @returns The prefix that identifies what a cache is for.
*/
async function cachePrefix(
codeql: CodeQL,
features: FeatureEnablement,
language: Language,
minimizeJavaJars: boolean,
): Promise<string> {
const runnerOs = getRequiredEnvParam("RUNNER_OS");
const customPrefix = process.env[EnvVar.DEPENDENCY_CACHING_PREFIX];
@@ -333,12 +520,18 @@ async function cachePrefix(
prefix = `${prefix}-${customPrefix}`;
}
// To ensure a safe rollout of JAR minimization, we change the key when the feature is enabled.
if (language === KnownLanguage.java && minimizeJavaJars) {
prefix = `minify-${prefix}`;
}
// Calculate the feature prefix for the cache, if any. This is a hash that identifies
// experimental features that affect the cache contents.
const featurePrefix = await getFeaturePrefix(codeql, features, language);
return `${prefix}-${CODEQL_DEPENDENCY_CACHE_VERSION}-${runnerOs}-${language}-`;
// Assemble the cache key. For backwards compatibility with the JAR minification experiment's existing
// feature prefix usage, we add that feature prefix at the start. Other feature prefixes are inserted
// after the general CodeQL dependency cache prefix.
if (featurePrefix === "minify-") {
return `${featurePrefix}${prefix}-${CODEQL_DEPENDENCY_CACHE_VERSION}-${runnerOs}-${language}-`;
} else {
return `${prefix}-${featurePrefix}${CODEQL_DEPENDENCY_CACHE_VERSION}-${runnerOs}-${language}-`;
}
}
/** Represents information about our overall cache usage for CodeQL dependency caches. */
@@ -371,3 +564,7 @@ export async function getDependencyCacheUsage(
return undefined;
}
export const internal = {
makePatternCheck,
};

View File

@@ -188,6 +188,10 @@ test(
);
function runGetDiffRanges(changes: number, patch: string[] | undefined): any {
sinon
.stub(actionsUtil, "getRequiredInput")
.withArgs("checkout_path")
.returns("/checkout/path");
return exportedForTesting.getDiffRanges(
{
filename: "test.txt",
@@ -207,7 +211,7 @@ test("getDiffRanges: file diff too large", async (t) => {
const diffRanges = runGetDiffRanges(1000000, undefined);
t.deepEqual(diffRanges, [
{
path: "test.txt",
path: "/checkout/path/test.txt",
startLine: 0,
endLine: 0,
},
@@ -228,7 +232,7 @@ test("getDiffRanges: diff thunk with single addition range", async (t) => {
]);
t.deepEqual(diffRanges, [
{
path: "test.txt",
path: "/checkout/path/test.txt",
startLine: 53,
endLine: 54,
},
@@ -264,7 +268,7 @@ test("getDiffRanges: diff thunk with single update range", async (t) => {
]);
t.deepEqual(diffRanges, [
{
path: "test.txt",
path: "/checkout/path/test.txt",
startLine: 53,
endLine: 53,
},
@@ -286,12 +290,12 @@ test("getDiffRanges: diff thunk with addition ranges", async (t) => {
]);
t.deepEqual(diffRanges, [
{
path: "test.txt",
path: "/checkout/path/test.txt",
startLine: 53,
endLine: 53,
},
{
path: "test.txt",
path: "/checkout/path/test.txt",
startLine: 55,
endLine: 55,
},
@@ -318,12 +322,12 @@ test("getDiffRanges: diff thunk with mixed ranges", async (t) => {
]);
t.deepEqual(diffRanges, [
{
path: "test.txt",
path: "/checkout/path/test.txt",
startLine: 54,
endLine: 54,
},
{
path: "test.txt",
path: "/checkout/path/test.txt",
startLine: 57,
endLine: 58,
},
@@ -353,12 +357,12 @@ test("getDiffRanges: multiple diff thunks", async (t) => {
]);
t.deepEqual(diffRanges, [
{
path: "test.txt",
path: "/checkout/path/test.txt",
startLine: 53,
endLine: 54,
},
{
path: "test.txt",
path: "/checkout/path/test.txt",
startLine: 153,
endLine: 154,
},
@@ -369,7 +373,7 @@ test("getDiffRanges: no diff context lines", async (t) => {
const diffRanges = runGetDiffRanges(2, ["@@ -30 +50,2 @@", "+1", "+2"]);
t.deepEqual(diffRanges, [
{
path: "test.txt",
path: "/checkout/path/test.txt",
startLine: 50,
endLine: 51,
},

View File

@@ -191,6 +191,13 @@ function getDiffRanges(
fileDiff: FileDiff,
logger: Logger,
): DiffThunkRange[] | undefined {
// Diff-informed queries expect the file path to be absolute. CodeQL always
// uses forward slashes as the path separator, so on Windows we need to
// replace any backslashes with forward slashes.
const filename = path
.join(actionsUtil.getRequiredInput("checkout_path"), fileDiff.filename)
.replaceAll(path.sep, "/");
if (fileDiff.patch === undefined) {
if (fileDiff.changes === 0) {
// There are situations where a changed file legitimately has no diff.
@@ -205,7 +212,7 @@ function getDiffRanges(
// to a special diff range that covers the entire file.
return [
{
path: fileDiff.filename,
path: filename,
startLine: 0,
endLine: 0,
},
@@ -240,7 +247,7 @@ function getDiffRanges(
// Any line that does not start with a "+" or "-" terminates the current
// range of added lines.
diffRanges.push({
path: fileDiff.filename,
path: filename,
startLine: additionRangeStartLine,
endLine: currentLine - 1,
});

View File

@@ -47,6 +47,7 @@ export enum Feature {
AnalyzeUseNewUpload = "analyze_use_new_upload",
CleanupTrapCaches = "cleanup_trap_caches",
CppDependencyInstallation = "cpp_dependency_installation_enabled",
CsharpNewCacheKey = "csharp_new_cache_key",
DiffInformedQueries = "diff_informed_queries",
DisableCsharpBuildless = "disable_csharp_buildless",
DisableJavaBuildlessEnabled = "disable_java_buildless_enabled",
@@ -132,6 +133,11 @@ export const featureConfig: Record<
legacyApi: true,
minimumVersion: "2.15.0",
},
[Feature.CsharpNewCacheKey]: {
defaultValue: false,
envVar: "CODEQL_ACTION_CSHARP_NEW_CACHE_KEY",
minimumVersion: undefined,
},
[Feature.DiffInformedQueries]: {
defaultValue: true,
envVar: "CODEQL_ACTION_DIFF_INFORMED_QUERIES",

View File

@@ -578,15 +578,12 @@ async function run() {
}
// Restore dependency cache(s), if they exist.
const minimizeJavaJars = await features.getValue(
Feature.JavaMinimizeDependencyJars,
codeql,
);
if (shouldRestoreCache(config.dependencyCachingEnabled)) {
dependencyCachingResults = await downloadDependencyCaches(
codeql,
features,
config.languages,
logger,
minimizeJavaJars,
);
}
@@ -648,7 +645,7 @@ async function run() {
`${EnvVar.JAVA_EXTRACTOR_MINIMIZE_DEPENDENCY_JARS} is already set to '${process.env[EnvVar.JAVA_EXTRACTOR_MINIMIZE_DEPENDENCY_JARS]}', so the Action will not override it.`,
);
} else if (
minimizeJavaJars &&
(await features.getValue(Feature.JavaMinimizeDependencyJars, codeql)) &&
config.dependencyCachingEnabled &&
config.buildMode === BuildMode.None &&
config.languages.includes(KnownLanguage.java)

View File

@@ -1,4 +1,3 @@
import * as crypto from "crypto";
import * as fs from "fs";
import * as path from "path";
@@ -11,6 +10,7 @@ import {
getWorkflowRunID,
} from "./actions-util";
import { getAutomationID } from "./api-client";
import { createCacheKeyHash } from "./caching-utils";
import { type CodeQL } from "./codeql";
import { type Config } from "./config-utils";
import { getCommitOid, getFileOidsUnderPath } from "./git-utils";
@@ -514,27 +514,3 @@ export async function getCacheRestoreKeyPrefix(
// easier to debug and understand the cache key structure.
return `${CACHE_PREFIX}-${CACHE_VERSION}-${componentsHash}-${languages}-${codeQlVersion}-`;
}
/**
* Creates a SHA-256 hash of the cache key components to ensure uniqueness
* while keeping the cache key length manageable.
*
* @param components Object containing all components that should influence cache key uniqueness
* @returns A short SHA-256 hash (first 16 characters) of the components
*/
function createCacheKeyHash(components: Record<string, any>): string {
// From https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/JSON/stringify
//
// "Properties are visited using the same algorithm as Object.keys(), which
// has a well-defined order and is stable across implementations. For example,
// JSON.stringify on the same object will always produce the same string, and
// JSON.parse(JSON.stringify(obj)) would produce an object with the same key
// ordering as the original (assuming the object is completely
// JSON-serializable)."
const componentsJson = JSON.stringify(components);
return crypto
.createHash("sha256")
.update(componentsJson)
.digest("hex")
.substring(0, 16);
}

View File

@@ -1,8 +0,0 @@
extensions:
- addsTo:
pack: codeql/util
extensible: restrictAlertsTo
checkPresence: false
data:
- ['/checkout/path/main.js', 10, 20]

View File

@@ -1,178 +0,0 @@
{
"$schema": "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json",
"version": "2.1.0",
"runs": [{
"tool": {
"driver": {
"name": "LGTM.com",
"organization": "Semmle",
"version": "1.24.0-SNAPSHOT",
"rules": [{
"id": "js/unused-local-variable",
"name": "js/unused-local-variable",
"shortDescription": {
"text": "Unused variable, import, function or class"
},
"fullDescription": {
"text": "Unused variables, imports, functions or classes may be a symptom of a bug and should be examined carefully."
},
"defaultConfiguration": {
"level": "note"
},
"properties": {
"tags": ["maintainability"],
"kind": "problem",
"precision": "very-high",
"name": "Unused variable, import, function or class",
"description": "Unused variables, imports, functions or classes may be a symptom of a bug\n and should be examined carefully.",
"id": "js/unused-local-variable",
"problem.severity": "recommendation"
}
}]
}
},
"results": [{
"ruleId": "js/unused-local-variable",
"ruleIndex": 0,
"message": {
"text": "Unused variable foo."
},
"locations": [{
"physicalLocation": {
"artifactLocation": {
"uri": "main.js",
"uriBaseId": "%SRCROOT%",
"index": 0
},
"region": {
"startLine": 2,
"startColumn": 7,
"endColumn": 10
}
}
}],
"partialFingerprints": {
"primaryLocationLineHash": "39fa2ee980eb94b0:1",
"primaryLocationStartColumnFingerprint": "4"
}
}],
"columnKind": "utf16CodeUnits",
"properties": {
"semmle.formatSpecifier": "2.1.0",
"semmle.sourceLanguage": "java"
}
},
{
"tool" : {
"driver" : {
"name" : "CodeQL command-line toolchain",
"organization" : "GitHub",
"semanticVersion" : "2.0.0",
"rules" : [ {
"id" : "js/unused-local-variable",
"name" : "js/unused-local-variable",
"shortDescription" : {
"text" : "Unused variable, import, function or class"
},
"fullDescription" : {
"text" : "Unused variables, imports, functions or classes may be a symptom of a bug and should be examined carefully."
},
"defaultConfiguration" : {
"level": "note"
},
"properties" : {
"tags" : [ "maintainability" ],
"kind" : "problem",
"precision" : "very-high",
"name" : "Unused variable, import, function or class",
"description" : "Unused variables, imports, functions or classes may be a symptom of a bug\n and should be examined carefully.",
"id" : "js/unused-local-variable",
"problem.severity" : "recommendation"
}
},
{
"id": "js/inconsistent-use-of-new",
"name": "js/inconsistent-use-of-new",
"shortDescription": {
"text": "Inconsistent use of 'new'"
},
"fullDescription": {
"text": "If a function is intended to be a constructor, it should always be invoked with 'new'. Otherwise, it should always be invoked as a normal function, that is, without 'new'."
},
"defaultConfiguration": {
"level": "note"
},
"properties": {
"tags": [
"reliability",
"correctness",
"language-features"
],
"kind": "problem",
"precision": "very-high",
"problem.severity": "warning"
}
} ]
}
},
"artifacts" : [ {
"location" : {
"uri" : "main.js",
"uriBaseId" : "%SRCROOT%",
"index" : 0
}
},
{
"location": {
"uri": "src/promiseUtils.js",
"uriBaseId": "%SRCROOT%",
"index": 1
}
},
{
"location": {
"uri": "src/LiveQueryClient.js",
"uriBaseId": "%SRCROOT%",
"index": 2
}
},
{
"location": {
"uri": "src/ParseObject.js",
"uriBaseId": "%SRCROOT%",
"index": 3
}
} ],
"results" : [ {
"ruleId" : "js/unused-local-variable",
"ruleIndex" : 0,
"message" : {
"text" : "Unused variable foo."
},
"locations" : [ {
"physicalLocation" : {
"artifactLocation" : {
"uri" : "main.js",
"uriBaseId" : "%SRCROOT%",
"index" : 0
},
"region" : {
"startLine" : 2,
"startColumn" : 7,
"endColumn" : 10
}
}
} ],
"partialFingerprints" : {
"primaryLocationLineHash" : "39fa2ee980eb94b0:1",
"primaryLocationStartColumnFingerprint" : "4"
}
}],
"newlineSequences" : [ "\r\n", "\n", "", "" ],
"columnKind" : "utf16CodeUnits",
"properties" : {
"semmle.formatSpecifier" : "sarif-latest"
}
}
]
}

View File

@@ -9,16 +9,10 @@ import * as sinon from "sinon";
import * as analyses from "./analyses";
import { AnalysisKind, CodeQuality, CodeScanning } from "./analyses";
import * as api from "./api-client";
import * as diffUtils from "./diff-informed-analysis-utils";
import { getRunnerLogger, Logger } from "./logging";
import { setupTests } from "./testing-utils";
import * as uploadLib from "./upload-lib";
import {
GitHubVariant,
initializeEnvironment,
SarifFile,
withTmpDir,
} from "./util";
import { GitHubVariant, initializeEnvironment, withTmpDir } from "./util";
setupTests(test);
@@ -966,30 +960,3 @@ for (const analysis of [CodeScanning, CodeQuality]) {
});
});
}
function runFilterAlertsByDiffRange(
input: SarifFile,
diffRanges: diffUtils.DiffThunkRange[],
): SarifFile {
sinon.stub(diffUtils, "readDiffRangesJsonFile").returns(diffRanges);
return uploadLib.filterAlertsByDiffRange(getRunnerLogger(true), input);
}
test("filterAlertsByDiffRange filters out alerts outside diff-range", (t) => {
const input = uploadLib.readSarifFile(
`${__dirname}/../src/testdata/valid-sarif.sarif`,
);
const actualOutput = runFilterAlertsByDiffRange(input, [
{
path: "main.js",
startLine: 1,
endLine: 3,
},
]);
const expectedOutput = uploadLib.readSarifFile(
`${__dirname}/../src/testdata/valid-sarif-diff-filtered.sarif`,
);
t.deepEqual(actualOutput, expectedOutput);
});

View File

@@ -1134,15 +1134,14 @@ export class InvalidSarifUploadError extends Error {
}
}
export function filterAlertsByDiffRange(
logger: Logger,
sarif: SarifFile,
): SarifFile {
function filterAlertsByDiffRange(logger: Logger, sarif: SarifFile): SarifFile {
const diffRanges = readDiffRangesJsonFile(logger);
if (!diffRanges?.length) {
return sarif;
}
const checkoutPath = actionsUtil.getRequiredInput("checkout_path");
for (const run of sarif.runs) {
if (run.results) {
run.results = run.results.filter((result) => {
@@ -1157,6 +1156,11 @@ export function filterAlertsByDiffRange(
if (!locationUri || locationStartLine === undefined) {
return false;
}
// CodeQL always uses forward slashes as the path separator, so on Windows we
// need to replace any backslashes with forward slashes.
const locationPath = path
.join(checkoutPath, locationUri)
.replaceAll(path.sep, "/");
// Alert filtering here replicates the same behavior as the restrictAlertsTo
// extensible predicate in CodeQL. See the restrictAlertsTo documentation
// https://codeql.github.com/codeql-standard-libraries/csharp/codeql/util/AlertFiltering.qll/predicate.AlertFiltering$restrictAlertsTo.3.html
@@ -1164,7 +1168,7 @@ export function filterAlertsByDiffRange(
// of an alert location.
return diffRanges.some(
(range) =>
range.path === locationUri &&
range.path === locationPath &&
((range.startLine <= locationStartLine &&
range.endLine >= locationStartLine) ||
(range.startLine === 0 && range.endLine === 0)),

View File

@@ -0,0 +1,6 @@
{
"sdk": {
"version": "9.0.307",
"rollForward": "latestFeature"
}
}