github/codeql-action
1
0
Fork 0
mirror of https://github.com/github/codeql-action.git synced 2026-01-03 21:20:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix dependency cycle between util and config-utils

Browse Source
This commit is contained in:
Henry Mercer
2023-07-19 16:48:21 +01:00
parent d577d6f6b1
commit 004f976bef
21 changed files with 283 additions and 288 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/analyze-action.ts
View File

@@ -17,7 +17,7 @@ import {
import { getApiDetails, getGitHubVersion } from "./api-client";
import { runAutobuild } from "./autobuild";
import { getCodeQL } from "./codeql";
import { Config, getConfig } from "./config-utils";
import { Config, getConfig, getMlPoweredJsQueriesStatus } from "./config-utils";
import { uploadDatabases } from "./database-upload";
import { EnvVar } from "./environment";
import { Feature, Features } from "./feature-flags";
@@ -71,8 +71,7 @@ export async function sendStatusReport(
...statusReportBase,
...(config
? {
ml_powered_javascript_queries:
util.getMlPoweredJsQueriesStatus(config),
ml_powered_javascript_queries: getMlPoweredJsQueriesStatus(config),
}
: {}),
...(stats || {}),

8
src/codeql.ts
View File

@@ -7,7 +7,11 @@ import * as yaml from "js-yaml";
import { getOptionalInput } from "./actions-util";
import * as api from "./api-client";
import { Config, getGeneratedCodeScanningConfigPath } from "./config-utils";
import {
Config,
getGeneratedCodeScanningConfigPath,
getMlPoweredJsQueriesPack,
} from "./config-utils";
import { EnvVar } from "./environment";
import { errorMatchers } from "./error-matcher";
import {
@@ -1212,7 +1216,7 @@ async function generateCodeScanningConfig(
if (config.augmentationProperties.injectedMlQueries) {
// We need to inject the ML queries into the original user input before
// we pass this on to the CLI, to make sure these get run.
const packString = await util.getMlPoweredJsQueriesPack(codeql);
const packString = await getMlPoweredJsQueriesPack(codeql);
if (augmentedConfig.packs === undefined) augmentedConfig.packs = [];
if (Array.isArray(augmentedConfig.packs)) {

73
src/config-utils.test.ts
View File

@@ -14,6 +14,7 @@ import {
setCodeQL,
} from "./codeql";
import * as configUtils from "./config-utils";
import { ML_POWERED_JS_QUERIES_PACK_NAME } from "./config-utils";
import { Feature } from "./feature-flags";
import { Language } from "./languages";
import { getRunnerLogger, Logger } from "./logging";
@@ -23,7 +24,14 @@ import {
createFeatures,
mockLanguagesInRepo as mockLanguagesInRepo,
} from "./testing-utils";
import { GitHubVariant, GitHubVersion, UserError, withTmpDir } from "./util";
import {
DEFAULT_DEBUG_ARTIFACT_NAME,
DEFAULT_DEBUG_DATABASE_NAME,
GitHubVariant,
GitHubVersion,
UserError,
withTmpDir,
} from "./util";
setupTests(test);
@@ -2809,3 +2817,66 @@ const mockRepositoryNwo = parseRepositoryNwo("owner/repo");
t.deepEqual(mockRequest.called, args.expectedApiCall);
});
});
const ML_POWERED_JS_STATUS_TESTS: Array<[string[], string]> = [
// If no packs are loaded, status is false.
[[], "false"],
// If another pack is loaded but not the ML-powered query pack, status is false.
[["some-other/pack"], "false"],
// If the ML-powered query pack is loaded with a specific version, status is that version.
[[`${ML_POWERED_JS_QUERIES_PACK_NAME}@~0.1.0`], "~0.1.0"],
// If the ML-powered query pack is loaded with a specific version and another pack is loaded, the
// status is the version of the ML-powered query pack.
[["some-other/pack", `${ML_POWERED_JS_QUERIES_PACK_NAME}@~0.1.0`], "~0.1.0"],
// If the ML-powered query pack is loaded without a version, the status is "latest".
[[ML_POWERED_JS_QUERIES_PACK_NAME], "latest"],
// If the ML-powered query pack is loaded with two different versions, the status is "other".
[
[
`${ML_POWERED_JS_QUERIES_PACK_NAME}@~0.0.1`,
`${ML_POWERED_JS_QUERIES_PACK_NAME}@~0.0.2`,
],
"other",
],
// If the ML-powered query pack is loaded with no specific version, and another pack is loaded,
// the status is "latest".
[["some-other/pack", ML_POWERED_JS_QUERIES_PACK_NAME], "latest"],
];
for (const [packs, expectedStatus] of ML_POWERED_JS_STATUS_TESTS) {
const packDescriptions = `[${packs
.map((pack) => JSON.stringify(pack))
.join(", ")}]`;
test(`ML-powered JS queries status report is "${expectedStatus}" for packs = ${packDescriptions}`, (t) => {
return withTmpDir(async (tmpDir) => {
const config: configUtils.Config = {
languages: [],
queries: {},
paths: [],
pathsIgnore: [],
originalUserInput: {},
tempDir: tmpDir,
codeQLCmd: "",
gitHubVersion: {
type: GitHubVariant.DOTCOM,
} as GitHubVersion,
dbLocation: "",
packs: {
javascript: packs,
},
debugMode: false,
debugArtifactName: DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: DEFAULT_DEBUG_DATABASE_NAME,
augmentationProperties: {
injectedMlQueries: false,
packsInputCombines: false,
queriesInputCombines: false,
},
trapCaches: {},
trapCacheDownloadTime: 0,
};
t.is(configUtils.getMlPoweredJsQueriesStatus(config), expectedStatus);
});
});
}

73
src/config-utils.ts
View File

@@ -28,13 +28,7 @@ import {
import { Logger } from "./logging";
import { RepositoryNwo } from "./repository";
import { downloadTrapCaches } from "./trap-caching";
import {
codeQlVersionAbove,
getMlPoweredJsQueriesPack,
GitHubVersion,
ML_POWERED_JS_QUERIES_PACK_NAME,
UserError,
} from "./util";
import { codeQlVersionAbove, GitHubVersion, UserError } from "./util";
// Property names from the user-supplied config file.
const NAME_PROPERTY = "name";
@@ -1669,6 +1663,71 @@ function combinePacks(packs1: Packs, packs2: Packs): Packs {
return packs;
}
export const ML_POWERED_JS_QUERIES_PACK_NAME =
"codeql/javascript-experimental-atm-queries";
/**
* Gets the ML-powered JS query pack to add to the analysis if a repo is opted into the ML-powered
* queries beta.
*/
export async function getMlPoweredJsQueriesPack(
codeQL: CodeQL
): Promise<string> {
let version;
if (await codeQlVersionAbove(codeQL, "2.11.3")) {
version = "~0.4.0";
} else {
version = `~0.3.0`;
}
return prettyPrintPack({
name: ML_POWERED_JS_QUERIES_PACK_NAME,
version,
});
}
/**
* Get information about ML-powered JS queries to populate status reports with.
*
* This will be:
*
* - The version string if the analysis is using a single version of the ML-powered query pack.
* - "latest" if the version string of the ML-powered query pack is undefined. This is unlikely to
* occur in practice (see comment below).
* - "false" if the analysis won't run any ML-powered JS queries.
* - "other" in all other cases.
*
* Our goal of the status report here is to allow us to compare the occurrence of timeouts and other
* errors with ML-powered queries turned on and off. We also want to be able to compare minor
* version bumps caused by us bumping the version range of `ML_POWERED_JS_QUERIES_PACK` in a new
* version of the CodeQL Action. For instance, we might want to compare the `~0.1.0` and `~0.0.2`
* version strings.
*
* This function lives here rather than in `init-action.ts` so it's easier to test, since tests for
* `init-action.ts` would each need to live in their own file. See `analyze-action-env.ts` for an
* explanation as to why this is.
*/
export function getMlPoweredJsQueriesStatus(config: Config): string {
const mlPoweredJsQueryPacks = (config.packs.javascript || [])
.map((p) => parsePacksSpecification(p))
.filter(
(pack) =>
pack.name === "codeql/javascript-experimental-atm-queries" && !pack.path
);
switch (mlPoweredJsQueryPacks.length) {
case 1:
// We should always specify an explicit version string in `getMlPoweredJsQueriesPack`,
// otherwise we won't be able to make changes to the pack unless those changes are compatible
// with each version of the CodeQL Action. Therefore in practice we should only hit the
// `latest` case here when customers have explicitly added the ML-powered query pack to their
// CodeQL config.
return mlPoweredJsQueryPacks[0].version || "latest";
case 0:
return "false";
default:
return "other";
}
}
function dbLocationOrDefault(
dbLocation: string | undefined,
tempDir: string

2
src/init-action.ts
View File

@@ -16,6 +16,7 @@ import {
import { getGitHubVersion } from "./api-client";
import { CodeQL } from "./codeql";
import * as configUtils from "./config-utils";
import { getMlPoweredJsQueriesStatus } from "./config-utils";
import { EnvVar } from "./environment";
import { Feature, Features } from "./feature-flags";
import {
@@ -35,7 +36,6 @@ import {
DEFAULT_DEBUG_ARTIFACT_NAME,
DEFAULT_DEBUG_DATABASE_NAME,
getMemoryFlagValue,
getMlPoweredJsQueriesStatus,
getRequiredEnvParam,
getThreadsFlagValue,
initializeEnvironment,

67
src/util.test.ts
View File

@@ -4,7 +4,6 @@ import path from "path";
import test from "ava";
import { Config } from "./config-utils";
import { getRunnerLogger } from "./logging";
import { getRecordingLogger, LoggedMessage, setupTests } from "./testing-utils";
import * as util from "./util";
@@ -189,72 +188,6 @@ test("allowed API versions", async (t) => {
);
});
const ML_POWERED_JS_STATUS_TESTS: Array<[string[], string]> = [
// If no packs are loaded, status is false.
[[], "false"],
// If another pack is loaded but not the ML-powered query pack, status is false.
[["some-other/pack"], "false"],
// If the ML-powered query pack is loaded with a specific version, status is that version.
[[`${util.ML_POWERED_JS_QUERIES_PACK_NAME}@~0.1.0`], "~0.1.0"],
// If the ML-powered query pack is loaded with a specific version and another pack is loaded, the
// status is the version of the ML-powered query pack.
[
["some-other/pack", `${util.ML_POWERED_JS_QUERIES_PACK_NAME}@~0.1.0`],
"~0.1.0",
],
// If the ML-powered query pack is loaded without a version, the status is "latest".
[[util.ML_POWERED_JS_QUERIES_PACK_NAME], "latest"],
// If the ML-powered query pack is loaded with two different versions, the status is "other".
[
[
`${util.ML_POWERED_JS_QUERIES_PACK_NAME}@~0.0.1`,
`${util.ML_POWERED_JS_QUERIES_PACK_NAME}@~0.0.2`,
],
"other",
],
// If the ML-powered query pack is loaded with no specific version, and another pack is loaded,
// the status is "latest".
[["some-other/pack", util.ML_POWERED_JS_QUERIES_PACK_NAME], "latest"],
];
for (const [packs, expectedStatus] of ML_POWERED_JS_STATUS_TESTS) {
const packDescriptions = `[${packs
.map((pack) => JSON.stringify(pack))
.join(", ")}]`;
test(`ML-powered JS queries status report is "${expectedStatus}" for packs = ${packDescriptions}`, (t) => {
return util.withTmpDir(async (tmpDir) => {
const config: Config = {
languages: [],
queries: {},
paths: [],
pathsIgnore: [],
originalUserInput: {},
tempDir: tmpDir,
codeQLCmd: "",
gitHubVersion: {
type: util.GitHubVariant.DOTCOM,
} as util.GitHubVersion,
dbLocation: "",
packs: {
javascript: packs,
},
debugMode: false,
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
augmentationProperties: {
injectedMlQueries: false,
packsInputCombines: false,
queriesInputCombines: false,
},
trapCaches: {},
trapCacheDownloadTime: 0,
};
t.is(util.getMlPoweredJsQueriesStatus(config), expectedStatus);
});
});
}
test("doesDirectoryExist", async (t) => {
// Returns false if no file/dir of this name exists
t.false(util.doesDirectoryExist("non-existent-file.txt"));

71
src/util.ts
View File

@@ -10,11 +10,7 @@ import * as semver from "semver";
import * as apiCompatibility from "./api-compatibility.json";
import type { CodeQL } from "./codeql";
import {
Config,
parsePacksSpecification,
prettyPrintPack,
} from "./config-utils";
import type { Config } from "./config-utils";
import { EnvVar } from "./environment";
import { Language } from "./languages";
import { Logger } from "./logging";
@@ -539,71 +535,6 @@ export async function supportExpectDiscardedCache(
return codeQlVersionAbove(codeQL, "2.12.1");
}
export const ML_POWERED_JS_QUERIES_PACK_NAME =
"codeql/javascript-experimental-atm-queries";
/**
* Gets the ML-powered JS query pack to add to the analysis if a repo is opted into the ML-powered
* queries beta.
*/
export async function getMlPoweredJsQueriesPack(
codeQL: CodeQL
): Promise<string> {
let version;
if (await codeQlVersionAbove(codeQL, "2.11.3")) {
version = "~0.4.0";
} else {
version = `~0.3.0`;
}
return prettyPrintPack({
name: ML_POWERED_JS_QUERIES_PACK_NAME,
version,
});
}
/**
* Get information about ML-powered JS queries to populate status reports with.
*
* This will be:
*
* - The version string if the analysis is using a single version of the ML-powered query pack.
* - "latest" if the version string of the ML-powered query pack is undefined. This is unlikely to
* occur in practice (see comment below).
* - "false" if the analysis won't run any ML-powered JS queries.
* - "other" in all other cases.
*
* Our goal of the status report here is to allow us to compare the occurrence of timeouts and other
* errors with ML-powered queries turned on and off. We also want to be able to compare minor
* version bumps caused by us bumping the version range of `ML_POWERED_JS_QUERIES_PACK` in a new
* version of the CodeQL Action. For instance, we might want to compare the `~0.1.0` and `~0.0.2`
* version strings.
*
* This function lives here rather than in `init-action.ts` so it's easier to test, since tests for
* `init-action.ts` would each need to live in their own file. See `analyze-action-env.ts` for an
* explanation as to why this is.
*/
export function getMlPoweredJsQueriesStatus(config: Config): string {
const mlPoweredJsQueryPacks = (config.packs.javascript || [])
.map((p) => parsePacksSpecification(p))
.filter(
(pack) =>
pack.name === "codeql/javascript-experimental-atm-queries" && !pack.path
);
switch (mlPoweredJsQueryPacks.length) {
case 1:
// We should always specify an explicit version string in `getMlPoweredJsQueriesPack`,
// otherwise we won't be able to make changes to the pack unless those changes are compatible
// with each version of the CodeQL Action. Therefore in practice we should only hit the
// `latest` case here when customers have explicitly added the ML-powered query pack to their
// CodeQL config.
return mlPoweredJsQueryPacks[0].version || "latest";
case 0:
return "false";
default:
return "other";
}
}
/*
* Returns whether we are in test mode.
*