mirror of
https://github.com/github/codeql-action.git
synced 2025-12-27 09:40:17 +08:00
531 lines
18 KiB
TypeScript
531 lines
18 KiB
TypeScript
import * as fs from "fs";
|
|
import * as path from "path";
|
|
|
|
import test, { ExecutionContext } from "ava";
|
|
|
|
import * as defaults from "./defaults.json";
|
|
import {
|
|
Feature,
|
|
featureConfig,
|
|
FeatureEnablement,
|
|
Features,
|
|
FEATURE_FLAGS_FILE_NAME,
|
|
} from "./feature-flags";
|
|
import { getRunnerLogger } from "./logging";
|
|
import { parseRepositoryNwo } from "./repository";
|
|
import {
|
|
getRecordingLogger,
|
|
LoggedMessage,
|
|
mockCodeQLVersion,
|
|
mockFeatureFlagApiEndpoint,
|
|
setupActionsVars,
|
|
setupTests,
|
|
} from "./testing-utils";
|
|
import * as util from "./util";
|
|
import { GitHubVariant, initializeEnvironment, withTmpDir } from "./util";
|
|
|
|
setupTests(test);
|
|
|
|
test.beforeEach(() => {
|
|
initializeEnvironment("1.2.3");
|
|
});
|
|
|
|
const testRepositoryNwo = parseRepositoryNwo("github/example");
|
|
|
|
const ALL_FEATURES_DISABLED_VARIANTS: Array<{
|
|
description: string;
|
|
gitHubVersion: util.GitHubVersion;
|
|
}> = [
|
|
{
|
|
description: "GHES",
|
|
gitHubVersion: { type: GitHubVariant.GHES, version: "3.0.0" },
|
|
},
|
|
{ description: "GHAE", gitHubVersion: { type: GitHubVariant.GHAE } },
|
|
];
|
|
|
|
for (const variant of ALL_FEATURES_DISABLED_VARIANTS) {
|
|
test(`All features are disabled if running against ${variant.description}`, async (t) => {
|
|
await withTmpDir(async (tmpDir) => {
|
|
const loggedMessages = [];
|
|
const features = setUpFeatureFlagTests(
|
|
tmpDir,
|
|
getRecordingLogger(loggedMessages),
|
|
variant.gitHubVersion,
|
|
);
|
|
|
|
for (const feature of Object.values(Feature)) {
|
|
t.deepEqual(
|
|
await features.getValue(feature, includeCodeQlIfRequired(feature)),
|
|
featureConfig[feature].defaultValue,
|
|
);
|
|
}
|
|
|
|
t.assert(
|
|
loggedMessages.find(
|
|
(v: LoggedMessage) =>
|
|
v.type === "debug" &&
|
|
v.message ===
|
|
"Not running against github.com. Disabling all toggleable features.",
|
|
) !== undefined,
|
|
);
|
|
});
|
|
});
|
|
}
|
|
|
|
test("API response missing and features use default value", async (t) => {
|
|
await withTmpDir(async (tmpDir) => {
|
|
const loggedMessages: LoggedMessage[] = [];
|
|
const features = setUpFeatureFlagTests(
|
|
tmpDir,
|
|
getRecordingLogger(loggedMessages),
|
|
);
|
|
|
|
mockFeatureFlagApiEndpoint(403, {});
|
|
|
|
for (const feature of Object.values(Feature)) {
|
|
t.assert(
|
|
(await features.getValue(feature, includeCodeQlIfRequired(feature))) ===
|
|
featureConfig[feature].defaultValue,
|
|
);
|
|
}
|
|
assertAllFeaturesUndefinedInApi(t, loggedMessages);
|
|
});
|
|
});
|
|
|
|
test("Features use default value if they're not returned in API response", async (t) => {
|
|
await withTmpDir(async (tmpDir) => {
|
|
const loggedMessages: LoggedMessage[] = [];
|
|
const features = setUpFeatureFlagTests(
|
|
tmpDir,
|
|
getRecordingLogger(loggedMessages),
|
|
);
|
|
|
|
mockFeatureFlagApiEndpoint(200, {});
|
|
|
|
for (const feature of Object.values(Feature)) {
|
|
t.assert(
|
|
(await features.getValue(feature, includeCodeQlIfRequired(feature))) ===
|
|
featureConfig[feature].defaultValue,
|
|
);
|
|
}
|
|
|
|
assertAllFeaturesUndefinedInApi(t, loggedMessages);
|
|
});
|
|
});
|
|
|
|
test("Feature flags exception is propagated if the API request errors", async (t) => {
|
|
await withTmpDir(async (tmpDir) => {
|
|
const features = setUpFeatureFlagTests(tmpDir);
|
|
|
|
mockFeatureFlagApiEndpoint(500, {});
|
|
|
|
const someFeature = Object.values(Feature)[0];
|
|
|
|
await t.throwsAsync(
|
|
async () =>
|
|
features.getValue(someFeature, includeCodeQlIfRequired(someFeature)),
|
|
{
|
|
message:
|
|
"Encountered an error while trying to determine feature enablement: Error: some error message",
|
|
},
|
|
);
|
|
});
|
|
});
|
|
|
|
for (const feature of Object.keys(featureConfig)) {
|
|
test(`Only feature '${feature}' is enabled if enabled in the API response. Other features disabled`, async (t) => {
|
|
await withTmpDir(async (tmpDir) => {
|
|
const features = setUpFeatureFlagTests(tmpDir);
|
|
|
|
// set all features to false except the one we're testing
|
|
const expectedFeatureEnablement: { [feature: string]: boolean } = {};
|
|
for (const f of Object.keys(featureConfig)) {
|
|
expectedFeatureEnablement[f] = f === feature;
|
|
}
|
|
mockFeatureFlagApiEndpoint(200, expectedFeatureEnablement);
|
|
|
|
// retrieve the values of the actual features
|
|
const actualFeatureEnablement: { [feature: string]: boolean } = {};
|
|
for (const f of Object.keys(featureConfig)) {
|
|
actualFeatureEnablement[f] = await features.getValue(
|
|
f as Feature,
|
|
includeCodeQlIfRequired(f),
|
|
);
|
|
}
|
|
|
|
// All features should be false except the one we're testing
|
|
t.deepEqual(actualFeatureEnablement, expectedFeatureEnablement);
|
|
});
|
|
});
|
|
|
|
test(`Only feature '${feature}' is enabled if the associated environment variable is true. Others disabled.`, async (t) => {
|
|
await withTmpDir(async (tmpDir) => {
|
|
const features = setUpFeatureFlagTests(tmpDir);
|
|
|
|
const expectedFeatureEnablement = initializeFeatures(false);
|
|
mockFeatureFlagApiEndpoint(200, expectedFeatureEnablement);
|
|
|
|
// feature should be disabled initially
|
|
t.assert(
|
|
!(await features.getValue(
|
|
feature as Feature,
|
|
includeCodeQlIfRequired(feature),
|
|
)),
|
|
);
|
|
|
|
// set env var to true and check that the feature is now enabled
|
|
process.env[featureConfig[feature].envVar] = "true";
|
|
t.assert(
|
|
await features.getValue(
|
|
feature as Feature,
|
|
includeCodeQlIfRequired(feature),
|
|
),
|
|
);
|
|
});
|
|
});
|
|
|
|
test(`Feature '${feature}' is disabled if the associated environment variable is false, even if enabled in API`, async (t) => {
|
|
await withTmpDir(async (tmpDir) => {
|
|
const features = setUpFeatureFlagTests(tmpDir);
|
|
|
|
const expectedFeatureEnablement = initializeFeatures(true);
|
|
mockFeatureFlagApiEndpoint(200, expectedFeatureEnablement);
|
|
|
|
// feature should be enabled initially
|
|
t.assert(
|
|
await features.getValue(
|
|
feature as Feature,
|
|
includeCodeQlIfRequired(feature),
|
|
),
|
|
);
|
|
|
|
// set env var to false and check that the feature is now disabled
|
|
process.env[featureConfig[feature].envVar] = "false";
|
|
t.assert(
|
|
!(await features.getValue(
|
|
feature as Feature,
|
|
includeCodeQlIfRequired(feature),
|
|
)),
|
|
);
|
|
});
|
|
});
|
|
|
|
if (featureConfig[feature].minimumVersion !== undefined) {
|
|
test(`Getting feature '${feature} should throw if no codeql is provided`, async (t) => {
|
|
await withTmpDir(async (tmpDir) => {
|
|
const features = setUpFeatureFlagTests(tmpDir);
|
|
|
|
const expectedFeatureEnablement = initializeFeatures(true);
|
|
mockFeatureFlagApiEndpoint(200, expectedFeatureEnablement);
|
|
|
|
await t.throwsAsync(async () => features.getValue(feature as Feature), {
|
|
message: `Internal error: A minimum version is specified for feature ${feature}, but no instance of CodeQL was provided.`,
|
|
});
|
|
});
|
|
});
|
|
}
|
|
|
|
if (featureConfig[feature].minimumVersion !== undefined) {
|
|
test(`Feature '${feature}' is disabled if the minimum CLI version is below ${featureConfig[feature].minimumVersion}`, async (t) => {
|
|
await withTmpDir(async (tmpDir) => {
|
|
const features = setUpFeatureFlagTests(tmpDir);
|
|
|
|
const expectedFeatureEnablement = initializeFeatures(true);
|
|
mockFeatureFlagApiEndpoint(200, expectedFeatureEnablement);
|
|
|
|
// feature should be disabled when an old CLI version is set
|
|
let codeql = mockCodeQLVersion("2.0.0");
|
|
t.assert(!(await features.getValue(feature as Feature, codeql)));
|
|
|
|
// even setting the env var to true should not enable the feature if
|
|
// the minimum CLI version is not met
|
|
process.env[featureConfig[feature].envVar] = "true";
|
|
t.assert(!(await features.getValue(feature as Feature, codeql)));
|
|
|
|
// feature should be enabled when a new CLI version is set
|
|
// and env var is not set
|
|
process.env[featureConfig[feature].envVar] = "";
|
|
codeql = mockCodeQLVersion(featureConfig[feature].minimumVersion);
|
|
t.assert(await features.getValue(feature as Feature, codeql));
|
|
|
|
// set env var to false and check that the feature is now disabled
|
|
process.env[featureConfig[feature].envVar] = "false";
|
|
t.assert(!(await features.getValue(feature as Feature, codeql)));
|
|
});
|
|
});
|
|
}
|
|
}
|
|
|
|
// If we ever run into a situation where we no longer have any features that
|
|
// specify a minimum version, then we will have a bunch of code no longer being
|
|
// tested. This is unlikely, and this test will fail if that happens.
|
|
// If we do end up in that situation, then we should consider adding a synthetic
|
|
// feature with a minimum version that is only used for tests.
|
|
test("At least one feature has a minimum version specified", (t) => {
|
|
t.assert(
|
|
Object.values(featureConfig).some((f) => f.minimumVersion !== undefined),
|
|
"At least one feature should have a minimum version specified",
|
|
);
|
|
|
|
// An even less likely scenario is that we no longer have any features.
|
|
t.assert(
|
|
Object.values(featureConfig).length > 0,
|
|
"There should be at least one feature",
|
|
);
|
|
});
|
|
|
|
test("Feature flags are saved to disk", async (t) => {
|
|
await withTmpDir(async (tmpDir) => {
|
|
const features = setUpFeatureFlagTests(tmpDir);
|
|
const expectedFeatureEnablement = initializeFeatures(true);
|
|
mockFeatureFlagApiEndpoint(200, expectedFeatureEnablement);
|
|
|
|
const cachedFeatureFlags = path.join(tmpDir, FEATURE_FLAGS_FILE_NAME);
|
|
|
|
t.false(
|
|
fs.existsSync(cachedFeatureFlags),
|
|
"Feature flag cached file should not exist before getting feature flags",
|
|
);
|
|
|
|
t.true(
|
|
await features.getValue(
|
|
Feature.CliConfigFileEnabled,
|
|
includeCodeQlIfRequired(Feature.CliConfigFileEnabled),
|
|
),
|
|
"Feature flag should be enabled initially",
|
|
);
|
|
|
|
t.true(
|
|
fs.existsSync(cachedFeatureFlags),
|
|
"Feature flag cached file should exist after getting feature flags",
|
|
);
|
|
|
|
const actualFeatureEnablement = JSON.parse(
|
|
fs.readFileSync(cachedFeatureFlags, "utf8"),
|
|
);
|
|
t.deepEqual(actualFeatureEnablement, expectedFeatureEnablement);
|
|
|
|
// now test that we actually use the feature flag cache instead of the server
|
|
actualFeatureEnablement[Feature.CliConfigFileEnabled] = false;
|
|
fs.writeFileSync(
|
|
cachedFeatureFlags,
|
|
JSON.stringify(actualFeatureEnablement),
|
|
);
|
|
|
|
// delete the in memory cache so that we are forced to use the cached file
|
|
(features as any).gitHubFeatureFlags.cachedApiResponse = undefined;
|
|
|
|
t.false(
|
|
await features.getValue(
|
|
Feature.CliConfigFileEnabled,
|
|
includeCodeQlIfRequired(Feature.CliConfigFileEnabled),
|
|
),
|
|
"Feature flag should be enabled after reading from cached file",
|
|
);
|
|
});
|
|
});
|
|
|
|
test("Environment variable can override feature flag cache", async (t) => {
|
|
await withTmpDir(async (tmpDir) => {
|
|
const features = setUpFeatureFlagTests(tmpDir);
|
|
const expectedFeatureEnablement = initializeFeatures(true);
|
|
mockFeatureFlagApiEndpoint(200, expectedFeatureEnablement);
|
|
|
|
const cachedFeatureFlags = path.join(tmpDir, FEATURE_FLAGS_FILE_NAME);
|
|
t.true(
|
|
await features.getValue(
|
|
Feature.CliConfigFileEnabled,
|
|
includeCodeQlIfRequired(Feature.CliConfigFileEnabled),
|
|
),
|
|
"Feature flag should be enabled initially",
|
|
);
|
|
|
|
t.true(
|
|
fs.existsSync(cachedFeatureFlags),
|
|
"Feature flag cached file should exist after getting feature flags",
|
|
);
|
|
process.env.CODEQL_PASS_CONFIG_TO_CLI = "false";
|
|
|
|
t.false(
|
|
await features.getValue(
|
|
Feature.CliConfigFileEnabled,
|
|
includeCodeQlIfRequired(Feature.CliConfigFileEnabled),
|
|
),
|
|
"Feature flag should be disabled after setting env var",
|
|
);
|
|
});
|
|
});
|
|
|
|
for (const variant of [GitHubVariant.GHAE, GitHubVariant.GHES]) {
|
|
test(`selects CLI from defaults.json on ${GitHubVariant[variant]}`, async (t) => {
|
|
await withTmpDir(async (tmpDir) => {
|
|
const features = setUpFeatureFlagTests(tmpDir);
|
|
|
|
const defaultCliVersion = await features.getDefaultCliVersion(variant);
|
|
t.deepEqual(defaultCliVersion, {
|
|
cliVersion: defaults.cliVersion,
|
|
tagName: defaults.bundleVersion,
|
|
});
|
|
});
|
|
});
|
|
}
|
|
|
|
test("selects CLI v2.20.1 on Dotcom when feature flags enable v2.20.0 and v2.20.1", async (t) => {
|
|
await withTmpDir(async (tmpDir) => {
|
|
const features = setUpFeatureFlagTests(tmpDir);
|
|
const expectedFeatureEnablement = initializeFeatures(true);
|
|
expectedFeatureEnablement["default_codeql_version_2_20_0_enabled"] = true;
|
|
expectedFeatureEnablement["default_codeql_version_2_20_1_enabled"] = true;
|
|
expectedFeatureEnablement["default_codeql_version_2_20_2_enabled"] = false;
|
|
expectedFeatureEnablement["default_codeql_version_2_20_3_enabled"] = false;
|
|
expectedFeatureEnablement["default_codeql_version_2_20_4_enabled"] = false;
|
|
expectedFeatureEnablement["default_codeql_version_2_20_5_enabled"] = false;
|
|
mockFeatureFlagApiEndpoint(200, expectedFeatureEnablement);
|
|
|
|
const defaultCliVersion = await features.getDefaultCliVersion(
|
|
GitHubVariant.DOTCOM,
|
|
);
|
|
t.deepEqual(defaultCliVersion, {
|
|
cliVersion: "2.20.1",
|
|
tagName: "codeql-bundle-v2.20.1",
|
|
toolsFeatureFlagsValid: true,
|
|
});
|
|
});
|
|
});
|
|
|
|
test("includes tag name when feature flags enable version greater than v2.13.4", async (t) => {
|
|
await withTmpDir(async (tmpDir) => {
|
|
const features = setUpFeatureFlagTests(tmpDir);
|
|
const expectedFeatureEnablement = initializeFeatures(true);
|
|
expectedFeatureEnablement["default_codeql_version_2_20_0_enabled"] = true;
|
|
mockFeatureFlagApiEndpoint(200, expectedFeatureEnablement);
|
|
|
|
const defaultCliVersion = await features.getDefaultCliVersion(
|
|
GitHubVariant.DOTCOM,
|
|
);
|
|
t.deepEqual(defaultCliVersion, {
|
|
cliVersion: "2.20.0",
|
|
tagName: "codeql-bundle-v2.20.0",
|
|
toolsFeatureFlagsValid: true,
|
|
});
|
|
});
|
|
});
|
|
|
|
test(`selects CLI from defaults.json on Dotcom when no default version feature flags are enabled`, async (t) => {
|
|
await withTmpDir(async (tmpDir) => {
|
|
const features = setUpFeatureFlagTests(tmpDir);
|
|
const expectedFeatureEnablement = initializeFeatures(true);
|
|
mockFeatureFlagApiEndpoint(200, expectedFeatureEnablement);
|
|
|
|
const defaultCliVersion = await features.getDefaultCliVersion(
|
|
GitHubVariant.DOTCOM,
|
|
);
|
|
t.deepEqual(defaultCliVersion, {
|
|
cliVersion: defaults.cliVersion,
|
|
tagName: defaults.bundleVersion,
|
|
toolsFeatureFlagsValid: false,
|
|
});
|
|
});
|
|
});
|
|
|
|
test(`selects CLI from defaults.json on Dotcom when default version feature flags are unsupported`, async (t) => {
|
|
await withTmpDir(async (tmpDir) => {
|
|
const features = setUpFeatureFlagTests(tmpDir);
|
|
const expectedFeatureEnablement = initializeFeatures(true);
|
|
// Doesn't have a semantically versioned bundle
|
|
expectedFeatureEnablement["default_codeql_version_2_13_3_enabled"] = true;
|
|
mockFeatureFlagApiEndpoint(200, expectedFeatureEnablement);
|
|
|
|
const defaultCliVersion = await features.getDefaultCliVersion(
|
|
GitHubVariant.DOTCOM,
|
|
);
|
|
t.deepEqual(defaultCliVersion, {
|
|
cliVersion: defaults.cliVersion,
|
|
tagName: defaults.bundleVersion,
|
|
toolsFeatureFlagsValid: false,
|
|
});
|
|
});
|
|
});
|
|
|
|
test("ignores invalid version numbers in default version feature flags", async (t) => {
|
|
await withTmpDir(async (tmpDir) => {
|
|
const loggedMessages = [];
|
|
const features = setUpFeatureFlagTests(
|
|
tmpDir,
|
|
getRecordingLogger(loggedMessages),
|
|
);
|
|
const expectedFeatureEnablement = initializeFeatures(true);
|
|
expectedFeatureEnablement["default_codeql_version_2_20_0_enabled"] = true;
|
|
expectedFeatureEnablement["default_codeql_version_2_20_1_enabled"] = true;
|
|
expectedFeatureEnablement["default_codeql_version_2_20_invalid_enabled"] =
|
|
true;
|
|
mockFeatureFlagApiEndpoint(200, expectedFeatureEnablement);
|
|
|
|
const defaultCliVersion = await features.getDefaultCliVersion(
|
|
GitHubVariant.DOTCOM,
|
|
);
|
|
t.deepEqual(defaultCliVersion, {
|
|
cliVersion: "2.20.1",
|
|
tagName: "codeql-bundle-v2.20.1",
|
|
toolsFeatureFlagsValid: true,
|
|
});
|
|
|
|
t.assert(
|
|
loggedMessages.find(
|
|
(v: LoggedMessage) =>
|
|
v.type === "warning" &&
|
|
v.message ===
|
|
"Ignoring feature flag default_codeql_version_2_20_invalid_enabled as it does not specify a valid CodeQL version.",
|
|
) !== undefined,
|
|
);
|
|
});
|
|
});
|
|
|
|
test("feature flags should end with _enabled", async (t) => {
|
|
for (const feature of Object.values(Feature)) {
|
|
t.assert(
|
|
feature.endsWith("_enabled"),
|
|
`${feature} should end with '_enabled'`,
|
|
);
|
|
}
|
|
});
|
|
|
|
function assertAllFeaturesUndefinedInApi(
|
|
t: ExecutionContext<unknown>,
|
|
loggedMessages: LoggedMessage[],
|
|
) {
|
|
for (const feature of Object.keys(featureConfig)) {
|
|
t.assert(
|
|
loggedMessages.find(
|
|
(v) =>
|
|
v.type === "debug" &&
|
|
(v.message as string).includes(feature) &&
|
|
(v.message as string).includes("undefined in API response"),
|
|
) !== undefined,
|
|
);
|
|
}
|
|
}
|
|
|
|
function initializeFeatures(initialValue: boolean) {
|
|
return Object.keys(featureConfig).reduce((features, key) => {
|
|
features[key] = initialValue;
|
|
return features;
|
|
}, {});
|
|
}
|
|
|
|
function setUpFeatureFlagTests(
|
|
tmpDir: string,
|
|
logger = getRunnerLogger(true),
|
|
gitHubVersion = { type: GitHubVariant.DOTCOM } as util.GitHubVersion,
|
|
): FeatureEnablement {
|
|
setupActionsVars(tmpDir, tmpDir);
|
|
|
|
return new Features(gitHubVersion, testRepositoryNwo, tmpDir, logger);
|
|
}
|
|
|
|
function includeCodeQlIfRequired(feature: string) {
|
|
return featureConfig[feature].minimumVersion !== undefined
|
|
? mockCodeQLVersion("9.9.9")
|
|
: undefined;
|
|
}
|