github/codeql-action
1
0
Fork 0
mirror of https://github.com/github/codeql-action.git synced 2025-12-30 11:10:22 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into henrymercer/delete-runner-part-2

Browse Source
This commit is contained in:
Henry Mercer
2022-11-15 19:35:30 +00:00
parent 9df773d1a3 f47c93c5fd
commit 0dea34e91c
70 changed files with 743 additions and 4870 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/actions-util.ts
View File

@@ -513,7 +513,11 @@ export async function getRef(): Promise<string> {
);
}
const ref = refInput || getRequiredEnvParam("GITHUB_REF");
// Workaround for a limitation of Actions dynamic workflows not setting
// the GITHUB_REF in some cases
const maybeCSRef = process.env["CODE_SCANNING_REF"];
const ref = refInput || maybeCSRef || getRequiredEnvParam("GITHUB_REF");
const sha = shaInput || getRequiredEnvParam("GITHUB_SHA");
// If the ref is a user-provided input, we have to skip logic

42
src/analyze-action.ts
View File

@@ -19,7 +19,7 @@ import { runAutobuild } from "./autobuild";
import { getCodeQL } from "./codeql";
import { Config, getConfig } from "./config-utils";
import { uploadDatabases } from "./database-upload";
import { FeatureEnablement, Features } from "./feature-flags";
import { Features } from "./feature-flags";
import { Language } from "./languages";
import { getActionsLogger, Logger } from "./logging";
import { parseRepositoryNwo } from "./repository";
@@ -127,11 +127,9 @@ function doesGoExtractionOutputExist(config: Config): boolean {
}
/**
* When Go extraction reconciliation is enabled, either via the feature
* or an environment variable, we will attempt to autobuild Go to preserve
* compatibility for users who have set up Go using a legacy scanning style
* CodeQL workflow, i.e. one without an autobuild step or manual build
* steps.
* We attempt to autobuild Go to preserve compatibility for users who have
* set up Go using a legacy scanning style CodeQL workflow, i.e. one without
* an autobuild step or manual build steps.
*
* - We detect whether an autobuild step is present by checking the
* `util.DID_AUTOBUILD_GO_ENV_VAR_NAME` environment variable, which is set
@@ -139,33 +137,26 @@ function doesGoExtractionOutputExist(config: Config): boolean {
* - We approximate whether manual build steps are present by looking at
* whether any extraction output already exists for Go.
*/
async function runAutobuildIfLegacyGoWorkflow(
config: Config,
featureEnablement: FeatureEnablement,
logger: Logger
) {
async function runAutobuildIfLegacyGoWorkflow(config: Config, logger: Logger) {
if (!config.languages.includes(Language.go)) {
return;
}
if (!(await util.isGoExtractionReconciliationEnabled(featureEnablement))) {
logger.debug(
"Won't run Go autobuild since Go extraction reconciliation is not enabled."
);
return;
}
if (process.env[util.DID_AUTOBUILD_GO_ENV_VAR_NAME] === "true") {
// This log line is info level while Go extraction reconciliation is in beta.
// We will make it debug level once Go extraction reconciliation is GA.
logger.info("Won't run Go autobuild since it has already been run.");
logger.debug("Won't run Go autobuild since it has already been run.");
return;
}
// This captures whether a user has added manual build steps for Go
if (doesGoExtractionOutputExist(config)) {
// This log line is info level while Go extraction reconciliation is in beta.
// We will make it debug level once Go extraction reconciliation is GA.
logger.info(
logger.debug(
"Won't run Go autobuild since at least one file of Go code has already been extracted."
);
// If the user has run the manual build step, and has set the `CODEQL_EXTRACTOR_GO_BUILD_TRACING`
// variable, we suggest they remove it from their workflow.
if ("CODEQL_EXTRACTOR_GO_BUILD_TRACING" in process.env) {
logger.warning(
`The CODEQL_EXTRACTOR_GO_BUILD_TRACING environment variable has no effect on workflows with manual build steps, so we recommend that you remove it from your workflow.`
);
}
return;
}
await runAutobuild(Language.go, config, logger);
@@ -228,15 +219,14 @@ async function run() {
const features = new Features(gitHubVersion, repositoryNwo, logger);
await runAutobuildIfLegacyGoWorkflow(config, features, logger);
await runAutobuildIfLegacyGoWorkflow(config, logger);
dbCreationTimings = await runFinalize(
outputDir,
threads,
memory,
config,
logger,
features
logger
);
if (actionsUtil.getRequiredInput("skip-queries") !== "true") {

32
src/analyze.ts
View File

@@ -49,6 +49,8 @@ export interface QueriesStatusReport {
analyze_builtin_queries_python_duration_ms?: number;
/** Time taken in ms to run builtin queries for ruby (or undefined if this language was not analyzed). */
analyze_builtin_queries_ruby_duration_ms?: number;
/** Time taken in ms to run builtin queries for swift (or undefined if this language was not analyzed). */
analyze_builtin_queries_swift_duration_ms?: number;
/** Time taken in ms to run custom queries for cpp (or undefined if this language was not analyzed). */
analyze_custom_queries_cpp_duration_ms?: number;
/** Time taken in ms to run custom queries for csharp (or undefined if this language was not analyzed). */
@@ -63,6 +65,8 @@ export interface QueriesStatusReport {
analyze_custom_queries_python_duration_ms?: number;
/** Time taken in ms to run custom queries for ruby (or undefined if this language was not analyzed). */
analyze_custom_queries_ruby_duration_ms?: number;
/** Time taken in ms to run custom queries for swift (or undefined if this language was not analyzed). */
analyze_custom_queries_swift_duration_ms?: number;
/** Time taken in ms to interpret results for cpp (or undefined if this language was not analyzed). */
interpret_results_cpp_duration_ms?: number;
/** Time taken in ms to interpret results for csharp (or undefined if this language was not analyzed). */
@@ -77,6 +81,8 @@ export interface QueriesStatusReport {
interpret_results_python_duration_ms?: number;
/** Time taken in ms to interpret results for ruby (or undefined if this language was not analyzed). */
interpret_results_ruby_duration_ms?: number;
/** Time taken in ms to interpret results for swift (or undefined if this language was not analyzed). */
interpret_results_swift_duration_ms?: number;
/** Name of language that errored during analysis (or undefined if no language failed). */
analyze_failure_language?: string;
}
@@ -121,8 +127,7 @@ async function setupPythonExtractor(logger: Logger) {
export async function createdDBForScannedLanguages(
codeql: CodeQL,
config: configUtils.Config,
logger: Logger,
featureEnablement: FeatureEnablement
logger: Logger
) {
// Insert the LGTM_INDEX_X env vars at this point so they are set when
// we extract any scanned languages.
@@ -130,11 +135,7 @@ export async function createdDBForScannedLanguages(
for (const language of config.languages) {
if (
isScannedLanguage(
language,
await util.isGoExtractionReconciliationEnabled(featureEnablement),
logger
) &&
isScannedLanguage(language) &&
!dbIsFinalized(config, language, logger)
) {
logger.startGroup(`Extracting ${language}`);
@@ -172,13 +173,12 @@ async function finalizeDatabaseCreation(
config: configUtils.Config,
threadsFlag: string,
memoryFlag: string,
logger: Logger,
featureEnablement: FeatureEnablement
logger: Logger
): Promise<DatabaseCreationTimings> {
const codeql = await getCodeQL(config.codeQLCmd);
const extractionStart = performance.now();
await createdDBForScannedLanguages(codeql, config, logger, featureEnablement);
await createdDBForScannedLanguages(codeql, config, logger);
const extractionTime = performance.now() - extractionStart;
const trapImportStart = performance.now();
@@ -500,8 +500,7 @@ export async function runFinalize(
threadsFlag: string,
memoryFlag: string,
config: configUtils.Config,
logger: Logger,
featureEnablement: FeatureEnablement
logger: Logger
): Promise<DatabaseCreationTimings> {
try {
await del(outputDir, { force: true });
@@ -516,8 +515,7 @@ export async function runFinalize(
config,
threadsFlag,
memoryFlag,
logger,
featureEnablement
logger
);
const codeql = await getCodeQL(config.codeQLCmd);
@@ -528,11 +526,7 @@ export async function runFinalize(
// step.
if (await util.codeQlVersionAbove(codeql, CODEQL_VERSION_NEW_TRACING)) {
// Delete variables as specified by the end-tracing script
await endTracingForCluster(
config,
await util.isGoExtractionReconciliationEnabled(featureEnablement),
logger
);
await endTracingForCluster(config);
} else {
// Delete the tracer config env var to avoid tracing ourselves
delete process.env[sharedEnv.ODASA_TRACER_CONFIGURATION];

11
src/autobuild-action.ts
View File

@@ -11,15 +11,12 @@ import {
import { getGitHubVersion } from "./api-client";
import { determineAutobuildLanguages, runAutobuild } from "./autobuild";
import * as configUtils from "./config-utils";
import { Features } from "./feature-flags";
import { Language } from "./languages";
import { getActionsLogger } from "./logging";
import { parseRepositoryNwo } from "./repository";
import {
DID_AUTOBUILD_GO_ENV_VAR_NAME,
checkActionVersion,
checkGitHubVersionInRange,
getRequiredEnvParam,
initializeEnvironment,
} from "./util";
@@ -75,12 +72,6 @@ async function run() {
const gitHubVersion = await getGitHubVersion();
checkGitHubVersionInRange(gitHubVersion, logger);
const features = new Features(
gitHubVersion,
parseRepositoryNwo(getRequiredEnvParam("GITHUB_REPOSITORY")),
logger
);
const config = await configUtils.getConfig(getTemporaryDirectory(), logger);
if (config === undefined) {
throw new Error(
@@ -88,7 +79,7 @@ async function run() {
);
}
languages = await determineAutobuildLanguages(config, features, logger);
languages = await determineAutobuildLanguages(config, logger);
if (languages !== undefined) {
const workingDirectory = getOptionalInput("working-directory");
if (workingDirectory) {

7
src/autobuild.ts
View File

@@ -1,23 +1,18 @@
import { getCodeQL } from "./codeql";
import * as configUtils from "./config-utils";
import { FeatureEnablement } from "./feature-flags";
import { Language, isTracedLanguage } from "./languages";
import { Logger } from "./logging";
import * as util from "./util";
export async function determineAutobuildLanguages(
config: configUtils.Config,
featureEnablement: FeatureEnablement,
logger: Logger
): Promise<Language[] | undefined> {
const isGoExtractionReconciliationEnabled =
await util.isGoExtractionReconciliationEnabled(featureEnablement);
// Attempt to find a language to autobuild
// We want pick the dominant language in the repo from the ones we're able to build
// The languages are sorted in order specified by user or by lines of code if we got
// them from the GitHub API, so try to build the first language on the list.
const autobuildLanguages = config.languages.filter((l) =>
isTracedLanguage(l, isGoExtractionReconciliationEnabled, logger)
isTracedLanguage(l)
);
if (!autobuildLanguages) {

21
src/codeql.ts
View File

@@ -799,19 +799,12 @@ async function getCodeQLForCmd(
config: Config,
sourceRoot: string,
processName: string | undefined,
featureEnablement: FeatureEnablement,
logger: Logger
featureEnablement: FeatureEnablement
) {
const extraArgs = config.languages.map(
(language) => `--language=${language}`
);
const isGoExtractionReconciliationEnabled =
await util.isGoExtractionReconciliationEnabled(featureEnablement);
if (
config.languages.filter((l) =>
isTracedLanguage(l, isGoExtractionReconciliationEnabled, logger)
).length > 0
) {
if (config.languages.filter((l) => isTracedLanguage(l)).length > 0) {
extraArgs.push("--begin-tracing");
extraArgs.push(...(await getTrapCachingExtractorConfigArgs(config)));
extraArgs.push(`--trace-process-name=${processName}`);
@@ -824,11 +817,7 @@ async function getCodeQLForCmd(
CODEQL_VERSION_LUA_TRACER_CONFIG
)) &&
config.languages.includes(Language.go) &&
isTracedLanguage(
Language.go,
isGoExtractionReconciliationEnabled,
logger
) &&
isTracedLanguage(Language.go) &&
process.platform === "win32" &&
!(await util.codeQlVersionAbove(
this,
@@ -861,8 +850,12 @@ async function getCodeQLForCmd(
async runAutobuild(language: Language) {
const cmdName =
process.platform === "win32" ? "autobuild.cmd" : "autobuild.sh";
// The autobuilder for Swift is located in the experimental/ directory.
const possibleExperimentalDir =
language === Language.swift ? "experimental" : "";
const autobuildCmd = path.join(
path.dirname(cmd),
possibleExperimentalDir,
language,
"tools",
cmdName

1
src/count-loc.ts
View File

@@ -13,6 +13,7 @@ const linguistToMetrics: Record<string, Language> = {
javascript: Language.javascript,
python: Language.python,
ruby: Language.ruby,
swift: Language.swift,
typescript: Language.javascript,
};

7
src/init.ts
View File

@@ -117,12 +117,7 @@ export async function runInit(
} catch (e) {
throw processError(e);
}
return await getCombinedTracerConfig(
config,
codeql,
await util.isGoExtractionReconciliationEnabled(featureEnablement),
logger
);
return await getCombinedTracerConfig(config, codeql);
}
/**

55
src/languages.test.ts
View File

@@ -6,7 +6,6 @@ import {
isTracedLanguage,
parseLanguage,
} from "./languages";
import { getRunnerLogger } from "./logging";
import { setupTests } from "./testing-utils";
setupTests(test);
@@ -32,40 +31,26 @@ test("parseLanguage", async (t) => {
t.deepEqual(parseLanguage(""), undefined);
});
for (const isReconciliationOn of [false, true]) {
test(`isTracedLanguage (go reconciliation ${
isReconciliationOn ? "enabled" : "disabled"
})`, async (t) => {
const logger = getRunnerLogger(true);
test("isTracedLanguage", async (t) => {
t.true(isTracedLanguage(Language.cpp));
t.true(isTracedLanguage(Language.csharp));
t.true(isTracedLanguage(Language.go));
t.true(isTracedLanguage(Language.java));
t.true(isTracedLanguage(Language.swift));
t.true(isTracedLanguage(Language.cpp, isReconciliationOn, logger));
t.true(isTracedLanguage(Language.java, isReconciliationOn, logger));
t.true(isTracedLanguage(Language.csharp, isReconciliationOn, logger));
t.false(isTracedLanguage(Language.javascript));
t.false(isTracedLanguage(Language.python));
t.false(isTracedLanguage(Language.ruby));
});
t.is(
isTracedLanguage(Language.go, isReconciliationOn, logger),
isReconciliationOn
);
test("isScannedLanguage", async (t) => {
t.false(isScannedLanguage(Language.cpp));
t.false(isScannedLanguage(Language.csharp));
t.false(isScannedLanguage(Language.go));
t.false(isScannedLanguage(Language.java));
t.false(isScannedLanguage(Language.swift));
t.false(isTracedLanguage(Language.javascript, isReconciliationOn, logger));
t.false(isTracedLanguage(Language.python, isReconciliationOn, logger));
});
test(`isScannedLanguage (go reconciliation ${
isReconciliationOn ? "enabled" : "disabled"
}`, async (t) => {
const logger = getRunnerLogger(true);
t.false(isScannedLanguage(Language.cpp, isReconciliationOn, logger));
t.false(isScannedLanguage(Language.java, isReconciliationOn, logger));
t.false(isScannedLanguage(Language.csharp, isReconciliationOn, logger));
t.is(
isScannedLanguage(Language.go, isReconciliationOn, logger),
!isReconciliationOn
);
t.true(isScannedLanguage(Language.javascript, isReconciliationOn, logger));
t.true(isScannedLanguage(Language.python, isReconciliationOn, logger));
});
}
t.true(isScannedLanguage(Language.javascript));
t.true(isScannedLanguage(Language.python));
t.true(isScannedLanguage(Language.ruby));
});

46
src/languages.ts
View File

@@ -1,7 +1,3 @@
import * as core from "@actions/core";
import { Logger } from "./logging";
// All the languages supported by CodeQL
export enum Language {
csharp = "csharp",
@@ -40,38 +36,16 @@ export function parseLanguage(language: string): Language | undefined {
return undefined;
}
export function isTracedLanguage(
language: Language,
isGoExtractionReconciliationEnabled: boolean,
logger: Logger
): boolean {
if (process.env["CODEQL_EXTRACTOR_GO_BUILD_TRACING"] === "true") {
logger.warning(
"The CODEQL_EXTRACTOR_GO_BUILD_TRACING environment variable was set to 'true', but it must " +
"be 'on' to enable Go build tracing. Setting it to 'on'."
);
process.env["CODEQL_EXTRACTOR_GO_BUILD_TRACING"] = "on";
core.exportVariable("CODEQL_EXTRACTOR_GO_BUILD_TRACING", "on");
}
const shouldTraceGo =
process.env["CODEQL_EXTRACTOR_GO_BUILD_TRACING"] === "on" ||
isGoExtractionReconciliationEnabled;
return (
["cpp", "java", "csharp", "swift"].includes(language) ||
(shouldTraceGo && language === Language.go)
);
export function isTracedLanguage(language: Language): boolean {
return [
Language.cpp,
Language.csharp,
Language.go,
Language.java,
Language.swift,
].includes(language);
}
export function isScannedLanguage(
language: Language,
isGoExtractionReconciliationEnabled: boolean,
logger: Logger
): boolean {
return !isTracedLanguage(
language,
isGoExtractionReconciliationEnabled,
logger
);
export function isScannedLanguage(language: Language): boolean {
return !isTracedLanguage(language);
}

18
src/tracer-config.test.ts
View File

@@ -6,7 +6,6 @@ import test from "ava";
import { setCodeQL } from "./codeql";
import * as configUtils from "./config-utils";
import { Language } from "./languages";
import { getRunnerLogger } from "./logging";
import { setupTests } from "./testing-utils";
import {
concatTracerConfigs,
@@ -330,15 +329,7 @@ test("getCombinedTracerConfig - return undefined when no languages are traced la
},
});
t.deepEqual(
await getCombinedTracerConfig(
config,
codeQL,
false, // Disable Go extraction reconciliation
getRunnerLogger(true)
),
undefined
);
t.deepEqual(await getCombinedTracerConfig(config, codeQL), undefined);
});
});
@@ -368,12 +359,7 @@ test("getCombinedTracerConfig - valid spec file", async (t) => {
},
});
const result = await getCombinedTracerConfig(
config,
codeQL,
false, // Disable Go extraction reconciliation
getRunnerLogger(true)
);
const result = await getCombinedTracerConfig(config, codeQL);
t.notDeepEqual(result, undefined);
const expectedEnv = {

20
src/tracer-config.ts
View File

@@ -4,7 +4,6 @@ import * as path from "path";
import { CodeQL, CODEQL_VERSION_NEW_TRACING } from "./codeql";
import * as configUtils from "./config-utils";
import { Language, isTracedLanguage } from "./languages";
import { Logger } from "./logging";
import * as util from "./util";
import { codeQlVersionAbove } from "./util";
@@ -22,17 +21,10 @@ const CRITICAL_TRACER_VARS = new Set([
]);
export async function endTracingForCluster(
config: configUtils.Config,
isGoExtractionReconciliationEnabled: boolean,
logger: Logger
config: configUtils.Config
): Promise<void> {
// If there are no traced languages, we don't need to do anything.
if (
!config.languages.some((l) =>
isTracedLanguage(l, isGoExtractionReconciliationEnabled, logger)
)
)
return;
if (!config.languages.some((l) => isTracedLanguage(l))) return;
const envVariablesFile = path.resolve(
config.dbLocation,
@@ -231,14 +223,10 @@ export function concatTracerConfigs(
export async function getCombinedTracerConfig(
config: configUtils.Config,
codeql: CodeQL,
isGoExtractionReconciliationEnabled: boolean,
logger: Logger
codeql: CodeQL
): Promise<TracerConfig | undefined> {
// Abort if there are no traced languages as there's nothing to do
const tracedLanguages = config.languages.filter((l) =>
isTracedLanguage(l, isGoExtractionReconciliationEnabled, logger)
);
const tracedLanguages = config.languages.filter((l) => isTracedLanguage(l));
if (tracedLanguages.length === 0) {
return undefined;
}

8
src/util.ts
View File

@@ -725,14 +725,6 @@ export function listFolder(dir: string): string[] {
return files;
}
export async function isGoExtractionReconciliationEnabled(
featureEnablement: FeatureEnablement
): Promise<boolean> {
return await featureEnablement.getValue(
Feature.GolangExtractionReconciliationEnabled
);
}
/**
* Get the size a folder in bytes. This will log any filesystem errors
* as a warning and then return undefined.