Merge pull request #2235 from github/henrymercer/autobuild-with-direct-tracing

Improve reliability and performance when using the `autobuild` build mode
This commit is contained in:
Henry Mercer
2024-04-16 18:59:43 +01:00
committed by GitHub
47 changed files with 488 additions and 208 deletions

View File

@@ -20,7 +20,7 @@ import { getCodeQL } from "./codeql";
import { Config, getConfig } from "./config-utils";
import { uploadDatabases } from "./database-upload";
import { EnvVar } from "./environment";
import { Features } from "./feature-flags";
import { FeatureEnablement, Features } from "./feature-flags";
import { Language } from "./languages";
import { getActionsLogger, Logger } from "./logging";
import { parseRepositoryNwo } from "./repository";
@@ -140,7 +140,11 @@ function doesGoExtractionOutputExist(config: Config): boolean {
* - We approximate whether manual build steps are present by looking at
* whether any extraction output already exists for Go.
*/
async function runAutobuildIfLegacyGoWorkflow(config: Config, logger: Logger) {
async function runAutobuildIfLegacyGoWorkflow(
config: Config,
features: FeatureEnablement,
logger: Logger,
) {
if (!config.languages.includes(Language.go)) {
return;
}
@@ -177,7 +181,7 @@ async function runAutobuildIfLegacyGoWorkflow(config: Config, logger: Logger) {
logger.debug(
"Running Go autobuild because extraction output (TRAP files) for Go code has not been found.",
);
await runAutobuild(Language.go, config, logger);
await runAutobuild(config, Language.go, features, logger);
}
async function run() {
@@ -211,6 +215,8 @@ async function run() {
);
}
const codeql = await getCodeQL(config.codeQLCmd);
if (hasBadExpectErrorInput()) {
throw new util.ConfigurationError(
"`expect-error` input parameter is for internal use only. It should only be set by codeql-action or a fork.",
@@ -245,13 +251,15 @@ async function run() {
);
await warnIfGoInstalledAfterInit(config, logger);
await runAutobuildIfLegacyGoWorkflow(config, logger);
await runAutobuildIfLegacyGoWorkflow(config, features, logger);
dbCreationTimings = await runFinalize(
outputDir,
threads,
memory,
codeql,
config,
features,
logger,
);
@@ -300,7 +308,6 @@ async function run() {
// Possibly upload the TRAP caches for later re-use
const trapCacheUploadStartTime = performance.now();
const codeql = await getCodeQL(config.codeQLCmd);
didUploadTrapCaches = await uploadTrapCaches(codeql, config, logger);
trapCacheUploadTime = performance.now() - trapCacheUploadStartTime;

View File

@@ -13,7 +13,6 @@ import {
getCodeQL,
} from "./codeql";
import * as configUtils from "./config-utils";
import { BuildMode } from "./config-utils";
import { addDiagnostic, makeDiagnostic } from "./diagnostics";
import { EnvVar } from "./environment";
import { FeatureEnablement, Feature } from "./feature-flags";
@@ -24,6 +23,7 @@ import { ToolsFeature } from "./tools-features";
import { endTracingForCluster } from "./tracer-config";
import { validateSarifFileSchema } from "./upload-lib";
import * as util from "./util";
import { BuildMode } from "./util";
export class CodeQLAnalysisError extends Error {
queriesStatusReport: QueriesStatusReport;
@@ -158,26 +158,7 @@ export async function runExtraction(
) {
await setupCppAutobuild(codeql, logger);
}
try {
await codeql.extractUsingBuildMode(config, language);
} catch (e) {
if (config.buildMode === BuildMode.Autobuild) {
const prefix =
"We were unable to automatically build your code. " +
"Please change the build mode for this language to manual and specify build steps " +
"for your project. For more information, see " +
"https://docs.github.com/en/code-security/code-scanning/troubleshooting-code-scanning/automatic-build-failed.";
const ErrorConstructor =
e instanceof util.ConfigurationError
? util.ConfigurationError
: Error;
throw new ErrorConstructor(
`${prefix} ${util.wrapError(e).message}`,
);
} else {
throw e;
}
}
await codeql.extractUsingBuildMode(config, language);
} else {
await codeql.extractScannedLanguage(config, language);
}
@@ -218,13 +199,12 @@ export function dbIsFinalized(
}
async function finalizeDatabaseCreation(
codeql: CodeQL,
config: configUtils.Config,
threadsFlag: string,
memoryFlag: string,
logger: Logger,
): Promise<DatabaseCreationTimings> {
const codeql = await getCodeQL(config.codeQLCmd);
const extractionStart = performance.now();
await runExtraction(codeql, config, logger);
const extractionTime = performance.now() - extractionStart;
@@ -400,7 +380,9 @@ export async function runFinalize(
outputDir: string,
threadsFlag: string,
memoryFlag: string,
codeql: CodeQL,
config: configUtils.Config,
features: FeatureEnablement,
logger: Logger,
): Promise<DatabaseCreationTimings> {
try {
@@ -413,6 +395,7 @@ export async function runFinalize(
await fs.promises.mkdir(outputDir, { recursive: true });
const timings = await finalizeDatabaseCreation(
codeql,
config,
threadsFlag,
memoryFlag,
@@ -425,7 +408,7 @@ export async function runFinalize(
// However, it will stop tracing for all steps past the codeql-action/analyze
// step.
// Delete variables as specified by the end-tracing script
await endTracingForCluster(config);
await endTracingForCluster(codeql, config, features);
return timings;
}

View File

@@ -10,8 +10,10 @@ import { determineAutobuildLanguages, runAutobuild } from "./autobuild";
import { getCodeQL } from "./codeql";
import { Config, getConfig } from "./config-utils";
import { EnvVar } from "./environment";
import { Features } from "./feature-flags";
import { Language } from "./languages";
import { Logger, getActionsLogger } from "./logging";
import { parseRepositoryNwo } from "./repository";
import {
StatusReportBase,
getActionsStatus,
@@ -23,6 +25,7 @@ import {
checkActionVersion,
checkDiskUsage,
checkGitHubVersionInRange,
getRequiredEnvParam,
initializeEnvironment,
wrapError,
} from "./util";
@@ -88,6 +91,17 @@ async function run() {
checkGitHubVersionInRange(gitHubVersion, logger);
checkActionVersion(getActionVersion(), gitHubVersion);
const repositoryNwo = parseRepositoryNwo(
getRequiredEnvParam("GITHUB_REPOSITORY"),
);
const features = new Features(
gitHubVersion,
repositoryNwo,
getTemporaryDirectory(),
logger,
);
config = await getConfig(getTemporaryDirectory(), logger);
if (config === undefined) {
throw new Error(
@@ -108,7 +122,7 @@ async function run() {
}
for (const language of languages) {
currentLanguage = language;
await runAutobuild(language, config, logger);
await runAutobuild(config, language, features, logger);
}
}
} catch (unwrappedError) {

View File

@@ -4,14 +4,18 @@ import { getTemporaryDirectory, getWorkflowEventName } from "./actions-util";
import { getGitHubVersion } from "./api-client";
import { CodeQL, getCodeQL } from "./codeql";
import * as configUtils from "./config-utils";
import { BuildMode } from "./config-utils";
import { EnvVar } from "./environment";
import { Feature, featureConfig, Features } from "./feature-flags";
import {
Feature,
featureConfig,
FeatureEnablement,
Features,
} from "./feature-flags";
import { isTracedLanguage, Language } from "./languages";
import { Logger } from "./logging";
import { parseRepositoryNwo } from "./repository";
import { ToolsFeature } from "./tools-features";
import { getRequiredEnvParam } from "./util";
import { BuildMode, getRequiredEnvParam } from "./util";
export async function determineAutobuildLanguages(
codeql: CodeQL,
@@ -153,8 +157,9 @@ export async function setupCppAutobuild(codeql: CodeQL, logger: Logger) {
}
export async function runAutobuild(
language: Language,
config: configUtils.Config,
language: Language,
features: FeatureEnablement,
logger: Logger,
) {
logger.startGroup(`Attempting to automatically build ${language} code`);
@@ -162,7 +167,14 @@ export async function runAutobuild(
if (language === Language.cpp) {
await setupCppAutobuild(codeQL, logger);
}
await codeQL.runAutobuild(language, config.debugMode);
if (
config.buildMode &&
(await features.getValue(Feature.AutobuildDirectTracingEnabled, codeQL))
) {
await codeQL.extractUsingBuildMode(config, language);
} else {
await codeQL.runAutobuild(config, language, features);
}
if (language === Language.go) {
core.exportVariable(EnvVar.DID_AUTOBUILD_GOLANG, "true");
}

View File

@@ -482,6 +482,7 @@ const injectedConfigMacro = test.macro({
"",
undefined,
undefined,
createFeatures([]),
getRunnerLogger(true),
);
@@ -695,6 +696,7 @@ test("passes a code scanning config AND qlconfig to the CLI", async (t: Executio
"",
undefined,
"/path/to/qlconfig.yml",
createFeatures([]),
getRunnerLogger(true),
);
@@ -724,6 +726,7 @@ test("does not pass a qlconfig to the CLI when it is undefined", async (t: Execu
"",
undefined,
undefined, // undefined qlconfigFile
createFeatures([]),
getRunnerLogger(true),
);
@@ -907,7 +910,12 @@ test("runTool summarizes autobuilder errors", async (t) => {
sinon.stub(safeWhich, "safeWhich").resolves("");
await t.throwsAsync(
async () => await codeqlObject.runAutobuild(Language.java, false),
async () =>
await codeqlObject.runAutobuild(
stubConfig,
Language.java,
createFeatures([]),
),
{
instanceOf: CommandInvocationError,
message:
@@ -935,7 +943,12 @@ test("runTool truncates long autobuilder errors", async (t) => {
sinon.stub(safeWhich, "safeWhich").resolves("");
await t.throwsAsync(
async () => await codeqlObject.runAutobuild(Language.java, false),
async () =>
await codeqlObject.runAutobuild(
stubConfig,
Language.java,
createFeatures([]),
),
{
instanceOf: CommandInvocationError,
message:

View File

@@ -16,7 +16,7 @@ import {
CommandInvocationError,
wrapCliConfigurationError,
} from "./cli-errors";
import type { Config } from "./config-utils";
import { type Config } from "./config-utils";
import { EnvVar } from "./environment";
import {
CODEQL_VERSION_FINE_GRAINED_PARALLELISM,
@@ -24,12 +24,13 @@ import {
Feature,
FeatureEnablement,
} from "./feature-flags";
import { isTracedLanguage, Language } from "./languages";
import { Language } from "./languages";
import { Logger } from "./logging";
import * as setupCodeql from "./setup-codeql";
import { ToolsFeature, isSupportedToolsFeature } from "./tools-features";
import { shouldEnableIndirectTracing } from "./tracer-config";
import * as util from "./util";
import { wrapError } from "./util";
import { BuildMode, wrapError } from "./util";
type Options = Array<string | number | boolean>;
@@ -81,12 +82,17 @@ export interface CodeQL {
sourceRoot: string,
processName: string | undefined,
qlconfigFile: string | undefined,
features: FeatureEnablement,
logger: Logger,
): Promise<void>;
/**
* Runs the autobuilder for the given language.
*/
runAutobuild(language: Language, enableDebugLogging: boolean): Promise<void>;
runAutobuild(
config: Config,
language: Language,
features: FeatureEnablement,
): Promise<void>;
/**
* Extract code for a scanned language using 'codeql database trace-command'
* and running the language extractor.
@@ -556,12 +562,13 @@ export async function getCodeQLForCmd(
sourceRoot: string,
processName: string | undefined,
qlconfigFile: string | undefined,
features: FeatureEnablement,
logger: Logger,
) {
const extraArgs = config.languages.map(
(language) => `--language=${language}`,
);
if (config.languages.filter((l) => isTracedLanguage(l)).length > 0) {
if (await shouldEnableIndirectTracing(codeql, config, features)) {
extraArgs.push("--begin-tracing");
extraArgs.push(...(await getTrapCachingExtractorConfigArgs(config)));
extraArgs.push(`--trace-process-name=${processName}`);
@@ -625,27 +632,35 @@ export async function getCodeQLForCmd(
{ stdin: externalRepositoryToken },
);
},
async runAutobuild(language: Language, enableDebugLogging: boolean) {
async runAutobuild(
config: Config,
language: Language,
features: FeatureEnablement,
) {
applyAutobuildAzurePipelinesTimeoutFix();
if (
await features.getValue(Feature.AutobuildDirectTracingEnabled, this)
) {
await runTool(cmd, [
"database",
"trace-command",
...(await getTrapCachingExtractorConfigArgsForLang(config, language)),
...getExtractionVerbosityArguments(config.debugMode),
...getExtraOptionsFromEnv(["database", "trace-command"]),
util.getCodeQLDatabasePath(config, language),
]);
return;
}
const autobuildCmd = path.join(
await this.resolveExtractor(language),
"tools",
process.platform === "win32" ? "autobuild.cmd" : "autobuild.sh",
);
// Update JAVA_TOOL_OPTIONS to contain '-Dhttp.keepAlive=false'
// This is because of an issue with Azure pipelines timing out connections after 4 minutes
// and Maven not properly handling closed connections
// Otherwise long build processes will timeout when pulling down Java packages
// https://developercommunity.visualstudio.com/content/problem/292284/maven-hosted-agent-connection-timeout.html
const javaToolOptions = process.env["JAVA_TOOL_OPTIONS"] || "";
process.env["JAVA_TOOL_OPTIONS"] = [
...javaToolOptions.split(/\s+/),
"-Dhttp.keepAlive=false",
"-Dmaven.wagon.http.pool=false",
].join(" ");
// Bump the verbosity of the autobuild command if we're in debug mode
if (enableDebugLogging) {
if (config.debugMode) {
process.env[EnvVar.CLI_VERBOSITY] =
process.env[EnvVar.CLI_VERBOSITY] || EXTRACTION_DEBUG_MODE_VERBOSITY;
}
@@ -678,15 +693,35 @@ export async function getCodeQLForCmd(
]);
},
async extractUsingBuildMode(config: Config, language: Language) {
await runTool(cmd, [
"database",
"trace-command",
"--use-build-mode",
...(await getTrapCachingExtractorConfigArgsForLang(config, language)),
...getExtractionVerbosityArguments(config.debugMode),
...getExtraOptionsFromEnv(["database", "trace-command"]),
util.getCodeQLDatabasePath(config, language),
]);
if (config.buildMode === BuildMode.Autobuild) {
applyAutobuildAzurePipelinesTimeoutFix();
}
try {
await runTool(cmd, [
"database",
"trace-command",
"--use-build-mode",
...(await getTrapCachingExtractorConfigArgsForLang(config, language)),
...getExtractionVerbosityArguments(config.debugMode),
...getExtraOptionsFromEnv(["database", "trace-command"]),
util.getCodeQLDatabasePath(config, language),
]);
} catch (e) {
if (config.buildMode === BuildMode.Autobuild) {
const prefix =
"We were unable to automatically build your code. " +
"Please change the build mode for this language to manual and specify build steps " +
"for your project. For more information, see " +
"https://docs.github.com/en/code-security/code-scanning/troubleshooting-code-scanning/automatic-build-failed.";
const ErrorConstructor =
e instanceof util.ConfigurationError
? util.ConfigurationError
: Error;
throw new ErrorConstructor(`${prefix} ${util.wrapError(e).message}`);
} else {
throw e;
}
}
},
async finalizeDatabase(
databasePath: string,
@@ -1402,3 +1437,19 @@ function getExtractionVerbosityArguments(
? [`--verbosity=${EXTRACTION_DEBUG_MODE_VERBOSITY}`]
: [];
}
/**
* Updates the `JAVA_TOOL_OPTIONS` environment variable to resolve an issue with Azure Pipelines
* timing out connections after 4 minutes and Maven not properly handling closed connections.
*
* Without the fix, long build processes will timeout when pulling down Java packages
* https://developercommunity.visualstudio.com/content/problem/292284/maven-hosted-agent-connection-timeout.html
*/
function applyAutobuildAzurePipelinesTimeoutFix() {
const javaToolOptions = process.env["JAVA_TOOL_OPTIONS"] || "";
process.env["JAVA_TOOL_OPTIONS"] = [
...javaToolOptions.split(/\s+/),
"-Dhttp.keepAlive=false",
"-Dmaven.wagon.http.pool=false",
].join(" ");
}

View File

@@ -14,7 +14,6 @@ import {
setCodeQL,
} from "./codeql";
import * as configUtils from "./config-utils";
import { BuildMode } from "./config-utils";
import { Feature } from "./feature-flags";
import { Language } from "./languages";
import { getRunnerLogger } from "./logging";
@@ -32,6 +31,7 @@ import {
prettyPrintPack,
ConfigurationError,
withTmpDir,
BuildMode,
} from "./util";
setupTests(test);

View File

@@ -17,6 +17,7 @@ import {
GitHubVersion,
prettyPrintPack,
ConfigurationError,
BuildMode,
} from "./util";
// Property names from the user-supplied config file.
@@ -73,12 +74,6 @@ interface IncludeQueryFilter {
include: Record<string, string[] | string>;
}
export enum BuildMode {
None = "none",
Autobuild = "autobuild",
Manual = "manual",
}
/**
* Format of the parsed config file.
*/

View File

@@ -519,6 +519,7 @@ async function run() {
"Runner.Worker.exe",
getOptionalInput("registries"),
apiDetails,
features,
logger,
);
if (tracerConfig !== undefined) {

View File

@@ -7,7 +7,7 @@ import * as safeWhich from "@chrisgavin/safe-which";
import { GitHubApiCombinedDetails, GitHubApiDetails } from "./api-client";
import { CodeQL, setupCodeQL } from "./codeql";
import * as configUtils from "./config-utils";
import { CodeQLDefaultVersionInfo } from "./feature-flags";
import { CodeQLDefaultVersionInfo, FeatureEnablement } from "./feature-flags";
import { Language, isScannedLanguage } from "./languages";
import { Logger } from "./logging";
import { ToolsSource } from "./setup-codeql";
@@ -69,6 +69,7 @@ export async function runInit(
processName: string | undefined,
registriesInput: string | undefined,
apiDetails: GitHubApiCombinedDetails,
features: FeatureEnablement,
logger: Logger,
): Promise<TracerConfig | undefined> {
fs.mkdirSync(config.dbLocation, { recursive: true });
@@ -92,10 +93,11 @@ export async function runInit(
sourceRoot,
processName,
qlconfigFile,
features,
logger,
),
);
return await getCombinedTracerConfig(codeql, config);
return await getCombinedTracerConfig(codeql, config, features);
}
export function printPathFiltersWarning(

View File

@@ -2,7 +2,6 @@ import test from "ava";
import * as sinon from "sinon";
import * as actionsUtil from "./actions-util";
import { BuildMode } from "./config-utils";
import { EnvVar } from "./environment";
import { Language } from "./languages";
import { getRunnerLogger } from "./logging";
@@ -12,7 +11,7 @@ import {
setupActionsVars,
createTestConfig,
} from "./testing-utils";
import { withTmpDir } from "./util";
import { BuildMode, withTmpDir } from "./util";
setupTests(test);

View File

@@ -12,7 +12,7 @@ import {
getRequiredInput,
} from "./actions-util";
import { getAnalysisKey, getApiClient } from "./api-client";
import { BuildMode, Config } from "./config-utils";
import { type Config } from "./config-utils";
import { EnvVar } from "./environment";
import { Logger } from "./logging";
import {
@@ -24,6 +24,7 @@ import {
GITHUB_DOTCOM_URL,
DiskUsage,
assertNever,
BuildMode,
} from "./util";
export enum ActionName {

View File

@@ -6,6 +6,7 @@ import test from "ava";
import * as configUtils from "./config-utils";
import { Language } from "./languages";
import {
createFeatures,
createTestConfig,
mockCodeQLVersion,
setupTests,
@@ -29,7 +30,11 @@ test("getCombinedTracerConfig - return undefined when no languages are traced la
// No traced languages
config.languages = [Language.javascript, Language.python];
t.deepEqual(
await getCombinedTracerConfig(mockCodeQLVersion("1.0.0"), config),
await getCombinedTracerConfig(
mockCodeQLVersion("1.0.0"),
config,
createFeatures([]),
),
undefined,
);
});
@@ -67,6 +72,7 @@ test("getCombinedTracerConfig - with start-tracing.json environment file", async
const result = await getCombinedTracerConfig(
mockCodeQLVersion("1.0.0"),
config,
createFeatures([]),
);
t.notDeepEqual(result, undefined);
@@ -127,6 +133,7 @@ test("getCombinedTracerConfig - with SetsCodeqlRunnerEnvVar feature enabled in C
const result = await getCombinedTracerConfig(
mockCodeQLVersion("1.0.0", { setsCodeqlRunnerEnvVar: true }),
config,
createFeatures([]),
);
t.notDeepEqual(result, undefined);

View File

@@ -1,20 +1,39 @@
import * as fs from "fs";
import * as path from "path";
import { CodeQL } from "./codeql";
import * as configUtils from "./config-utils";
import { type CodeQL } from "./codeql";
import { type Config } from "./config-utils";
import { Feature, FeatureEnablement } from "./feature-flags";
import { isTracedLanguage } from "./languages";
import { ToolsFeature } from "./tools-features";
import { BuildMode } from "./util";
export type TracerConfig = {
env: { [key: string]: string };
};
export async function shouldEnableIndirectTracing(
codeql: CodeQL,
config: Config,
features: FeatureEnablement,
): Promise<boolean> {
return (
(!config.buildMode ||
config.buildMode === BuildMode.Manual ||
!(await features.getValue(
Feature.AutobuildDirectTracingEnabled,
codeql,
))) &&
config.languages.some((l) => isTracedLanguage(l))
);
}
export async function endTracingForCluster(
config: configUtils.Config,
codeql: CodeQL,
config: Config,
features: FeatureEnablement,
): Promise<void> {
// If there are no traced languages, we don't need to do anything.
if (!config.languages.some((l) => isTracedLanguage(l))) return;
if (!(await shouldEnableIndirectTracing(codeql, config, features))) return;
const envVariablesFile = path.resolve(
config.dbLocation,
@@ -44,7 +63,7 @@ export async function endTracingForCluster(
}
export async function getTracerConfigForCluster(
config: configUtils.Config,
config: Config,
): Promise<TracerConfig> {
const tracingEnvVariables = JSON.parse(
fs.readFileSync(
@@ -62,13 +81,11 @@ export async function getTracerConfigForCluster(
export async function getCombinedTracerConfig(
codeql: CodeQL,
config: configUtils.Config,
config: Config,
features: FeatureEnablement,
): Promise<TracerConfig | undefined> {
// Abort if there are no traced languages as there's nothing to do
const tracedLanguages = config.languages.filter((l) => isTracedLanguage(l));
if (tracedLanguages.length === 0) {
if (!(await shouldEnableIndirectTracing(codeql, config, features)))
return undefined;
}
const mainTracerConfig = await getTracerConfigForCluster(config);

View File

@@ -1082,3 +1082,18 @@ export function checkActionVersion(
}
}
}
/**
* Supported build modes.
*
* These specify whether the CodeQL database should be created by tracing a build, and if so, how
* this build will be invoked.
*/
export enum BuildMode {
/** The database will be created without building the source root. */
None = "none",
/** The database will be created by attempting to automatically build the source root. */
Autobuild = "autobuild",
/** The database will be created by building the source root using manually specified build steps. */
Manual = "manual",
}