github/codeql-action
1
0
Fork 0
mirror of https://github.com/github/codeql-action.git synced 2026-01-02 20:50:05 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: use http agent that respects environment proxy settings when streaming

Browse Source
This commit is contained in:
Fotis Koutoulakis (@NlightNFotis)
2024-12-04 18:30:46 +00:00
parent 3096afedf9
commit 0dc76a996d
16 changed files with 193 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
lib/setup-codeql.js generated
View File

@@ -417,7 +417,7 @@ const downloadCodeQL = async function (codeqlURL, maybeBundleVersion, maybeCliVe
const extractedBundlePath = extractToToolcache
? toolcacheInfo.path
: getTempExtractionDir(tempDir);
let statusReport = await (0, tools_download_1.downloadAndExtract)(codeqlURL, extractedBundlePath, authorization, { "User-Agent": "CodeQL Action", ...headers }, tarVersion, features, logger);
let statusReport = await (0, tools_download_1.downloadAndExtract)(codeqlURL, extractedBundlePath, authorization, { "User-Agent": "CodeQL Action", ...headers }, tarVersion, logger);
if (!toolcacheInfo) {
logger.debug("Could not cache CodeQL tools because we could not determine the bundle version from the " +
`URL ${codeqlURL}.`);

2
lib/setup-codeql.js.map
View File

File diff suppressed because one or more lines are too long

40
lib/tools-download.js generated
View File

@@ -41,10 +41,11 @@ const fs = __importStar(require("fs"));
const os = __importStar(require("os"));
const path = __importStar(require("path"));
const perf_hooks_1 = require("perf_hooks");
const core = __importStar(require("@actions/core"));
const http_client_1 = require("@actions/http-client");
const toolcache = __importStar(require("@actions/tool-cache"));
const follow_redirects_1 = require("follow-redirects");
const semver = __importStar(require("semver"));
const feature_flags_1 = require("./feature-flags");
const logging_1 = require("./logging");
const tar = __importStar(require("./tar"));
const util_1 = require("./util");
@@ -72,23 +73,26 @@ function makeStreamedToolsDownloadDurations(combinedDurationMs) {
streamExtraction: true,
};
}
async function downloadAndExtract(codeqlURL, dest, authorization, headers, tarVersion, features, logger) {
async function downloadAndExtract(codeqlURL, dest, authorization, headers, tarVersion, logger) {
logger.info(`Downloading CodeQL tools from ${codeqlURL} . This may take a while.`);
const compressionMethod = tar.inferCompressionMethod(codeqlURL);
// TODO: Re-enable streaming when we have a more reliable way to respect proxy settings.
if ((await features.getValue(feature_flags_1.Feature.ZstdBundleStreamingExtraction)) &&
compressionMethod === "zstd" &&
process.platform === "linux") {
logger.info(`Streaming the extraction of the CodeQL bundle.`);
const toolsInstallStart = perf_hooks_1.performance.now();
await downloadAndExtractZstdWithStreaming(codeqlURL, dest, authorization, headers, tarVersion, logger);
const combinedDurationMs = Math.round(perf_hooks_1.performance.now() - toolsInstallStart);
logger.info(`Finished downloading and extracting CodeQL bundle to ${dest} (${(0, logging_1.formatDuration)(combinedDurationMs)}).`);
return {
compressionMethod,
toolsUrl: sanitizeUrlForStatusReport(codeqlURL),
...makeStreamedToolsDownloadDurations(combinedDurationMs),
};
try {
if (compressionMethod === "zstd" && process.platform === "linux") {
logger.info(`Streaming the extraction of the CodeQL bundle.`);
const toolsInstallStart = perf_hooks_1.performance.now();
await downloadAndExtractZstdWithStreaming(codeqlURL, dest, authorization, headers, tarVersion, logger);
const combinedDurationMs = Math.round(perf_hooks_1.performance.now() - toolsInstallStart);
logger.info(`Finished downloading and extracting CodeQL bundle to ${dest} (${(0, logging_1.formatDuration)(combinedDurationMs)}).`);
return {
compressionMethod,
toolsUrl: sanitizeUrlForStatusReport(codeqlURL),
...makeStreamedToolsDownloadDurations(combinedDurationMs),
};
}
}
catch (e) {
core.warning(`Failed to download and extract CodeQL bundle using streaming. Falling back to downloading the bundle before extracting.`);
core.warning((0, util_1.getErrorMessage)(e));
}
const toolsDownloadStart = perf_hooks_1.performance.now();
const archivedBundlePath = await toolcache.downloadTool(codeqlURL, undefined, authorization, headers);
@@ -114,12 +118,16 @@ async function downloadAndExtract(codeqlURL, dest, authorization, headers, tarVe
async function downloadAndExtractZstdWithStreaming(codeqlURL, dest, authorization, headers, tarVersion, logger) {
// Ensure destination exists
fs.mkdirSync(dest, { recursive: true });
// Get HTTP Agent to use (respects proxy settings).
const agent = new http_client_1.HttpClient().getAgent(codeqlURL);
// Add User-Agent header and Authorization header if provided.
headers = Object.assign({ "User-Agent": "CodeQL Action" }, authorization ? { authorization } : {}, headers);
const response = await new Promise((resolve) => follow_redirects_1.https.get(codeqlURL, {
headers,
// Increase the high water mark to improve performance.
highWaterMark: exports.STREAMING_HIGH_WATERMARK_BYTES,
// Use the agent to respect proxy settings.
agent,
}, (r) => resolve(r)));
if (response.statusCode !== 200) {
throw new Error(`Failed to download CodeQL bundle from ${codeqlURL}. HTTP status code: ${response.statusCode}.`);

2
lib/tools-download.js.map
View File

@@ -1 +1 @@
{"version":3,"file":"tools-download.js","sourceRoot":"","sources":["../src/tools-download.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAiFA,gDA+FC;AAyCD,sDAOC;AAED,4DAOC;AAzOD,uCAAyB;AAEzB,uCAAyB;AACzB,2CAA6B;AAC7B,2CAAyC;AAEzC,+DAAiD;AACjD,uDAAyC;AACzC,+CAAiC;AAEjC,mDAA6D;AAC7D,uCAAmD;AACnD,2CAA6B;AAC7B,iCAA0D;AAE1D;;GAEG;AACU,QAAA,8BAA8B,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,QAAQ;AAEvE;;GAEG;AACH,MAAM,mBAAmB,GAAG,QAAQ,CAAC;AAarC,SAAS,uCAAuC,CAC9C,kBAA0B,EAC1B,oBAA4B;IAE5B,OAAO;QACL,kBAAkB,EAAE,kBAAkB,GAAG,oBAAoB;QAC7D,kBAAkB;QAClB,oBAAoB;QACpB,gBAAgB,EAAE,KAAK;KACxB,CAAC;AACJ,CAAC;AAaD,SAAS,kCAAkC,CACzC,kBAA0B;IAE1B,OAAO;QACL,kBAAkB;QAClB,kBAAkB,EAAE,SAAS;QAC7B,oBAAoB,EAAE,SAAS;QAC/B,gBAAgB,EAAE,IAAI;KACvB,CAAC;AACJ,CAAC;AAaM,KAAK,UAAU,kBAAkB,CACtC,SAAiB,EACjB,IAAY,EACZ,aAAiC,EACjC,OAA4B,EAC5B,UAAsC,EACtC,QAA2B,EAC3B,MAAc;IAEd,MAAM,CAAC,IAAI,CACT,iCAAiC,SAAS,2BAA2B,CACtE,CAAC;IAEF,MAAM,iBAAiB,GAAG,GAAG,CAAC,sBAAsB,CAAC,SAAS,CAAC,CAAC;IAEhE,wFAAwF;IAExF,IACE,CAAC,MAAM,QAAQ,CAAC,QAAQ,CAAC,uBAAO,CAAC,6BAA6B,CAAC,CAAC;QAChE,iBAAiB,KAAK,MAAM;QAC5B,OAAO,CAAC,QAAQ,KAAK,OAAO,EAC5B,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,gDAAgD,CAAC,CAAC;QAE9D,MAAM,iBAAiB,GAAG,wBAAW,CAAC,GAAG,EAAE,CAAC;QAC5C,MAAM,mCAAmC,CACvC,SAAS,EACT,IAAI,EACJ,aAAa,EACb,OAAO,EACP,UAAW,EACX,MAAM,CACP,CAAC;QAEF,MAAM,kBAAkB,GAAG,IAAI,CAAC,KAAK,CACnC,wBAAW,CAAC,GAAG,EAAE,GAAG,iBAAiB,CACtC,CAAC;QACF,MAAM,CAAC,IAAI,CACT,wDAAwD,IAAI,KAAK,IAAA,wBAAc,EAC7E,kBAAkB,CACnB,IAAI,CACN,CAAC;QAEF,OAAO;YACL,iBAAiB;YACjB,QAAQ,EAAE,0BAA0B,CAAC,SAAS,CAAC;YAC/C,GAAG,kCAAkC,CAAC,kBAAkB,CAAC;SAC1D,CAAC;IACJ,CAAC;IAED,MAAM,kBAAkB,GAAG,wBAAW,CAAC,GAAG,EAAE,CAAC;IAC7C,MAAM,kBAAkB,GAAG,MAAM,SAAS,CAAC,YAAY,CACrD,SAAS,EACT,SAAS,EACT,aAAa,EACb,OAAO,CACR,CAAC;IACF,MAAM,kBAAkB,GAAG,IAAI,CAAC,KAAK,CAAC,wBAAW,CAAC,GAAG,EAAE,GAAG,kBAAkB,CAAC,CAAC;IAE9E,MAAM,CAAC,IAAI,CACT,yCAAyC,kBAAkB,KAAK,IAAA,wBAAc,EAC5E,kBAAkB,CACnB,IAAI,CACN,CAAC;IAEF,IAAI,oBAA4B,CAAC;IAEjC,IAAI,CAAC;QACH,MAAM,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;QACzC,MAAM,eAAe,GAAG,wBAAW,CAAC,GAAG,EAAE,CAAC;QAC1C,MAAM,GAAG,CAAC,OAAO,CACf,kBAAkB,EAClB,IAAI,EACJ,iBAAiB,EACjB,UAAU,EACV,MAAM,CACP,CAAC;QACF,oBAAoB,GAAG,IAAI,CAAC,KAAK,CAAC,wBAAW,CAAC,GAAG,EAAE,GAAG,eAAe,CAAC,CAAC;QACvE,MAAM,CAAC,IAAI,CACT,wCAAwC,IAAI,KAAK,IAAA,wBAAc,EAC7D,oBAAoB,CACrB,IAAI,CACN,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,MAAM,IAAA,kBAAW,EAAC,kBAAkB,EAAE,uBAAuB,EAAE,MAAM,CAAC,CAAC;IACzE,CAAC;IAED,OAAO;QACL,iBAAiB;QACjB,QAAQ,EAAE,0BAA0B,CAAC,SAAS,CAAC;QAC/C,GAAG,uCAAuC,CACxC,kBAAkB,EAClB,oBAAoB,CACrB;KACF,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,mCAAmC,CAChD,SAAiB,EACjB,IAAY,EACZ,aAAiC,EACjC,OAA4B,EAC5B,UAA0B,EAC1B,MAAc;IAEd,4BAA4B;IAC5B,EAAE,CAAC,SAAS,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAExC,8DAA8D;IAC9D,OAAO,GAAG,MAAM,CAAC,MAAM,CACrB,EAAE,YAAY,EAAE,eAAe,EAAE,EACjC,aAAa,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,EACtC,OAAO,CACR,CAAC;IACF,MAAM,QAAQ,GAAG,MAAM,IAAI,OAAO,CAAkB,CAAC,OAAO,EAAE,EAAE,CAC9D,wBAAK,CAAC,GAAG,CACP,SAAS,EACT;QACE,OAAO;QACP,uDAAuD;QACvD,aAAa,EAAE,sCAA8B;KACjB,EAC9B,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,CAClB,CACF,CAAC;IAEF,IAAI,QAAQ,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;QAChC,MAAM,IAAI,KAAK,CACb,yCAAyC,SAAS,uBAAuB,QAAQ,CAAC,UAAU,GAAG,CAChG,CAAC;IACJ,CAAC;IAED,MAAM,GAAG,CAAC,aAAa,CAAC,QAAQ,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;AAC9D,CAAC;AAED,8FAA8F;AAC9F,SAAgB,qBAAqB,CAAC,OAAe;IACnD,OAAO,IAAI,CAAC,IAAI,CACd,IAAA,0BAAmB,EAAC,mBAAmB,CAAC,EACxC,mBAAmB,EACnB,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,OAAO,EAChC,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,CAChB,CAAC;AACJ,CAAC;AAED,SAAgB,wBAAwB,CACtC,aAAqB,EACrB,MAAc;IAEd,MAAM,cAAc,GAAG,GAAG,aAAa,WAAW,CAAC;IACnD,EAAE,CAAC,aAAa,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;IACrC,MAAM,CAAC,IAAI,CAAC,iCAAiC,cAAc,EAAE,CAAC,CAAC;AACjE,CAAC;AAED,SAAS,0BAA0B,CAAC,GAAW;IAC7C,OAAO,CAAC,sBAAsB,EAAE,kCAAkC,CAAC,CAAC,IAAI,CACtE,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,sBAAsB,IAAI,qBAAqB,CAAC,CAC1E;QACC,CAAC,CAAC,GAAG;QACL,CAAC,CAAC,iBAAiB,CAAC;AACxB,CAAC"}
{"version":3,"file":"tools-download.js","sourceRoot":"","sources":["../src/tools-download.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkFA,gDA+FC;AA8CD,sDAOC;AAED,4DAOC;AA/OD,uCAAyB;AAEzB,uCAAyB;AACzB,2CAA6B;AAC7B,2CAAyC;AAEzC,oDAAsC;AACtC,sDAAkD;AAClD,+DAAiD;AACjD,uDAAyC;AACzC,+CAAiC;AAEjC,uCAAmD;AACnD,2CAA6B;AAC7B,iCAA2E;AAE3E;;GAEG;AACU,QAAA,8BAA8B,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,QAAQ;AAEvE;;GAEG;AACH,MAAM,mBAAmB,GAAG,QAAQ,CAAC;AAarC,SAAS,uCAAuC,CAC9C,kBAA0B,EAC1B,oBAA4B;IAE5B,OAAO;QACL,kBAAkB,EAAE,kBAAkB,GAAG,oBAAoB;QAC7D,kBAAkB;QAClB,oBAAoB;QACpB,gBAAgB,EAAE,KAAK;KACxB,CAAC;AACJ,CAAC;AAaD,SAAS,kCAAkC,CACzC,kBAA0B;IAE1B,OAAO;QACL,kBAAkB;QAClB,kBAAkB,EAAE,SAAS;QAC7B,oBAAoB,EAAE,SAAS;QAC/B,gBAAgB,EAAE,IAAI;KACvB,CAAC;AACJ,CAAC;AAaM,KAAK,UAAU,kBAAkB,CACtC,SAAiB,EACjB,IAAY,EACZ,aAAiC,EACjC,OAA4B,EAC5B,UAAsC,EACtC,MAAc;IAEd,MAAM,CAAC,IAAI,CACT,iCAAiC,SAAS,2BAA2B,CACtE,CAAC;IAEF,MAAM,iBAAiB,GAAG,GAAG,CAAC,sBAAsB,CAAC,SAAS,CAAC,CAAC;IAEhE,IAAI,CAAC;QACH,IAAI,iBAAiB,KAAK,MAAM,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE,CAAC;YACjE,MAAM,CAAC,IAAI,CAAC,gDAAgD,CAAC,CAAC;YAE9D,MAAM,iBAAiB,GAAG,wBAAW,CAAC,GAAG,EAAE,CAAC;YAC5C,MAAM,mCAAmC,CACvC,SAAS,EACT,IAAI,EACJ,aAAa,EACb,OAAO,EACP,UAAW,EACX,MAAM,CACP,CAAC;YAEF,MAAM,kBAAkB,GAAG,IAAI,CAAC,KAAK,CACnC,wBAAW,CAAC,GAAG,EAAE,GAAG,iBAAiB,CACtC,CAAC;YACF,MAAM,CAAC,IAAI,CACT,wDAAwD,IAAI,KAAK,IAAA,wBAAc,EAC7E,kBAAkB,CACnB,IAAI,CACN,CAAC;YAEF,OAAO;gBACL,iBAAiB;gBACjB,QAAQ,EAAE,0BAA0B,CAAC,SAAS,CAAC;gBAC/C,GAAG,kCAAkC,CAAC,kBAAkB,CAAC;aAC1D,CAAC;QACJ,CAAC;IACH,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAI,CAAC,OAAO,CACV,yHAAyH,CAC1H,CAAC;QACF,IAAI,CAAC,OAAO,CAAC,IAAA,sBAAe,EAAC,CAAC,CAAC,CAAC,CAAC;IACnC,CAAC;IAED,MAAM,kBAAkB,GAAG,wBAAW,CAAC,GAAG,EAAE,CAAC;IAC7C,MAAM,kBAAkB,GAAG,MAAM,SAAS,CAAC,YAAY,CACrD,SAAS,EACT,SAAS,EACT,aAAa,EACb,OAAO,CACR,CAAC;IACF,MAAM,kBAAkB,GAAG,IAAI,CAAC,KAAK,CAAC,wBAAW,CAAC,GAAG,EAAE,GAAG,kBAAkB,CAAC,CAAC;IAE9E,MAAM,CAAC,IAAI,CACT,yCAAyC,kBAAkB,KAAK,IAAA,wBAAc,EAC5E,kBAAkB,CACnB,IAAI,CACN,CAAC;IAEF,IAAI,oBAA4B,CAAC;IAEjC,IAAI,CAAC;QACH,MAAM,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;QACzC,MAAM,eAAe,GAAG,wBAAW,CAAC,GAAG,EAAE,CAAC;QAC1C,MAAM,GAAG,CAAC,OAAO,CACf,kBAAkB,EAClB,IAAI,EACJ,iBAAiB,EACjB,UAAU,EACV,MAAM,CACP,CAAC;QACF,oBAAoB,GAAG,IAAI,CAAC,KAAK,CAAC,wBAAW,CAAC,GAAG,EAAE,GAAG,eAAe,CAAC,CAAC;QACvE,MAAM,CAAC,IAAI,CACT,wCAAwC,IAAI,KAAK,IAAA,wBAAc,EAC7D,oBAAoB,CACrB,IAAI,CACN,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,MAAM,IAAA,kBAAW,EAAC,kBAAkB,EAAE,uBAAuB,EAAE,MAAM,CAAC,CAAC;IACzE,CAAC;IAED,OAAO;QACL,iBAAiB;QACjB,QAAQ,EAAE,0BAA0B,CAAC,SAAS,CAAC;QAC/C,GAAG,uCAAuC,CACxC,kBAAkB,EAClB,oBAAoB,CACrB;KACF,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,mCAAmC,CAChD,SAAiB,EACjB,IAAY,EACZ,aAAiC,EACjC,OAA4B,EAC5B,UAA0B,EAC1B,MAAc;IAEd,4BAA4B;IAC5B,EAAE,CAAC,SAAS,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAExC,mDAAmD;IACnD,MAAM,KAAK,GAAG,IAAI,wBAAU,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;IAEnD,8DAA8D;IAC9D,OAAO,GAAG,MAAM,CAAC,MAAM,CACrB,EAAE,YAAY,EAAE,eAAe,EAAE,EACjC,aAAa,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,EACtC,OAAO,CACR,CAAC;IACF,MAAM,QAAQ,GAAG,MAAM,IAAI,OAAO,CAAkB,CAAC,OAAO,EAAE,EAAE,CAC9D,wBAAK,CAAC,GAAG,CACP,SAAS,EACT;QACE,OAAO;QACP,uDAAuD;QACvD,aAAa,EAAE,sCAA8B;QAC7C,2CAA2C;QAC3C,KAAK;KACuB,EAC9B,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,CAClB,CACF,CAAC;IAEF,IAAI,QAAQ,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;QAChC,MAAM,IAAI,KAAK,CACb,yCAAyC,SAAS,uBAAuB,QAAQ,CAAC,UAAU,GAAG,CAChG,CAAC;IACJ,CAAC;IAED,MAAM,GAAG,CAAC,aAAa,CAAC,QAAQ,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;AAC9D,CAAC;AAED,8FAA8F;AAC9F,SAAgB,qBAAqB,CAAC,OAAe;IACnD,OAAO,IAAI,CAAC,IAAI,CACd,IAAA,0BAAmB,EAAC,mBAAmB,CAAC,EACxC,mBAAmB,EACnB,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,OAAO,EAChC,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,CAChB,CAAC;AACJ,CAAC;AAED,SAAgB,wBAAwB,CACtC,aAAqB,EACrB,MAAc;IAEd,MAAM,cAAc,GAAG,GAAG,aAAa,WAAW,CAAC;IACnD,EAAE,CAAC,aAAa,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;IACrC,MAAM,CAAC,IAAI,CAAC,iCAAiC,cAAc,EAAE,CAAC,CAAC;AACjE,CAAC;AAED,SAAS,0BAA0B,CAAC,GAAW;IAC7C,OAAO,CAAC,sBAAsB,EAAE,kCAAkC,CAAC,CAAC,IAAI,CACtE,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,sBAAsB,IAAI,qBAAqB,CAAC,CAC1E;QACC,CAAC,CAAC,GAAG;QACL,CAAC,CAAC,iBAAiB,CAAC;AACxB,CAAC"}