mirror of
https://github.com/github/codeql-action.git
synced 2025-12-29 18:50:25 +08:00
285 lines
16 KiB
JavaScript
285 lines
16 KiB
JavaScript
"use strict";
|
|
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
if (k2 === undefined) k2 = k;
|
|
Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
|
|
}) : (function(o, m, k, k2) {
|
|
if (k2 === undefined) k2 = k;
|
|
o[k2] = m[k];
|
|
}));
|
|
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
}) : function(o, v) {
|
|
o["default"] = v;
|
|
});
|
|
var __importStar = (this && this.__importStar) || function (mod) {
|
|
if (mod && mod.__esModule) return mod;
|
|
var result = {};
|
|
if (mod != null) for (var k in mod) if (k !== "default" && Object.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
__setModuleDefault(result, mod);
|
|
return result;
|
|
};
|
|
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
return new (P || (P = Promise))(function (resolve, reject) {
|
|
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
});
|
|
};
|
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
exports.DownloadHttpClient = void 0;
|
|
const fs = __importStar(require("fs"));
|
|
const core = __importStar(require("@actions/core"));
|
|
const zlib = __importStar(require("zlib"));
|
|
const utils_1 = require("./utils");
|
|
const url_1 = require("url");
|
|
const status_reporter_1 = require("./status-reporter");
|
|
const perf_hooks_1 = require("perf_hooks");
|
|
const http_manager_1 = require("./http-manager");
|
|
const config_variables_1 = require("./config-variables");
|
|
const requestUtils_1 = require("./requestUtils");
|
|
class DownloadHttpClient {
|
|
constructor() {
|
|
this.downloadHttpManager = new http_manager_1.HttpManager(config_variables_1.getDownloadFileConcurrency(), '@actions/artifact-download');
|
|
// downloads are usually significantly faster than uploads so display status information every second
|
|
this.statusReporter = new status_reporter_1.StatusReporter(1000);
|
|
}
|
|
/**
|
|
* Gets a list of all artifacts that are in a specific container
|
|
*/
|
|
listArtifacts() {
|
|
return __awaiter(this, void 0, void 0, function* () {
|
|
const artifactUrl = utils_1.getArtifactUrl();
|
|
// use the first client from the httpManager, `keep-alive` is not used so the connection will close immediately
|
|
const client = this.downloadHttpManager.getClient(0);
|
|
const headers = utils_1.getDownloadHeaders('application/json');
|
|
const response = yield requestUtils_1.retryHttpClientRequest('List Artifacts', () => __awaiter(this, void 0, void 0, function* () { return client.get(artifactUrl, headers); }));
|
|
const body = yield response.readBody();
|
|
return JSON.parse(body);
|
|
});
|
|
}
|
|
/**
|
|
* Fetches a set of container items that describe the contents of an artifact
|
|
* @param artifactName the name of the artifact
|
|
* @param containerUrl the artifact container URL for the run
|
|
*/
|
|
getContainerItems(artifactName, containerUrl) {
|
|
return __awaiter(this, void 0, void 0, function* () {
|
|
// the itemPath search parameter controls which containers will be returned
|
|
const resourceUrl = new url_1.URL(containerUrl);
|
|
resourceUrl.searchParams.append('itemPath', artifactName);
|
|
// use the first client from the httpManager, `keep-alive` is not used so the connection will close immediately
|
|
const client = this.downloadHttpManager.getClient(0);
|
|
const headers = utils_1.getDownloadHeaders('application/json');
|
|
const response = yield requestUtils_1.retryHttpClientRequest('Get Container Items', () => __awaiter(this, void 0, void 0, function* () { return client.get(resourceUrl.toString(), headers); }));
|
|
const body = yield response.readBody();
|
|
return JSON.parse(body);
|
|
});
|
|
}
|
|
/**
|
|
* Concurrently downloads all the files that are part of an artifact
|
|
* @param downloadItems information about what items to download and where to save them
|
|
*/
|
|
downloadSingleArtifact(downloadItems) {
|
|
return __awaiter(this, void 0, void 0, function* () {
|
|
const DOWNLOAD_CONCURRENCY = config_variables_1.getDownloadFileConcurrency();
|
|
// limit the number of files downloaded at a single time
|
|
core.debug(`Download file concurrency is set to ${DOWNLOAD_CONCURRENCY}`);
|
|
const parallelDownloads = [...new Array(DOWNLOAD_CONCURRENCY).keys()];
|
|
let currentFile = 0;
|
|
let downloadedFiles = 0;
|
|
core.info(`Total number of files that will be downloaded: ${downloadItems.length}`);
|
|
this.statusReporter.setTotalNumberOfFilesToProcess(downloadItems.length);
|
|
this.statusReporter.start();
|
|
yield Promise.all(parallelDownloads.map((index) => __awaiter(this, void 0, void 0, function* () {
|
|
while (currentFile < downloadItems.length) {
|
|
const currentFileToDownload = downloadItems[currentFile];
|
|
currentFile += 1;
|
|
const startTime = perf_hooks_1.performance.now();
|
|
yield this.downloadIndividualFile(index, currentFileToDownload.sourceLocation, currentFileToDownload.targetPath);
|
|
if (core.isDebug()) {
|
|
core.debug(`File: ${++downloadedFiles}/${downloadItems.length}. ${currentFileToDownload.targetPath} took ${(perf_hooks_1.performance.now() - startTime).toFixed(3)} milliseconds to finish downloading`);
|
|
}
|
|
this.statusReporter.incrementProcessedCount();
|
|
}
|
|
})))
|
|
.catch(error => {
|
|
throw new Error(`Unable to download the artifact: ${error}`);
|
|
})
|
|
.finally(() => {
|
|
this.statusReporter.stop();
|
|
// safety dispose all connections
|
|
this.downloadHttpManager.disposeAndReplaceAllClients();
|
|
});
|
|
});
|
|
}
|
|
/**
|
|
* Downloads an individual file
|
|
* @param httpClientIndex the index of the http client that is used to make all of the calls
|
|
* @param artifactLocation origin location where a file will be downloaded from
|
|
* @param downloadPath destination location for the file being downloaded
|
|
*/
|
|
downloadIndividualFile(httpClientIndex, artifactLocation, downloadPath) {
|
|
return __awaiter(this, void 0, void 0, function* () {
|
|
let retryCount = 0;
|
|
const retryLimit = config_variables_1.getRetryLimit();
|
|
let destinationStream = fs.createWriteStream(downloadPath);
|
|
const headers = utils_1.getDownloadHeaders('application/json', true, true);
|
|
// a single GET request is used to download a file
|
|
const makeDownloadRequest = () => __awaiter(this, void 0, void 0, function* () {
|
|
const client = this.downloadHttpManager.getClient(httpClientIndex);
|
|
return yield client.get(artifactLocation, headers);
|
|
});
|
|
// check the response headers to determine if the file was compressed using gzip
|
|
const isGzip = (incomingHeaders) => {
|
|
return ('content-encoding' in incomingHeaders &&
|
|
incomingHeaders['content-encoding'] === 'gzip');
|
|
};
|
|
// Increments the current retry count and then checks if the retry limit has been reached
|
|
// If there have been too many retries, fail so the download stops. If there is a retryAfterValue value provided,
|
|
// it will be used
|
|
const backOff = (retryAfterValue) => __awaiter(this, void 0, void 0, function* () {
|
|
retryCount++;
|
|
if (retryCount > retryLimit) {
|
|
return Promise.reject(new Error(`Retry limit has been reached. Unable to download ${artifactLocation}`));
|
|
}
|
|
else {
|
|
this.downloadHttpManager.disposeAndReplaceClient(httpClientIndex);
|
|
if (retryAfterValue) {
|
|
// Back off by waiting the specified time denoted by the retry-after header
|
|
core.info(`Backoff due to too many requests, retry #${retryCount}. Waiting for ${retryAfterValue} milliseconds before continuing the download`);
|
|
yield utils_1.sleep(retryAfterValue);
|
|
}
|
|
else {
|
|
// Back off using an exponential value that depends on the retry count
|
|
const backoffTime = utils_1.getExponentialRetryTimeInMilliseconds(retryCount);
|
|
core.info(`Exponential backoff for retry #${retryCount}. Waiting for ${backoffTime} milliseconds before continuing the download`);
|
|
yield utils_1.sleep(backoffTime);
|
|
}
|
|
core.info(`Finished backoff for retry #${retryCount}, continuing with download`);
|
|
}
|
|
});
|
|
const isAllBytesReceived = (expected, received) => {
|
|
// be lenient, if any input is missing, assume success, i.e. not truncated
|
|
if (!expected ||
|
|
!received ||
|
|
process.env['ACTIONS_ARTIFACT_SKIP_DOWNLOAD_VALIDATION']) {
|
|
core.info('Skipping download validation.');
|
|
return true;
|
|
}
|
|
return parseInt(expected) === received;
|
|
};
|
|
const resetDestinationStream = (fileDownloadPath) => __awaiter(this, void 0, void 0, function* () {
|
|
destinationStream.close();
|
|
yield utils_1.rmFile(fileDownloadPath);
|
|
destinationStream = fs.createWriteStream(fileDownloadPath);
|
|
});
|
|
// keep trying to download a file until a retry limit has been reached
|
|
while (retryCount <= retryLimit) {
|
|
let response;
|
|
try {
|
|
response = yield makeDownloadRequest();
|
|
}
|
|
catch (error) {
|
|
// if an error is caught, it is usually indicative of a timeout so retry the download
|
|
core.info('An error occurred while attempting to download a file');
|
|
// eslint-disable-next-line no-console
|
|
console.log(error);
|
|
// increment the retryCount and use exponential backoff to wait before making the next request
|
|
yield backOff();
|
|
continue;
|
|
}
|
|
let forceRetry = false;
|
|
if (utils_1.isSuccessStatusCode(response.message.statusCode)) {
|
|
// The body contains the contents of the file however calling response.readBody() causes all the content to be converted to a string
|
|
// which can cause some gzip encoded data to be lost
|
|
// Instead of using response.readBody(), response.message is a readableStream that can be directly used to get the raw body contents
|
|
try {
|
|
const isGzipped = isGzip(response.message.headers);
|
|
yield this.pipeResponseToFile(response, destinationStream, isGzipped);
|
|
if (isGzipped ||
|
|
isAllBytesReceived(response.message.headers['content-length'], yield utils_1.getFileSize(downloadPath))) {
|
|
return;
|
|
}
|
|
else {
|
|
forceRetry = true;
|
|
}
|
|
}
|
|
catch (error) {
|
|
// retry on error, most likely streams were corrupted
|
|
forceRetry = true;
|
|
}
|
|
}
|
|
if (forceRetry || utils_1.isRetryableStatusCode(response.message.statusCode)) {
|
|
core.info(`A ${response.message.statusCode} response code has been received while attempting to download an artifact`);
|
|
resetDestinationStream(downloadPath);
|
|
// if a throttled status code is received, try to get the retryAfter header value, else differ to standard exponential backoff
|
|
utils_1.isThrottledStatusCode(response.message.statusCode)
|
|
? yield backOff(utils_1.tryGetRetryAfterValueTimeInMilliseconds(response.message.headers))
|
|
: yield backOff();
|
|
}
|
|
else {
|
|
// Some unexpected response code, fail immediately and stop the download
|
|
utils_1.displayHttpDiagnostics(response);
|
|
return Promise.reject(new Error(`Unexpected http ${response.message.statusCode} during download for ${artifactLocation}`));
|
|
}
|
|
}
|
|
});
|
|
}
|
|
/**
|
|
* Pipes the response from downloading an individual file to the appropriate destination stream while decoding gzip content if necessary
|
|
* @param response the http response received when downloading a file
|
|
* @param destinationStream the stream where the file should be written to
|
|
* @param isGzip a boolean denoting if the content is compressed using gzip and if we need to decode it
|
|
*/
|
|
pipeResponseToFile(response, destinationStream, isGzip) {
|
|
return __awaiter(this, void 0, void 0, function* () {
|
|
yield new Promise((resolve, reject) => {
|
|
if (isGzip) {
|
|
const gunzip = zlib.createGunzip();
|
|
response.message
|
|
.on('error', error => {
|
|
core.error(`An error occurred while attempting to read the response stream`);
|
|
gunzip.close();
|
|
destinationStream.close();
|
|
reject(error);
|
|
})
|
|
.pipe(gunzip)
|
|
.on('error', error => {
|
|
core.error(`An error occurred while attempting to decompress the response stream`);
|
|
destinationStream.close();
|
|
reject(error);
|
|
})
|
|
.pipe(destinationStream)
|
|
.on('close', () => {
|
|
resolve();
|
|
})
|
|
.on('error', error => {
|
|
core.error(`An error occurred while writing a downloaded file to ${destinationStream.path}`);
|
|
reject(error);
|
|
});
|
|
}
|
|
else {
|
|
response.message
|
|
.on('error', error => {
|
|
core.error(`An error occurred while attempting to read the response stream`);
|
|
destinationStream.close();
|
|
reject(error);
|
|
})
|
|
.pipe(destinationStream)
|
|
.on('close', () => {
|
|
resolve();
|
|
})
|
|
.on('error', error => {
|
|
core.error(`An error occurred while writing a downloaded file to ${destinationStream.path}`);
|
|
reject(error);
|
|
});
|
|
}
|
|
});
|
|
return;
|
|
});
|
|
}
|
|
}
|
|
exports.DownloadHttpClient = DownloadHttpClient;
|
|
//# sourceMappingURL=download-http-client.js.map
|