codeql-action/src/tools-download.ts

import * as fs from "fs";
import { IncomingMessage, OutgoingHttpHeaders, RequestOptions } from "http";
import * as os from "os";
import * as path from "path";
import { performance } from "perf_hooks";

import * as toolcache from "@actions/tool-cache";
import { https } from "follow-redirects";
import * as semver from "semver";

import { Feature, FeatureEnablement } from "./feature-flags";
import { formatDuration, Logger } from "./logging";
import * as tar from "./tar";
import { cleanUpGlob, getRequiredEnvParam } from "./util";

/**
 * High watermark to use when streaming the download and extraction of the CodeQL tools.
 */
export const STREAMING_HIGH_WATERMARK_BYTES = 4 * 1024 * 1024; // 4 MiB

/**
 * The name of the tool cache directory for the CodeQL tools.
 */
const TOOLCACHE_TOOL_NAME = "CodeQL";

/**
 * Timing information for the download and extraction of the CodeQL tools when
 * we fully download the bundle before extracting.
 */
type DownloadFirstToolsDownloadDurations = {
  combinedDurationMs: number;
  downloadDurationMs: number;
  extractionDurationMs: number;
  streamExtraction: false;
};

function makeDownloadFirstToolsDownloadDurations(
  downloadDurationMs: number,
  extractionDurationMs: number,
): DownloadFirstToolsDownloadDurations {
  return {
    combinedDurationMs: downloadDurationMs + extractionDurationMs,
    downloadDurationMs,
    extractionDurationMs,
    streamExtraction: false,
  };
}

/**
 * Timing information for the download and extraction of the CodeQL tools when
 * we stream the download and extraction of the bundle.
 */
type StreamedToolsDownloadDurations = {
  combinedDurationMs: number;
  downloadDurationMs: undefined;
  extractionDurationMs: undefined;
  streamExtraction: true;
};

function makeStreamedToolsDownloadDurations(
  combinedDurationMs: number,
): StreamedToolsDownloadDurations {
  return {
    combinedDurationMs,
    downloadDurationMs: undefined,
    extractionDurationMs: undefined,
    streamExtraction: true,
  };
}

type ToolsDownloadDurations =
  | DownloadFirstToolsDownloadDurations
  | StreamedToolsDownloadDurations;

export type ToolsDownloadStatusReport = {
  compressionMethod: tar.CompressionMethod;
  toolsUrl: string;
  zstdFailureReason?: string;
} & ToolsDownloadDurations;

export async function downloadAndExtract(
  codeqlURL: string,
  dest: string,
  authorization: string | undefined,
  headers: OutgoingHttpHeaders,
  tarVersion: tar.TarVersion | undefined,
  features: FeatureEnablement,
  logger: Logger,
): Promise<ToolsDownloadStatusReport> {
  logger.info(
    `Downloading CodeQL tools from ${codeqlURL} . This may take a while.`,
  );

  const compressionMethod = tar.inferCompressionMethod(codeqlURL);

  // TODO: Re-enable streaming when we have a more reliable way to respect proxy settings.

  if (
    (await features.getValue(Feature.ZstdBundleStreamingExtraction)) &&
    compressionMethod === "zstd" &&
    process.platform === "linux"
  ) {
    logger.info(`Streaming the extraction of the CodeQL bundle.`);

    const toolsInstallStart = performance.now();
    await downloadAndExtractZstdWithStreaming(
      codeqlURL,
      dest,
      authorization,
      headers,
      tarVersion!,
      logger,
    );

    const combinedDurationMs = Math.round(
      performance.now() - toolsInstallStart,
    );
    logger.info(
      `Finished downloading and extracting CodeQL bundle to ${dest} (${formatDuration(
        combinedDurationMs,
      )}).`,
    );

    return {
      compressionMethod,
      toolsUrl: sanitizeUrlForStatusReport(codeqlURL),
      ...makeStreamedToolsDownloadDurations(combinedDurationMs),
    };
  }

  const toolsDownloadStart = performance.now();
  const archivedBundlePath = await toolcache.downloadTool(
    codeqlURL,
    undefined,
    authorization,
    headers,
  );
  const downloadDurationMs = Math.round(performance.now() - toolsDownloadStart);

  logger.info(
    `Finished downloading CodeQL bundle to ${archivedBundlePath} (${formatDuration(
      downloadDurationMs,
    )}).`,
  );

  let extractionDurationMs: number;

  try {
    logger.info("Extracting CodeQL bundle.");
    const extractionStart = performance.now();
    await tar.extract(
      archivedBundlePath,
      dest,
      compressionMethod,
      tarVersion,
      logger,
    );
    extractionDurationMs = Math.round(performance.now() - extractionStart);
    logger.info(
      `Finished extracting CodeQL bundle to ${dest} (${formatDuration(
        extractionDurationMs,
      )}).`,
    );
  } finally {
    await cleanUpGlob(archivedBundlePath, "CodeQL bundle archive", logger);
  }

  return {
    compressionMethod,
    toolsUrl: sanitizeUrlForStatusReport(codeqlURL),
    ...makeDownloadFirstToolsDownloadDurations(
      downloadDurationMs,
      extractionDurationMs,
    ),
  };
}

async function downloadAndExtractZstdWithStreaming(
  codeqlURL: string,
  dest: string,
  authorization: string | undefined,
  headers: OutgoingHttpHeaders,
  tarVersion: tar.TarVersion,
  logger: Logger,
): Promise<void> {
  headers = Object.assign(
    { "User-Agent": "CodeQL Action" },
    authorization ? { authorization } : {},
    headers,
  );
  const response = await new Promise<IncomingMessage>((resolve) =>
    https.get(
      codeqlURL,
      {
        headers,
        highWaterMark: STREAMING_HIGH_WATERMARK_BYTES,
      } as unknown as RequestOptions,
      (r) => resolve(r),
    ),
  );

  if (response.statusCode !== 200) {
    throw new Error(
      `Failed to download CodeQL bundle from ${codeqlURL}. HTTP status code: ${response.statusCode}.`,
    );
  }

  await tar.extractTarZst(response, dest, tarVersion, logger);
}

/** Gets the path to the toolcache directory for the specified version of the CodeQL tools. */
export function getToolcacheDirectory(version: string): string {
  return path.join(
    getRequiredEnvParam("RUNNER_TOOL_CACHE"),
    TOOLCACHE_TOOL_NAME,
    semver.clean(version) || version,
    os.arch() || "",
  );
}

export function writeToolcacheMarkerFile(
  extractedPath: string,
  logger: Logger,
): void {
  const markerFilePath = `${extractedPath}.complete`;
  fs.writeFileSync(markerFilePath, "");
  logger.info(`Wrote toolcache marker file to ${markerFilePath}`);
}

function sanitizeUrlForStatusReport(url: string): string {
  return ["github/codeql-action", "dsp-testing/codeql-cli-nightlies"].some(
    (repo) => url.startsWith(`https://github.com/${repo}/releases/download/`),
  )
    ? url
    : "sanitized-value";
}