Merge branch 'main' into ignore-temp-dir

src/actions-util.test.ts

@@ -1,13 +1,36 @@
 import test from "ava";
+import sinon from "sinon";
 
-import { getRef, prepareLocalRunEnvironment } from "./actions-util";
+import * as actionsutil from "./actions-util";
 import { setupTests } from "./testing-utils";
 
 setupTests(test);
 
-test("getRef() throws on the empty string", (t) => {
+test("getRef() throws on the empty string", async (t) => {
   process.env["GITHUB_REF"] = "";
-  t.throws(getRef);
+  await t.throwsAsync(actionsutil.getRef);
+});
+
+test("getRef() returns merge PR ref if GITHUB_SHA still checked out", async (t) => {
+  const expectedRef = "refs/pull/1/merge";
+  const currentSha = "a".repeat(40);
+  process.env["GITHUB_REF"] = expectedRef;
+  process.env["GITHUB_SHA"] = currentSha;
+
+  sinon.stub(actionsutil, "getCommitOid").resolves(currentSha);
+
+  const actualRef = await actionsutil.getRef();
+  t.deepEqual(actualRef, expectedRef);
+});
+
+test("getRef() returns head PR ref if GITHUB_SHA not currently checked out", async (t) => {
+  process.env["GITHUB_REF"] = "refs/pull/1/merge";
+  process.env["GITHUB_SHA"] = "a".repeat(40);
+
+  sinon.stub(actionsutil, "getCommitOid").resolves("b".repeat(40));
+
+  const actualRef = await actionsutil.getRef();
+  t.deepEqual(actualRef, "refs/pull/1/head");
 });
 
 test("prepareEnvironment() when a local run", (t) => {
@@ -16,21 +39,21 @@ test("prepareEnvironment() when a local run", (t) => {
   process.env.CODEQL_LOCAL_RUN = "false";
   process.env.GITHUB_JOB = "YYY";
 
-  prepareLocalRunEnvironment();
+  actionsutil.prepareLocalRunEnvironment();
 
   // unchanged
   t.deepEqual(process.env.GITHUB_JOB, "YYY");
 
   process.env.CODEQL_LOCAL_RUN = "true";
 
-  prepareLocalRunEnvironment();
+  actionsutil.prepareLocalRunEnvironment();
 
   // unchanged
   t.deepEqual(process.env.GITHUB_JOB, "YYY");
 
   process.env.GITHUB_JOB = "";
 
-  prepareLocalRunEnvironment();
+  actionsutil.prepareLocalRunEnvironment();
 
   // updated
   t.deepEqual(process.env.GITHUB_JOB, "UNKNOWN-JOB");

src/actions-util.ts

@@ -3,7 +3,7 @@ import * as toolrunnner from "@actions/exec/lib/toolrunner";
 
 import * as api from "./api-client";
 import * as sharedEnv from "./shared-environment";
-import { isLocalRun, GITHUB_DOTCOM_URL } from "./util";
+import { GITHUB_DOTCOM_URL, isLocalRun } from "./util";
 
 /**
  * Wrapper around core.getInput for inputs that always have a value.
@@ -57,7 +57,7 @@ export function prepareLocalRunEnvironment() {
 /**
  * Gets the SHA of the commit that is currently checked out.
  */
-export async function getCommitOid(): Promise<string> {
+export const getCommitOid = async function (): Promise<string> {
   // Try to use git to get the current commit SHA. If that fails then
   // log but otherwise silently fall back to using the SHA from the environment.
   // The only time these two values will differ is during analysis of a PR when
@@ -85,7 +85,7 @@ export async function getCommitOid(): Promise<string> {
     );
     return getRequiredEnvParam("GITHUB_SHA");
   }
-}
+};
 
 /**
  * Get the path of the currently executing workflow.
@@ -149,17 +149,22 @@ export async function getAnalysisKey(): Promise<string> {
 /**
  * Get the ref currently being analyzed.
  */
-export function getRef(): string {
+export async function getRef(): Promise<string> {
   // Will be in the form "refs/heads/master" on a push event
   // or in the form "refs/pull/N/merge" on a pull_request event
   const ref = getRequiredEnvParam("GITHUB_REF");
 
-  // For pull request refs we want to convert from the 'merge' ref
-  // to the 'head' ref, as that is what we want to analyse.
-  // There should have been some code earlier in the workflow to do
-  // the checkout, but we have no way of verifying that here.
+  // For pull request refs we want to detect whether the workflow
+  // has run `git checkout HEAD^2` to analyze the 'head' ref rather
+  // than the 'merge' ref. If so, we want to convert the ref that
+  // we report back.
   const pull_ref_regex = /refs\/pull\/(\d+)\/merge/;
-  if (pull_ref_regex.test(ref)) {
+  const checkoutSha = await getCommitOid();
+
+  if (
+    pull_ref_regex.test(ref) &&
+    checkoutSha !== getRequiredEnvParam("GITHUB_SHA")
+  ) {
     return ref.replace(pull_ref_regex, "refs/pull/$1/head");
   } else {
     return ref;
@@ -219,7 +224,7 @@ export async function createStatusReportBase(
   exception?: string
 ): Promise<StatusReportBase> {
   const commitOid = process.env["GITHUB_SHA"] || "";
-  const ref = getRef();
+  const ref = await getRef();
   const workflowRunIDStr = process.env["GITHUB_RUN_ID"];
   let workflowRunID = -1;
   if (workflowRunIDStr) {

src/analyze-action.ts

@@ -64,7 +64,7 @@ async function run() {
     stats = await runAnalyze(
       parseRepositoryNwo(actionsUtil.getRequiredEnvParam("GITHUB_REPOSITORY")),
       await actionsUtil.getCommitOid(),
-      actionsUtil.getRef(),
+      await actionsUtil.getRef(),
       await actionsUtil.getAnalysisKey(),
       actionsUtil.getRequiredEnvParam("GITHUB_WORKFLOW"),
       actionsUtil.getWorkflowRunID(),

src/analyze.test.ts

@@ -1,6 +1,7 @@
-import test from "ava";
 import * as fs from "fs";
 
+import test from "ava";
+
 import { runQueries } from "./analyze";
 import { setCodeQL } from "./codeql";
 import { Config } from "./config-utils";

src/analyze.ts

@@ -1,10 +1,12 @@
 import * as fs from "fs";
 import * as path from "path";
 
+import * as toolrunnner from "@actions/exec/lib/toolrunner";
+
 import * as analysisPaths from "./analysis-paths";
 import { getCodeQL } from "./codeql";
 import * as configUtils from "./config-utils";
-import { isScannedLanguage } from "./languages";
+import { isScannedLanguage, Language } from "./languages";
 import { Logger } from "./logging";
 import { RepositoryNwo } from "./repository";
 import * as sharedEnv from "./shared-environment";
@@ -44,6 +46,43 @@ export interface AnalysisStatusReport
   extends upload_lib.UploadStatusReport,
     QueriesStatusReport {}
 
+async function setupPythonExtractor(logger: Logger) {
+  const codeqlPython = process.env["CODEQL_PYTHON"];
+  if (codeqlPython === undefined || codeqlPython.length === 0) {
+    // If CODEQL_PYTHON is not set, no dependencies were installed, so we don't need to do anything
+    return;
+  }
+
+  let output = "";
+  const options = {
+    listeners: {
+      stdout: (data: Buffer) => {
+        output += data.toString();
+      },
+    },
+  };
+
+  await new toolrunnner.ToolRunner(
+    codeqlPython,
+    [
+      "-c",
+      "import os; import pip; print(os.path.dirname(os.path.dirname(pip.__file__)))",
+    ],
+    options
+  ).exec();
+  logger.info(`Setting LGTM_INDEX_IMPORT_PATH=${output}`);
+  process.env["LGTM_INDEX_IMPORT_PATH"] = output;
+
+  output = "";
+  await new toolrunnner.ToolRunner(
+    codeqlPython,
+    ["-c", "import sys; print(sys.version_info[0])"],
+    options
+  ).exec();
+  logger.info(`Setting LGTM_PYTHON_SETUP_VERSION=${output}`);
+  process.env["LGTM_PYTHON_SETUP_VERSION"] = output;
+}
+
 async function createdDBForScannedLanguages(
   config: configUtils.Config,
   logger: Logger
@@ -56,6 +95,11 @@ async function createdDBForScannedLanguages(
   for (const language of config.languages) {
     if (isScannedLanguage(language)) {
       logger.startGroup(`Extracting ${language}`);
+
+      if (language === Language.python) {
+        await setupPythonExtractor(logger);
+      }
+
       await codeql.extractScannedLanguage(
         util.getCodeQLDatabasePath(config.tempDir, language),
         language

src/api-client.ts

@@ -1,7 +1,8 @@
+import * as path from "path";
+
 import * as githubUtils from "@actions/github/lib/utils";
 import * as retry from "@octokit/plugin-retry";
 import consoleLogLevel from "console-log-level";
-import * as path from "path";
 
 import { getRequiredEnvParam, getRequiredInput } from "./actions-util";
 import { isLocalRun } from "./util";

src/codeql.test.ts

@@ -1,13 +1,14 @@
+import * as path from "path";
+
 import * as toolcache from "@actions/tool-cache";
 import test from "ava";
 import nock from "nock";
-import * as path from "path";
 
 import * as codeql from "./codeql";
+import * as defaults from "./defaults.json";
 import { getRunnerLogger } from "./logging";
 import { setupTests } from "./testing-utils";
 import * as util from "./util";
-import * as defaults from "./defaults.json";
 
 setupTests(test);
 
@@ -143,10 +144,16 @@ test("download codeql bundle cache with different version cached (not pinned)",
     );
 
     t.assert(toolcache.find("CodeQL", "0.0.0-20200601"));
+    const platform =
+      process.platform === "win32"
+        ? "win64"
+        : process.platform === "linux"
+        ? "linux64"
+        : "osx64";
 
     nock("https://github.com")
       .get(
-        `/github/codeql-action/releases/download/${defaults.bundleVersion}/codeql-bundle.tar.gz`
+        `/github/codeql-action/releases/download/${defaults.bundleVersion}/codeql-bundle-${platform}.tar.gz`
       )
       .replyWithFile(
         200,
@@ -169,7 +176,7 @@ test("download codeql bundle cache with different version cached (not pinned)",
   });
 });
 
-test('download codeql bundle cache with pinned different version cached if "latests" tools specied', async (t) => {
+test('download codeql bundle cache with pinned different version cached if "latests" tools specified', async (t) => {
   await util.withTmpDir(async (tmpDir) => {
     nock("https://example.com")
       .get(`/download/codeql-bundle-20200601/codeql-bundle.tar.gz`)
@@ -190,9 +197,16 @@ test('download codeql bundle cache with pinned different version cached if "late
 
     t.assert(toolcache.find("CodeQL", "0.0.0-20200601"));
 
+    const platform =
+      process.platform === "win32"
+        ? "win64"
+        : process.platform === "linux"
+        ? "linux64"
+        : "osx64";
+
     nock("https://github.com")
       .get(
-        `/github/codeql-action/releases/download/${defaults.bundleVersion}/codeql-bundle.tar.gz`
+        `/github/codeql-action/releases/download/${defaults.bundleVersion}/codeql-bundle-${platform}.tar.gz`
       )
       .replyWithFile(
         200,

src/codeql.ts

@@ -1,13 +1,14 @@
+import * as fs from "fs";
+import * as path from "path";
+import * as stream from "stream";
+import * as globalutil from "util";
+
 import * as toolrunnner from "@actions/exec/lib/toolrunner";
 import * as http from "@actions/http-client";
 import { IHeaders } from "@actions/http-client/interfaces";
 import * as toolcache from "@actions/tool-cache";
-import * as fs from "fs";
-import * as path from "path";
 import * as semver from "semver";
-import * as stream from "stream";
-import * as globalutil from "util";
-import uuidV4 from "uuid/v4";
+import { v4 as uuidV4 } from "uuid";
 
 import { getRequiredEnvParam } from "./actions-util";
 import * as api from "./api-client";
@@ -115,9 +116,22 @@ export interface ResolveQueriesOutput {
 let cachedCodeQL: CodeQL | undefined = undefined;
 
 const CODEQL_BUNDLE_VERSION = defaults.bundleVersion;
-const CODEQL_BUNDLE_NAME = "codeql-bundle.tar.gz";
 const CODEQL_DEFAULT_ACTION_REPOSITORY = "github/codeql-action";
 
+function getCodeQLBundleName(): string {
+  let platform: string;
+  if (process.platform === "win32") {
+    platform = "win64";
+  } else if (process.platform === "linux") {
+    platform = "linux64";
+  } else if (process.platform === "darwin") {
+    platform = "osx64";
+  } else {
+    return "codeql-bundle.tar.gz";
+  }
+  return `codeql-bundle-${platform}.tar.gz`;
+}
+
 function getCodeQLActionRepository(mode: util.Mode): string {
   if (mode !== "actions") {
     return CODEQL_DEFAULT_ACTION_REPOSITORY;
@@ -161,6 +175,7 @@ async function getCodeQLBundleDownloadURL(
   const uniqueDownloadSources = potentialDownloadSources.filter(
     (url, index, self) => index === self.indexOf(url)
   );
+  const codeQLBundleName = getCodeQLBundleName();
   for (const downloadSource of uniqueDownloadSources) {
     const [apiURL, repository] = downloadSource;
     // If we've reached the final case, short-circuit the API check since we know the bundle exists and is public.
@@ -180,7 +195,7 @@ async function getCodeQLBundleDownloadURL(
           tag: CODEQL_BUNDLE_VERSION,
         });
       for (const asset of release.data.assets) {
-        if (asset.name === CODEQL_BUNDLE_NAME) {
+        if (asset.name === codeQLBundleName) {
           logger.info(
             `Found CodeQL bundle in ${downloadSource[1]} on ${downloadSource[0]} with URL ${asset.url}.`
           );
@@ -193,7 +208,7 @@ async function getCodeQLBundleDownloadURL(
       );
     }
   }
-  return `https://github.com/${CODEQL_DEFAULT_ACTION_REPOSITORY}/releases/download/${CODEQL_BUNDLE_VERSION}/${CODEQL_BUNDLE_NAME}`;
+  return `https://github.com/${CODEQL_DEFAULT_ACTION_REPOSITORY}/releases/download/${CODEQL_BUNDLE_VERSION}/${codeQLBundleName}`;
 }
 
 // We have to download CodeQL manually because the toolcache doesn't support Accept headers.

src/config-utils.test.ts

@@ -1,7 +1,8 @@
-import * as github from "@actions/github";
-import test from "ava";
 import * as fs from "fs";
 import * as path from "path";
+
+import * as github from "@actions/github";
+import test from "ava";
 import sinon from "sinon";
 
 import * as api from "./api-client";

src/config-utils.ts

@@ -1,7 +1,8 @@
 import * as fs from "fs";
-import * as yaml from "js-yaml";
 import * as path from "path";
 
+import * as yaml from "js-yaml";
+
 import * as api from "./api-client";
 import { CodeQL, ResolveQueriesOutput } from "./codeql";
 import * as externalQueries from "./external-queries";

src/external-queries.test.ts

@@ -1,8 +1,9 @@
-import * as toolrunnner from "@actions/exec/lib/toolrunner";
-import test from "ava";
 import * as fs from "fs";
 import * as path from "path";
 
+import * as toolrunnner from "@actions/exec/lib/toolrunner";
+import test from "ava";
+
 import * as externalQueries from "./external-queries";
 import { getRunnerLogger } from "./logging";
 import { setupTests } from "./testing-utils";

src/external-queries.ts

@@ -1,7 +1,8 @@
-import * as toolrunnner from "@actions/exec/lib/toolrunner";
 import * as fs from "fs";
 import * as path from "path";
 
+import * as toolrunnner from "@actions/exec/lib/toolrunner";
+
 import { Logger } from "./logging";
 
 /**

src/fingerprints.test.ts

@@ -1,8 +1,9 @@
-import test from "ava";
-import * as ava from "ava";
 import * as fs from "fs";
 import * as path from "path";
 
+import * as ava from "ava";
+import test from "ava";
+
 import * as fingerprints from "./fingerprints";
 import { getRunnerLogger } from "./logging";
 import { setupTests } from "./testing-utils";

src/fingerprints.ts

@@ -1,4 +1,5 @@
 import * as fs from "fs";
+
 import Long from "long";
 
 import { Logger } from "./logging";

src/init-action.ts

@@ -3,7 +3,13 @@ import * as core from "@actions/core";
 import * as actionsUtil from "./actions-util";
 import { CodeQL } from "./codeql";
 import * as configUtils from "./config-utils";
-import { initCodeQL, initConfig, injectWindowsTracer, runInit } from "./init";
+import {
+  initCodeQL,
+  initConfig,
+  injectWindowsTracer,
+  installPythonDeps,
+  runInit,
+} from "./init";
 import { getActionsLogger } from "./logging";
 import { parseRepositoryNwo } from "./repository";
 
@@ -111,6 +117,14 @@ async function run() {
       actionsUtil.getRequiredEnvParam("GITHUB_SERVER_URL"),
       logger
     );
+
+    try {
+      await installPythonDeps(codeql, logger);
+    } catch (err) {
+      logger.warning(
+        `${err.message} You can call this action with 'setup-python-dependencies: false' to disable this process`
+      );
+    }
   } catch (e) {
     core.setFailed(e.message);
     console.log(e);

src/init.ts

@@ -1,7 +1,8 @@
-import * as toolrunnner from "@actions/exec/lib/toolrunner";
 import * as fs from "fs";
 import * as path from "path";
 
+import * as toolrunnner from "@actions/exec/lib/toolrunner";
+
 import * as analysisPaths from "./analysis-paths";
 import { CodeQL, setupCodeQL } from "./codeql";
 import * as configUtils from "./config-utils";
@@ -181,3 +182,47 @@ export async function injectWindowsTracer(
     { env: { ODASA_TRACER_CONFIGURATION: tracerConfig.spec } }
   ).exec();
 }
+
+export async function installPythonDeps(codeql: CodeQL, logger: Logger) {
+  logger.startGroup("Setup Python dependencies");
+
+  if (process.platform !== "linux") {
+    logger.info(
+      "Currently, auto-installing python dependancies is only supported on linux"
+    );
+    logger.endGroup();
+    return;
+  }
+
+  const scriptsFolder = path.resolve(__dirname, "../python-setup");
+
+  // Setup tools on the Github hosted runners
+  if (process.env["ImageOS"] !== undefined) {
+    try {
+      await new toolrunnner.ToolRunner(
+        path.join(scriptsFolder, "install_tools.sh")
+      ).exec();
+    } catch (e) {
+      // This script tries to install some needed tools in the runner. It should not fail, but if it does
+      // we just abort the process without failing the action
+      logger.endGroup();
+      logger.warning(
+        "Unable to download and extract the tools needed for installing the python dependecies. You can call this action with 'setup-python-dependencies: false' to disable this process."
+      );
+    }
+  }
+
+  // Install dependencies
+  try {
+    await new toolrunnner.ToolRunner(
+      path.join(scriptsFolder, "auto_install_packages.py"),
+      [path.dirname(codeql.getPath())]
+    ).exec();
+  } catch (e) {
+    logger.endGroup();
+    logger.warning(
+      "We were unable to install your python dependencies. You can call this action with 'setup-python-dependencies: false' to disable this process."
+    );
+  }
+  logger.endGroup();
+}

src/runner.ts

@@ -1,8 +1,9 @@
-import { Command } from "commander";
 import * as fs from "fs";
 import * as os from "os";
 import * as path from "path";
 
+import { Command } from "commander";
+
 import { runAnalyze } from "./analyze";
 import { determineAutobuildLanguage, runAutobuild } from "./autobuild";
 import { CodeQL, getCodeQL } from "./codeql";

src/tracer-config.test.ts

@@ -1,7 +1,8 @@
-import test from "ava";
 import * as fs from "fs";
 import * as path from "path";
 
+import test from "ava";
+
 import { setCodeQL } from "./codeql";
 import * as configUtils from "./config-utils";
 import { Language } from "./languages";

src/upload-lib.ts

@@ -1,10 +1,11 @@
-import * as core from "@actions/core";
-import fileUrl from "file-url";
 import * as fs from "fs";
-import * as jsonschema from "jsonschema";
 import * as path from "path";
 import zlib from "zlib";
 
+import * as core from "@actions/core";
+import fileUrl from "file-url";
+import * as jsonschema from "jsonschema";
+
 import * as api from "./api-client";
 import * as fingerprints from "./fingerprints";
 import { Logger } from "./logging";

src/upload-sarif-action.ts

@@ -45,7 +45,7 @@ async function run() {
       actionsUtil.getRequiredInput("sarif_file"),
       parseRepositoryNwo(actionsUtil.getRequiredEnvParam("GITHUB_REPOSITORY")),
       await actionsUtil.getCommitOid(),
-      actionsUtil.getRef(),
+      await actionsUtil.getRef(),
       await actionsUtil.getAnalysisKey(),
       actionsUtil.getRequiredEnvParam("GITHUB_WORKFLOW"),
       actionsUtil.getWorkflowRunID(),

src/util.test.ts

@@ -1,7 +1,8 @@
-import test from "ava";
 import * as fs from "fs";
 import * as os from "os";
 
+import test from "ava";
+
 import { getRunnerLogger } from "./logging";
 import { setupTests } from "./testing-utils";
 import * as util from "./util";