github/codeql-action
1
0
Fork 0
mirror of https://github.com/github/codeql-action.git synced 2025-12-28 10:10:17 +08:00
Code Issues Packages Projects Releases Wiki Activity

Convert rest of the actions

Browse Source
This commit is contained in:
Robert Brignull
2020-08-25 16:19:15 +01:00
parent aac5eb2aea
commit 217483dfd6
59 changed files with 1630 additions and 915 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
src/analysis-paths.ts
View File

@@ -1,6 +1,5 @@
import * as core from '@actions/core';
import * as configUtils from './config-utils';
import { Logger } from './logging';
function isInterpretedLanguage(language): boolean {
return language === 'javascript' || language === 'python';
@@ -22,6 +21,17 @@ function buildIncludeExcludeEnvVar(paths: string[]): string {
return paths.join('\n');
}
export function printPathFiltersWarning(config: configUtils.Config, logger: Logger) {
// Index include/exclude/filters only work in javascript and python.
// If any other languages are detected/configured then show a warning.
if ((config.paths.length !== 0 ||
config.pathsIgnore.length !== 0) &&
!config.languages.every(isInterpretedLanguage)) {
logger.warning('The "paths"/"paths-ignore" fields of the config only have effect for Javascript and Python');
}
}
export function includeAndExcludeAnalysisPaths(config: configUtils.Config) {
// The 'LGTM_INDEX_INCLUDE' and 'LGTM_INDEX_EXCLUDE' environment variables
// control which files/directories are traversed when scanning.
@@ -31,10 +41,10 @@ export function includeAndExcludeAnalysisPaths(config: configUtils.Config) {
// traverse the entire file tree to determine which files are matched.
// Any paths containing "*" are not included in these.
if (config.paths.length !== 0) {
core.exportVariable('LGTM_INDEX_INCLUDE', buildIncludeExcludeEnvVar(config.paths));
process.env['LGTM_INDEX_INCLUDE'] = buildIncludeExcludeEnvVar(config.paths);
}
if (config.pathsIgnore.length !== 0) {
core.exportVariable('LGTM_INDEX_EXCLUDE', buildIncludeExcludeEnvVar(config.pathsIgnore));
process.env['LGTM_INDEX_EXCLUDE'] = buildIncludeExcludeEnvVar(config.pathsIgnore);
}
// The 'LGTM_INDEX_FILTERS' environment variable controls which files are
@@ -44,15 +54,6 @@ export function includeAndExcludeAnalysisPaths(config: configUtils.Config) {
filters.push(...config.paths.map(p => 'include:' + p));
filters.push(...config.pathsIgnore.map(p => 'exclude:' + p));
if (filters.length !== 0) {
core.exportVariable('LGTM_INDEX_FILTERS', filters.join('\n'));
}
// Index include/exclude/filters only work in javascript and python.
// If any other languages are detected/configured then show a warning.
if ((config.paths.length !== 0 ||
config.pathsIgnore.length !== 0 ||
filters.length !== 0) &&
!config.languages.every(isInterpretedLanguage)) {
core.warning('The "paths"/"paths-ignore" fields of the config only have effect for Javascript and Python');
process.env['LGTM_INDEX_FILTERS'] = filters.join('\n');
}
}

163
src/analyze-action.ts
View File

@@ -1,174 +1,59 @@
import * as core from '@actions/core';
import * as fs from 'fs';
import * as path from 'path';
import { getCodeQL } from './codeql';
import * as configUtils from './config-utils';
import { isScannedLanguage } from './languages';
import { AnalysisStatusReport, runAnalyze } from './analyze';
import { getActionsLogger } from './logging';
import { parseRepositoryNwo } from './repository';
import * as sharedEnv from './shared-environment';
import * as upload_lib from './upload-lib';
import * as util from './util';
interface QueriesStatusReport {
// Time taken in ms to analyze builtin queries for cpp (or undefined if this language was not analyzed)
analyze_builtin_queries_cpp_duration_ms?: number;
// Time taken in ms to analyze builtin queries for csharp (or undefined if this language was not analyzed)
analyze_builtin_queries_csharp_duration_ms?: number;
// Time taken in ms to analyze builtin queries for go (or undefined if this language was not analyzed)
analyze_builtin_queries_go_duration_ms?: number;
// Time taken in ms to analyze builtin queries for java (or undefined if this language was not analyzed)
analyze_builtin_queries_java_duration_ms?: number;
// Time taken in ms to analyze builtin queries for javascript (or undefined if this language was not analyzed)
analyze_builtin_queries_javascript_duration_ms?: number;
// Time taken in ms to analyze builtin queries for python (or undefined if this language was not analyzed)
analyze_builtin_queries_python_duration_ms?: number;
// Time taken in ms to analyze custom queries for cpp (or undefined if this language was not analyzed)
analyze_custom_queries_cpp_duration_ms?: number;
// Time taken in ms to analyze custom queries for csharp (or undefined if this language was not analyzed)
analyze_custom_queries_csharp_duration_ms?: number;
// Time taken in ms to analyze custom queries for go (or undefined if this language was not analyzed)
analyze_custom_queries_go_duration_ms?: number;
// Time taken in ms to analyze custom queries for java (or undefined if this language was not analyzed)
analyze_custom_queries_java_duration_ms?: number;
// Time taken in ms to analyze custom queries for javascript (or undefined if this language was not analyzed)
analyze_custom_queries_javascript_duration_ms?: number;
// Time taken in ms to analyze custom queries for python (or undefined if this language was not analyzed)
analyze_custom_queries_python_duration_ms?: number;
// Name of language that errored during analysis (or undefined if no langauge failed)
analyze_failure_language?: string;
}
interface FinishStatusReport extends util.StatusReportBase, upload_lib.UploadStatusReport, QueriesStatusReport {}
interface FinishStatusReport extends util.StatusReportBase, AnalysisStatusReport {}
async function sendStatusReport(
startedAt: Date,
queriesStats: QueriesStatusReport | undefined,
uploadStats: upload_lib.UploadStatusReport | undefined,
stats: AnalysisStatusReport | undefined,
error?: Error) {
const status = queriesStats?.analyze_failure_language !== undefined || error !== undefined ? 'failure' : 'success';
const status = stats?.analyze_failure_language !== undefined || error !== undefined ? 'failure' : 'success';
const statusReportBase = await util.createStatusReportBase('finish', status, startedAt, error?.message, error?.stack);
const statusReport: FinishStatusReport = {
...statusReportBase,
...(queriesStats || {}),
...(uploadStats || {}),
...(stats || {}),
};
await util.sendStatusReport(statusReport);
}
async function createdDBForScannedLanguages(config: configUtils.Config) {
const codeql = getCodeQL(config.codeQLCmd);
for (const language of config.languages) {
if (isScannedLanguage(language)) {
core.startGroup('Extracting ' + language);
await codeql.extractScannedLanguage(util.getCodeQLDatabasePath(config.tempDir, language), language);
core.endGroup();
}
}
}
async function finalizeDatabaseCreation(config: configUtils.Config) {
await createdDBForScannedLanguages(config);
const codeql = getCodeQL(config.codeQLCmd);
for (const language of config.languages) {
core.startGroup('Finalizing ' + language);
await codeql.finalizeDatabase(util.getCodeQLDatabasePath(config.tempDir, language));
core.endGroup();
}
}
// Runs queries and creates sarif files in the given folder
async function runQueries(
sarifFolder: string,
config: configUtils.Config): Promise<QueriesStatusReport> {
const codeql = getCodeQL(config.codeQLCmd);
for (let language of config.languages) {
core.startGroup('Analyzing ' + language);
const queries = config.queries[language] || [];
if (queries.length === 0) {
throw new Error('Unable to analyse ' + language + ' as no queries were selected for this language');
}
try {
const databasePath = util.getCodeQLDatabasePath(config.tempDir, language);
// Pass the queries to codeql using a file instead of using the command
// line to avoid command line length restrictions, particularly on windows.
const querySuite = databasePath + '-queries.qls';
const querySuiteContents = queries.map(q => '- query: ' + q).join('\n');
fs.writeFileSync(querySuite, querySuiteContents);
core.debug('Query suite file for ' + language + '...\n' + querySuiteContents);
const sarifFile = path.join(sarifFolder, language + '.sarif');
await codeql.databaseAnalyze(databasePath, sarifFile, querySuite);
core.debug('SARIF results for database ' + language + ' created at "' + sarifFile + '"');
core.endGroup();
} catch (e) {
// For now the fields about query performance are not populated
return {
analyze_failure_language: language,
};
}
}
return {};
}
async function run() {
const startedAt = new Date();
let queriesStats: QueriesStatusReport | undefined = undefined;
let uploadStats: upload_lib.UploadStatusReport | undefined = undefined;
let stats: AnalysisStatusReport | undefined = undefined;
try {
util.prepareLocalRunEnvironment();
if (!await util.sendStatusReport(await util.createStatusReportBase('finish', 'starting', startedAt), true)) {
return;
}
const config = await configUtils.getConfig(util.getRequiredEnvParam('RUNNER_TEMP'));
core.exportVariable(sharedEnv.ODASA_TRACER_CONFIGURATION, '');
delete process.env[sharedEnv.ODASA_TRACER_CONFIGURATION];
const sarifFolder = core.getInput('output');
fs.mkdirSync(sarifFolder, { recursive: true });
core.info('Finalizing database creation');
await finalizeDatabaseCreation(config);
core.info('Analyzing database');
queriesStats = await runQueries(sarifFolder, config);
if ('true' === core.getInput('upload')) {
uploadStats = await upload_lib.upload(
sarifFolder,
parseRepositoryNwo(util.getRequiredEnvParam('GITHUB_REPOSITORY')),
await util.getCommitOid(),
util.getRef(),
await util.getAnalysisKey(),
util.getRequiredEnvParam('GITHUB_WORKFLOW'),
util.getWorkflowRunID(),
core.getInput('checkout_path'),
core.getInput('matrix'),
core.getInput('token'),
util.getRequiredEnvParam('GITHUB_API_URL'),
'actions',
getActionsLogger());
}
stats = await runAnalyze(
parseRepositoryNwo(util.getRequiredEnvParam('GITHUB_REPOSITORY')),
await util.getCommitOid(),
util.getRef(),
await util.getAnalysisKey(),
util.getRequiredEnvParam('GITHUB_WORKFLOW'),
util.getWorkflowRunID(),
core.getInput('checkout_path'),
core.getInput('matrix'),
core.getInput('token'),
util.getRequiredEnvParam('GITHUB_API_URL'),
core.getInput('upload') === 'true',
'actions',
core.getInput('output'),
util.getRequiredEnvParam('RUNNER_TEMP'),
getActionsLogger());
} catch (error) {
core.setFailed(error.message);
console.log(error);
await sendStatusReport(startedAt, queriesStats, uploadStats, error);
await sendStatusReport(startedAt, stats, error);
return;
}
await sendStatusReport(startedAt, queriesStats, uploadStats);
await sendStatusReport(startedAt, stats);
}
run().catch(e => {

170
src/analyze.ts Normal file
View File

@@ -0,0 +1,170 @@
import * as fs from 'fs';
import * as path from 'path';
import * as analysisPaths from './analysis-paths';
import { getCodeQL } from './codeql';
import * as configUtils from './config-utils';
import { isScannedLanguage } from './languages';
import { Logger } from './logging';
import { RepositoryNwo } from './repository';
import * as sharedEnv from './shared-environment';
import * as upload_lib from './upload-lib';
import * as util from './util';
export interface QueriesStatusReport {
// Time taken in ms to analyze builtin queries for cpp (or undefined if this language was not analyzed)
analyze_builtin_queries_cpp_duration_ms?: number;
// Time taken in ms to analyze builtin queries for csharp (or undefined if this language was not analyzed)
analyze_builtin_queries_csharp_duration_ms?: number;
// Time taken in ms to analyze builtin queries for go (or undefined if this language was not analyzed)
analyze_builtin_queries_go_duration_ms?: number;
// Time taken in ms to analyze builtin queries for java (or undefined if this language was not analyzed)
analyze_builtin_queries_java_duration_ms?: number;
// Time taken in ms to analyze builtin queries for javascript (or undefined if this language was not analyzed)
analyze_builtin_queries_javascript_duration_ms?: number;
// Time taken in ms to analyze builtin queries for python (or undefined if this language was not analyzed)
analyze_builtin_queries_python_duration_ms?: number;
// Time taken in ms to analyze custom queries for cpp (or undefined if this language was not analyzed)
analyze_custom_queries_cpp_duration_ms?: number;
// Time taken in ms to analyze custom queries for csharp (or undefined if this language was not analyzed)
analyze_custom_queries_csharp_duration_ms?: number;
// Time taken in ms to analyze custom queries for go (or undefined if this language was not analyzed)
analyze_custom_queries_go_duration_ms?: number;
// Time taken in ms to analyze custom queries for java (or undefined if this language was not analyzed)
analyze_custom_queries_java_duration_ms?: number;
// Time taken in ms to analyze custom queries for javascript (or undefined if this language was not analyzed)
analyze_custom_queries_javascript_duration_ms?: number;
// Time taken in ms to analyze custom queries for python (or undefined if this language was not analyzed)
analyze_custom_queries_python_duration_ms?: number;
// Name of language that errored during analysis (or undefined if no langauge failed)
analyze_failure_language?: string;
}
export interface AnalysisStatusReport extends upload_lib.UploadStatusReport, QueriesStatusReport {}
async function createdDBForScannedLanguages(
config: configUtils.Config,
logger: Logger) {
// Insert the LGTM_INDEX_X env vars at this point so they are set when
// we extract any scanned languages.
analysisPaths.includeAndExcludeAnalysisPaths(config);
const codeql = getCodeQL(config.codeQLCmd);
for (const language of config.languages) {
if (isScannedLanguage(language)) {
logger.startGroup('Extracting ' + language);
await codeql.extractScannedLanguage(util.getCodeQLDatabasePath(config.tempDir, language), language);
logger.endGroup();
}
}
}
async function finalizeDatabaseCreation(
config: configUtils.Config,
logger: Logger) {
await createdDBForScannedLanguages(config, logger);
const codeql = getCodeQL(config.codeQLCmd);
for (const language of config.languages) {
logger.startGroup('Finalizing ' + language);
await codeql.finalizeDatabase(util.getCodeQLDatabasePath(config.tempDir, language));
logger.endGroup();
}
}
// Runs queries and creates sarif files in the given folder
async function runQueries(
sarifFolder: string,
config: configUtils.Config,
logger: Logger): Promise<QueriesStatusReport> {
const codeql = getCodeQL(config.codeQLCmd);
for (let language of config.languages) {
logger.startGroup('Analyzing ' + language);
const queries = config.queries[language] || [];
if (queries.length === 0) {
throw new Error('Unable to analyse ' + language + ' as no queries were selected for this language');
}
try {
const databasePath = util.getCodeQLDatabasePath(config.tempDir, language);
// Pass the queries to codeql using a file instead of using the command
// line to avoid command line length restrictions, particularly on windows.
const querySuite = databasePath + '-queries.qls';
const querySuiteContents = queries.map(q => '- query: ' + q).join('\n');
fs.writeFileSync(querySuite, querySuiteContents);
logger.debug('Query suite file for ' + language + '...\n' + querySuiteContents);
const sarifFile = path.join(sarifFolder, language + '.sarif');
await codeql.databaseAnalyze(databasePath, sarifFile, querySuite);
logger.debug('SARIF results for database ' + language + ' created at "' + sarifFile + '"');
logger.endGroup();
} catch (e) {
// For now the fields about query performance are not populated
return {
analyze_failure_language: language,
};
}
}
return {};
}
export async function runAnalyze(
repositoryNwo: RepositoryNwo,
commitOid: string,
ref: string,
analysisKey: string | undefined,
analysisName: string | undefined,
workflowRunID: number | undefined,
checkoutPath: string,
environment: string | undefined,
githubAuth: string,
githubUrl: string,
doUpload: boolean,
mode: util.Mode,
outputDir: string,
tempDir: string,
logger: Logger): Promise<AnalysisStatusReport> {
const config = await configUtils.getConfig(tempDir, logger);
// Delete the tracer config env var to avoid tracing ourselves
delete process.env[sharedEnv.ODASA_TRACER_CONFIGURATION];
fs.mkdirSync(outputDir, { recursive: true });
logger.info('Finalizing database creation');
await finalizeDatabaseCreation(config, logger);
logger.info('Analyzing database');
const queriesStats = await runQueries(outputDir, config, logger);
if (!doUpload) {
logger.info('Not uploading results');
return { ...queriesStats };
}
const uploadStats = await upload_lib.upload(
outputDir,
repositoryNwo,
commitOid,
ref,
analysisKey,
analysisName,
workflowRunID,
checkoutPath,
environment,
githubAuth,
githubUrl,
mode,
logger);
return { ...queriesStats, ...uploadStats };
}

19
src/api-client.ts
View File

@@ -1,22 +1,37 @@
import * as core from "@actions/core";
import * as github from "@actions/github";
import consoleLogLevel from "console-log-level";
import * as path from 'path';
import { getRequiredEnvParam, isLocalRun } from "./util";
export const getApiClient = function(githubAuth: string, githubApiUrl: string, allowLocalRun = false) {
export const getApiClient = function(githubAuth: string, githubUrl: string, allowLocalRun = false) {
if (isLocalRun() && !allowLocalRun) {
throw new Error('Invalid API call in local run');
}
return new github.GitHub(
{
auth: parseAuth(githubAuth),
baseUrl: githubApiUrl,
baseUrl: getApiUrl(githubUrl),
userAgent: "CodeQL Action",
log: consoleLogLevel({ level: "debug" })
});
};
function getApiUrl(githubUrl: string): string {
const url = new URL(githubUrl);
// If we detect this is trying to be to github.com
// then return with a fixed canonical URL.
if (url.hostname === 'github.com' || url.hostname === 'api.github.com') {
return 'https://api.github.com';
}
// Add the /api/v3 API prefix
url.pathname = path.join(url.pathname, 'api', 'v3');
return url.toString();
}
// Parses the user input as either a single token,
// or a username and password / PAT.
function parseAuth(auth: string): string {

4
src/autobuild-action.ts
View File

@@ -3,6 +3,7 @@ import * as core from '@actions/core';
import { getCodeQL } from './codeql';
import * as config_utils from './config-utils';
import { isTracedLanguage } from './languages';
import { getActionsLogger } from './logging';
import * as util from './util';
interface AutobuildStatusReport extends util.StatusReportBase {
@@ -34,6 +35,7 @@ async function sendCompletedStatusReport(
}
async function run() {
const logger = getActionsLogger();
const startedAt = new Date();
let language;
try {
@@ -42,7 +44,7 @@ async function run() {
return;
}
const config = await config_utils.getConfig(util.getRequiredEnvParam('RUNNER_TEMP'));
const config = await config_utils.getConfig(util.getRequiredEnvParam('RUNNER_TEMP'), logger);
// Attempt to find a language to autobuild
// We want pick the dominant language in the repo from the ones we're able to build

21
src/autobuild.ts Normal file
View File

@@ -0,0 +1,21 @@
import { getCodeQL } from './codeql';
import * as config_utils from './config-utils';
import { isTracedLanguage, Language } from './languages';
import { Logger } from './logging';
export async function runAutobuild(
language: Language,
tmpDir: string,
logger: Logger) {
if (!isTracedLanguage(language)) {
throw new Error(`Cannot build "${language}" as it is not a traced language`);
}
const config = await config_utils.getConfig(tmpDir, logger);
logger.startGroup(`Attempting to automatically build ${language} code`);
const codeQL = getCodeQL(config.codeQLCmd);
await codeQL.runAutobuild(language);
logger.endGroup();
}

21
src/codeql.test.ts
View File

@@ -4,6 +4,7 @@ import nock from 'nock';
import * as path from 'path';
import * as codeql from './codeql';
import { getRunnerLogger } from './logging';
import {setupTests} from './testing-utils';
import * as util from './util';
@@ -12,12 +13,6 @@ setupTests(test);
test('download codeql bundle cache', async t => {
await util.withTmpDir(async tmpDir => {
process.env['GITHUB_WORKSPACE'] = tmpDir;
process.env['RUNNER_TEMP'] = path.join(tmpDir, 'temp');
process.env['RUNNER_TOOL_CACHE'] = path.join(tmpDir, 'cache');
const versions = ['20200601', '20200610'];
for (let i = 0; i < versions.length; i++) {
@@ -27,10 +22,14 @@ test('download codeql bundle cache', async t => {
.get(`/download/codeql-bundle-${version}/codeql-bundle.tar.gz`)
.replyWithFile(200, path.join(__dirname, `/../src/testdata/codeql-bundle.tar.gz`));
process.env['INPUT_TOOLS'] = `https://example.com/download/codeql-bundle-${version}/codeql-bundle.tar.gz`;
await codeql.setupCodeQL();
await codeql.setupCodeQL(
`https://example.com/download/codeql-bundle-${version}/codeql-bundle.tar.gz`,
'token',
'https://github.example.com',
tmpDir,
tmpDir,
'runner',
getRunnerLogger());
t.assert(toolcache.find('CodeQL', `0.0.0-${version}`));
}
@@ -56,7 +55,7 @@ test('parse codeql bundle url version', t => {
const url = `https://github.com/.../codeql-bundle-${version}/...`;
try {
const parsedVersion = codeql.getCodeQLURLVersion(url);
const parsedVersion = codeql.getCodeQLURLVersion(url, getRunnerLogger());
t.deepEqual(parsedVersion, expectedVersion);
} catch (e) {
t.fail(e.message);

107
src/codeql.ts
View File

@@ -1,4 +1,3 @@
import * as core from '@actions/core';
import * as exec from '@actions/exec';
import * as http from '@actions/http-client';
import { IHeaders } from '@actions/http-client/interfaces';
@@ -13,6 +12,7 @@ import uuidV4 from 'uuid/v4';
import * as api from './api-client';
import * as defaults from './defaults.json'; // Referenced from codeql-action-sync-tool!
import { Language } from './languages';
import { Logger } from './logging';
import * as util from './util';
type Options = (string|number|boolean)[];
@@ -101,31 +101,36 @@ const CODEQL_BUNDLE_VERSION = defaults.bundleVersion;
const CODEQL_BUNDLE_NAME = "codeql-bundle.tar.gz";
const CODEQL_DEFAULT_ACTION_REPOSITORY = "github/codeql-action";
function getCodeQLActionRepository(): string {
// Actions do not know their own repository name,
// so we currently use this hack to find the name based on where our files are.
// This can be removed once the change to the runner in https://github.com/actions/runner/pull/585 is deployed.
const runnerTemp = util.getRequiredEnvParam("RUNNER_TEMP");
const actionsDirectory = path.join(path.dirname(runnerTemp), "_actions");
const relativeScriptPath = path.relative(actionsDirectory, __filename);
// This handles the case where the Action does not come from an Action repository,
// e.g. our integration tests which use the Action code from the current checkout.
if (relativeScriptPath.startsWith("..") || path.isAbsolute(relativeScriptPath)) {
function getCodeQLActionRepository(mode: util.Mode): string {
if (mode === 'actions') {
// Actions do not know their own repository name,
// so we currently use this hack to find the name based on where our files are.
// This can be removed once the change to the runner in https://github.com/actions/runner/pull/585 is deployed.
const runnerTemp = util.getRequiredEnvParam("RUNNER_TEMP");
const actionsDirectory = path.join(path.dirname(runnerTemp), "_actions");
const relativeScriptPath = path.relative(actionsDirectory, __filename);
// This handles the case where the Action does not come from an Action repository,
// e.g. our integration tests which use the Action code from the current checkout.
if (relativeScriptPath.startsWith("..") || path.isAbsolute(relativeScriptPath)) {
return CODEQL_DEFAULT_ACTION_REPOSITORY;
}
const relativeScriptPathParts = relativeScriptPath.split(path.sep);
return relativeScriptPathParts[0] + "/" + relativeScriptPathParts[1];
} else {
return CODEQL_DEFAULT_ACTION_REPOSITORY;
}
const relativeScriptPathParts = relativeScriptPath.split(path.sep);
return relativeScriptPathParts[0] + "/" + relativeScriptPathParts[1];
}
async function getCodeQLBundleDownloadURL(): Promise<string> {
const codeQLActionRepository = getCodeQLActionRepository();
async function getCodeQLBundleDownloadURL(githubUrl: string, mode: util.Mode, logger: Logger): Promise<string> {
const codeQLActionRepository = getCodeQLActionRepository(mode);
const potentialDownloadSources = [
// This GitHub instance, and this Action.
[util.getInstanceAPIURL(), codeQLActionRepository],
[githubUrl, codeQLActionRepository],
// This GitHub instance, and the canonical Action.
[util.getInstanceAPIURL(), CODEQL_DEFAULT_ACTION_REPOSITORY],
[githubUrl, CODEQL_DEFAULT_ACTION_REPOSITORY],
// GitHub.com, and the canonical Action.
[util.GITHUB_DOTCOM_API_URL, CODEQL_DEFAULT_ACTION_REPOSITORY],
[util.GITHUB_DOTCOM_URL, CODEQL_DEFAULT_ACTION_REPOSITORY],
];
// We now filter out any duplicates.
// Duplicates will happen either because the GitHub instance is GitHub.com, or because the Action is not a fork.
@@ -133,7 +138,7 @@ async function getCodeQLBundleDownloadURL(): Promise<string> {
for (let downloadSource of uniqueDownloadSources) {
let [apiURL, repository] = downloadSource;
// If we've reached the final case, short-circuit the API check since we know the bundle exists and is public.
if (apiURL === util.GITHUB_DOTCOM_API_URL && repository === CODEQL_DEFAULT_ACTION_REPOSITORY) {
if (apiURL === util.GITHUB_DOTCOM_URL && repository === CODEQL_DEFAULT_ACTION_REPOSITORY) {
break;
}
let [repositoryOwner, repositoryName] = repository.split("/");
@@ -145,29 +150,29 @@ async function getCodeQLBundleDownloadURL(): Promise<string> {
});
for (let asset of release.data.assets) {
if (asset.name === CODEQL_BUNDLE_NAME) {
core.info(`Found CodeQL bundle in ${downloadSource[1]} on ${downloadSource[0]} with URL ${asset.url}.`);
logger.info(`Found CodeQL bundle in ${downloadSource[1]} on ${downloadSource[0]} with URL ${asset.url}.`);
return asset.url;
}
}
} catch (e) {
core.info(`Looked for CodeQL bundle in ${downloadSource[1]} on ${downloadSource[0]} but got error ${e}.`);
logger.info(`Looked for CodeQL bundle in ${downloadSource[1]} on ${downloadSource[0]} but got error ${e}.`);
}
}
return `https://github.com/${CODEQL_DEFAULT_ACTION_REPOSITORY}/releases/download/${CODEQL_BUNDLE_VERSION}/${CODEQL_BUNDLE_NAME}`;
}
// We have to download CodeQL manually because the toolcache doesn't support Accept headers.
// This can be removed once https://github.com/actions/toolkit/pull/530 is merged and released.
async function toolcacheDownloadTool(url: string, headers?: IHeaders): Promise<string> {
async function toolcacheDownloadTool(
url: string,
headers: IHeaders | undefined,
tempDir: string,
logger: Logger): Promise<string> {
const client = new http.HttpClient('CodeQL Action');
const dest = path.join(util.getRequiredEnvParam('RUNNER_TEMP'), uuidV4());
const dest = path.join(tempDir, uuidV4());
const response: http.HttpClientResponse = await client.get(url, headers);
if (response.message.statusCode !== 200) {
const err = new toolcache.HTTPError(response.message.statusCode);
core.info(
`Failed to download from "${url}". Code(${response.message.statusCode}) Message(${response.message.statusMessage})`
);
throw err;
logger.info(`Failed to download from "${url}". Code(${response.message.statusCode}) Message(${response.message.statusMessage})`);
throw new Error(`Unexpected HTTP response: ${response.message.statusCode}`);
}
const pipeline = globalutil.promisify(stream.pipeline);
fs.mkdirSync(path.dirname(dest), { recursive: true });
@@ -175,32 +180,44 @@ async function toolcacheDownloadTool(url: string, headers?: IHeaders): Promise<s
return dest;
}
export async function setupCodeQL(): Promise<CodeQL> {
export async function setupCodeQL(
codeqlURL: string | undefined,
githubAuth: string,
githubUrl: string,
tempDir: string,
toolsDir: string,
mode: util.Mode,
logger: Logger): Promise<CodeQL> {
// Setting these two env vars makes the toolcache code safe to use,
// but this is obviously not a great thing we're doing and it would
// be better to write our own implementation to use outside of actions.
process.env['RUNNER_TEMP'] = tempDir;
process.env['RUNNER_TOOL_CACHE'] = toolsDir;
try {
let codeqlURL = core.getInput('tools');
const codeqlURLVersion = getCodeQLURLVersion(codeqlURL || `/${CODEQL_BUNDLE_VERSION}/`);
const codeqlURLVersion = getCodeQLURLVersion(codeqlURL || `/${CODEQL_BUNDLE_VERSION}/`, logger);
let codeqlFolder = toolcache.find('CodeQL', codeqlURLVersion);
if (codeqlFolder) {
core.debug(`CodeQL found in cache ${codeqlFolder}`);
logger.debug(`CodeQL found in cache ${codeqlFolder}`);
} else {
if (!codeqlURL) {
codeqlURL = await getCodeQLBundleDownloadURL();
codeqlURL = await getCodeQLBundleDownloadURL(githubUrl, mode, logger);
}
const headers: IHeaders = {accept: 'application/octet-stream'};
// We only want to provide an authorization header if we are downloading
// from the same GitHub instance the Action is running on.
// This avoids leaking Enterprise tokens to dotcom.
if (codeqlURL.startsWith(util.getInstanceAPIURL() + "/")) {
core.debug('Downloading CodeQL bundle with token.');
let token = core.getInput('token', { required: true });
headers.authorization = `token ${token}`;
if (codeqlURL.startsWith(githubUrl + "/")) {
logger.debug('Downloading CodeQL bundle with token.');
headers.authorization = `token ${githubAuth}`;
} else {
core.debug('Downloading CodeQL bundle without token.');
logger.debug('Downloading CodeQL bundle without token.');
}
let codeqlPath = await toolcacheDownloadTool(codeqlURL, headers);
core.debug(`CodeQL bundle download to ${codeqlPath} complete.`);
let codeqlPath = await toolcacheDownloadTool(codeqlURL, headers, tempDir, logger);
logger.debug(`CodeQL bundle download to ${codeqlPath} complete.`);
const codeqlExtracted = await toolcache.extractTar(codeqlPath);
codeqlFolder = await toolcache.cacheDir(codeqlExtracted, 'CodeQL', codeqlURLVersion);
@@ -217,12 +234,12 @@ export async function setupCodeQL(): Promise<CodeQL> {
return cachedCodeQL;
} catch (e) {
core.error(e);
logger.error(e);
throw new Error("Unable to download and extract CodeQL CLI");
}
}
export function getCodeQLURLVersion(url: string): string {
export function getCodeQLURLVersion(url: string, logger: Logger): string {
const match = url.match(/\/codeql-bundle-(.*)\//);
if (match === null || match.length < 2) {
@@ -232,7 +249,7 @@ export function getCodeQLURLVersion(url: string): string {
let version = match[1];
if (!semver.valid(version)) {
core.debug(`Bundle version ${version} is not in SemVer format. Will treat it as pre-release 0.0.0-${version}.`);
logger.debug(`Bundle version ${version} is not in SemVer format. Will treat it as pre-release 0.0.0-${version}.`);
version = '0.0.0-' + version;
}

355
src/config-utils.test.ts
View File

@@ -8,22 +8,12 @@ import * as api from './api-client';
import { getCachedCodeQL, setCodeQL } from './codeql';
import * as configUtils from './config-utils';
import { Language } from "./languages";
import { getRunnerLogger } from "./logging";
import {setupTests} from './testing-utils';
import * as util from './util';
setupTests(test);
function setInput(name: string, value: string | undefined) {
// Transformation copied from
// https://github.com/actions/toolkit/blob/05e39f551d33e1688f61b209ab5cdd335198f1b8/packages/core/src/core.ts#L69
const envVar = `INPUT_${name.replace(/ /g, '_').toUpperCase()}`;
if (value !== undefined) {
process.env[envVar] = value;
} else {
delete process.env[envVar];
}
}
type GetContentsResponse = { content?: string; } | {}[];
function mockGetContents(content: GetContentsResponse): sinon.SinonStub<any, any> {
@@ -52,11 +42,8 @@ function mockListLanguages(languages: string[]) {
test("load empty config", async t => {
return await util.withTmpDir(async tmpDir => {
process.env['RUNNER_TEMP'] = tmpDir;
process.env['GITHUB_WORKSPACE'] = tmpDir;
setInput('config-file', undefined);
setInput('languages', 'javascript,python');
const logger = getRunnerLogger();
const languages = 'javascript,python';
const codeQL = setCodeQL({
resolveQueries: async function() {
@@ -68,19 +55,34 @@ test("load empty config", async t => {
},
});
const config = await configUtils.initConfig(tmpDir, tmpDir, codeQL);
const config = await configUtils.initConfig(
languages,
undefined,
undefined,
tmpDir,
tmpDir,
codeQL,
tmpDir,
'token',
'https://github.example.com',
logger);
t.deepEqual(config, await configUtils.getDefaultConfig(tmpDir, tmpDir, codeQL));
t.deepEqual(config, await configUtils.getDefaultConfig(
languages,
undefined,
tmpDir,
tmpDir,
codeQL,
tmpDir,
'token',
'https://github.example.com',
logger));
});
});
test("loading config saves config", async t => {
return await util.withTmpDir(async tmpDir => {
process.env['RUNNER_TEMP'] = tmpDir;
process.env['GITHUB_WORKSPACE'] = tmpDir;
setInput('config-file', undefined);
setInput('languages', 'javascript,python');
const logger = getRunnerLogger();
const codeQL = setCodeQL({
resolveQueries: async function() {
@@ -97,28 +99,43 @@ test("loading config saves config", async t => {
t.false(fs.existsSync(configUtils.getPathToParsedConfigFile(tmpDir)));
// Sanity check that getConfig throws before we have called initConfig
await t.throwsAsync(() => configUtils.getConfig(tmpDir));
await t.throwsAsync(() => configUtils.getConfig(tmpDir, logger));
const config1 = await configUtils.initConfig(tmpDir, tmpDir, codeQL);
const config1 = await configUtils.initConfig(
'javascript,python',
undefined,
undefined,
tmpDir,
tmpDir,
codeQL,
tmpDir,
'token',
'https://github.example.com',
logger);
// The saved config file should now exist
t.true(fs.existsSync(configUtils.getPathToParsedConfigFile(tmpDir)));
// And that same newly-initialised config should now be returned by getConfig
const config2 = await configUtils.getConfig(tmpDir);
const config2 = await configUtils.getConfig(tmpDir, logger);
t.deepEqual(config1, config2);
});
});
test("load input outside of workspace", async t => {
return await util.withTmpDir(async tmpDir => {
process.env['RUNNER_TEMP'] = tmpDir;
process.env['GITHUB_WORKSPACE'] = tmpDir;
setInput('config-file', '../input');
try {
await configUtils.initConfig(tmpDir, tmpDir, getCachedCodeQL());
await configUtils.initConfig(
undefined,
undefined,
'../input',
tmpDir,
tmpDir,
getCachedCodeQL(),
tmpDir,
'token',
'https://github.example.com',
getRunnerLogger());
throw new Error('initConfig did not throw error');
} catch (err) {
t.deepEqual(err, new Error(configUtils.getConfigFileOutsideWorkspaceErrorMessage(path.join(tmpDir, '../input'))));
@@ -128,14 +145,21 @@ test("load input outside of workspace", async t => {
test("load non-local input with invalid repo syntax", async t => {
return await util.withTmpDir(async tmpDir => {
process.env['RUNNER_TEMP'] = tmpDir;
process.env['GITHUB_WORKSPACE'] = tmpDir;
// no filename given, just a repo
setInput('config-file', 'octo-org/codeql-config@main');
const configFile = 'octo-org/codeql-config@main';
try {
await configUtils.initConfig(tmpDir, tmpDir, getCachedCodeQL());
await configUtils.initConfig(
undefined,
undefined,
configFile,
tmpDir,
tmpDir,
getCachedCodeQL(),
tmpDir,
'token',
'https://github.example.com',
getRunnerLogger());
throw new Error('initConfig did not throw error');
} catch (err) {
t.deepEqual(err, new Error(configUtils.getConfigFileRepoFormatInvalidMessage('octo-org/codeql-config@main')));
@@ -145,15 +169,22 @@ test("load non-local input with invalid repo syntax", async t => {
test("load non-existent input", async t => {
return await util.withTmpDir(async tmpDir => {
process.env['RUNNER_TEMP'] = tmpDir;
process.env['GITHUB_WORKSPACE'] = tmpDir;
t.false(fs.existsSync(path.join(tmpDir, 'input')));
setInput('config-file', 'input');
setInput('languages', 'javascript');
const languages = 'javascript';
const configFile = 'input';
t.false(fs.existsSync(path.join(tmpDir, configFile)));
try {
await configUtils.initConfig(tmpDir, tmpDir, getCachedCodeQL());
await configUtils.initConfig(
languages,
undefined,
configFile,
tmpDir,
tmpDir,
getCachedCodeQL(),
tmpDir,
'token',
'https://github.example.com',
getRunnerLogger());
throw new Error('initConfig did not throw error');
} catch (err) {
t.deepEqual(err, new Error(configUtils.getConfigFileDoesNotExistErrorMessage(path.join(tmpDir, 'input'))));
@@ -163,9 +194,6 @@ test("load non-existent input", async t => {
test("load non-empty input", async t => {
return await util.withTmpDir(async tmpDir => {
process.env['RUNNER_TEMP'] = tmpDir;
process.env['GITHUB_WORKSPACE'] = tmpDir;
const codeQL = setCodeQL({
resolveQueries: async function() {
return {
@@ -213,11 +241,21 @@ test("load non-empty input", async t => {
codeQLCmd: codeQL.getPath(),
};
fs.writeFileSync(path.join(tmpDir, 'input'), inputFileContents, 'utf8');
setInput('config-file', 'input');
setInput('languages', 'javascript');
const languages = 'javascript';
const configFile = 'input';
fs.writeFileSync(path.join(tmpDir, configFile), inputFileContents, 'utf8');
const actualConfig = await configUtils.initConfig(tmpDir, tmpDir, codeQL);
const actualConfig = await configUtils.initConfig(
languages,
undefined,
configFile,
tmpDir,
tmpDir,
codeQL,
tmpDir,
'token',
'https://github.example.com',
getRunnerLogger());
// Should exactly equal the object we constructed earlier
t.deepEqual(actualConfig, expectedConfig);
@@ -226,9 +264,6 @@ test("load non-empty input", async t => {
test("default queries are used", async t => {
return await util.withTmpDir(async tmpDir => {
process.env['RUNNER_TEMP'] = tmpDir;
process.env['GITHUB_WORKSPACE'] = tmpDir;
// Check that the default behaviour is to add the default queries.
// In this case if a config file is specified but does not include
// the disable-default-queries field.
@@ -260,11 +295,21 @@ test("default queries are used", async t => {
fs.mkdirSync(path.join(tmpDir, 'foo'));
fs.writeFileSync(path.join(tmpDir, 'input'), inputFileContents, 'utf8');
setInput('config-file', 'input');
setInput('languages', 'javascript');
const languages = 'javascript';
const configFile = 'input';
fs.writeFileSync(path.join(tmpDir, configFile), inputFileContents, 'utf8');
await configUtils.initConfig(tmpDir, tmpDir, codeQL);
await configUtils.initConfig(
languages,
undefined,
configFile,
tmpDir,
tmpDir,
codeQL,
tmpDir,
'token',
'https://github.example.com',
getRunnerLogger());
// Check resolve queries was called correctly
t.deepEqual(resolveQueriesArgs.length, 1);
@@ -275,16 +320,13 @@ test("default queries are used", async t => {
test("Queries can be specified in config file", async t => {
return await util.withTmpDir(async tmpDir => {
process.env['RUNNER_TEMP'] = tmpDir;
process.env['GITHUB_WORKSPACE'] = tmpDir;
const inputFileContents = `
name: my config
queries:
- uses: ./foo`;
fs.writeFileSync(path.join(tmpDir, 'input'), inputFileContents, 'utf8');
setInput('config-file', 'input');
const configFile = path.join(tmpDir, 'input');
fs.writeFileSync(configFile, inputFileContents, 'utf8');
fs.mkdirSync(path.join(tmpDir, 'foo'));
@@ -307,9 +349,19 @@ test("Queries can be specified in config file", async t => {
},
});
setInput('languages', 'javascript');
const languages = 'javascript';
const config = await configUtils.initConfig(tmpDir, tmpDir, codeQL);
const config = await configUtils.initConfig(
languages,
undefined,
configFile,
tmpDir,
tmpDir,
codeQL,
tmpDir,
'token',
'https://github.example.com',
getRunnerLogger());
// Check resolveQueries was called correctly
// It'll be called once for the default queries
@@ -327,19 +379,16 @@ test("Queries can be specified in config file", async t => {
test("Queries from config file can be overridden in workflow file", async t => {
return await util.withTmpDir(async tmpDir => {
process.env['RUNNER_TEMP'] = tmpDir;
process.env['GITHUB_WORKSPACE'] = tmpDir;
const inputFileContents = `
name: my config
queries:
- uses: ./foo`;
fs.writeFileSync(path.join(tmpDir, 'input'), inputFileContents, 'utf8');
setInput('config-file', 'input');
const configFile = path.join(tmpDir, 'input');
fs.writeFileSync(configFile, inputFileContents, 'utf8');
// This config item should take precedence over the config file but shouldn't affect the default queries.
setInput('queries', './override');
const queries = './override';
fs.mkdirSync(path.join(tmpDir, 'foo'));
fs.mkdirSync(path.join(tmpDir, 'override'));
@@ -363,9 +412,19 @@ test("Queries from config file can be overridden in workflow file", async t => {
},
});
setInput('languages', 'javascript');
const languages = 'javascript';
const config = await configUtils.initConfig(tmpDir, tmpDir, codeQL);
const config = await configUtils.initConfig(
languages,
queries,
configFile,
tmpDir,
tmpDir,
codeQL,
tmpDir,
'token',
'https://github.example.com',
getRunnerLogger());
// Check resolveQueries was called correctly
// It'll be called once for the default queries and once for `./override`,
@@ -383,13 +442,10 @@ test("Queries from config file can be overridden in workflow file", async t => {
test("Multiple queries can be specified in workflow file, no config file required", async t => {
return await util.withTmpDir(async tmpDir => {
process.env['RUNNER_TEMP'] = tmpDir;
process.env['GITHUB_WORKSPACE'] = tmpDir;
fs.mkdirSync(path.join(tmpDir, 'override1'));
fs.mkdirSync(path.join(tmpDir, 'override2'));
setInput('queries', './override1,./override2');
const queries = './override1,./override2';
const resolveQueriesArgs: {queries: string[], extraSearchPath: string | undefined}[] = [];
const codeQL = setCodeQL({
@@ -410,9 +466,19 @@ test("Multiple queries can be specified in workflow file, no config file require
},
});
setInput('languages', 'javascript');
const languages = 'javascript';
const config = await configUtils.initConfig(tmpDir, tmpDir, codeQL);
const config = await configUtils.initConfig(
languages,
queries,
undefined,
tmpDir,
tmpDir,
codeQL,
tmpDir,
'token',
'https://github.example.com',
getRunnerLogger());
// Check resolveQueries was called correctly:
// It'll be called once for the default queries,
@@ -433,11 +499,8 @@ test("Multiple queries can be specified in workflow file, no config file require
test("Invalid queries in workflow file handled correctly", async t => {
return await util.withTmpDir(async tmpDir => {
process.env['RUNNER_TEMP'] = tmpDir;
process.env['GITHUB_WORKSPACE'] = tmpDir;
setInput('queries', 'foo/bar@v1@v3');
setInput('languages', 'javascript');
const queries = 'foo/bar@v1@v3';
const languages = 'javascript';
// This function just needs to be type-correct; it doesn't need to do anything,
// since we're deliberately passing in invalid data
@@ -454,7 +517,17 @@ test("Invalid queries in workflow file handled correctly", async t => {
});
try {
await configUtils.initConfig(tmpDir, tmpDir, codeQL);
await configUtils.initConfig(
languages,
queries,
undefined,
tmpDir,
tmpDir,
codeQL,
tmpDir,
'token',
'https://github.example.com',
getRunnerLogger());
t.fail('initConfig did not throw error');
} catch (err) {
t.deepEqual(err, new Error(configUtils.getQueryUsesInvalid(undefined, "foo/bar@v1@v3")));
@@ -464,9 +537,6 @@ test("Invalid queries in workflow file handled correctly", async t => {
test("API client used when reading remote config", async t => {
return await util.withTmpDir(async tmpDir => {
process.env['RUNNER_TEMP'] = tmpDir;
process.env['GITHUB_WORKSPACE'] = tmpDir;
const codeQL = setCodeQL({
resolveQueries: async function() {
return {
@@ -501,26 +571,42 @@ test("API client used when reading remote config", async t => {
// Create checkout directory for remote queries repository
fs.mkdirSync(path.join(tmpDir, 'foo/bar/dev'), { recursive: true });
setInput('config-file', 'octo-org/codeql-config/config.yaml@main');
setInput('languages', 'javascript');
const configFile = 'octo-org/codeql-config/config.yaml@main';
const languages = 'javascript';
await configUtils.initConfig(tmpDir, tmpDir, codeQL);
await configUtils.initConfig(
languages,
undefined,
configFile,
tmpDir,
tmpDir,
codeQL,
tmpDir,
'token',
'https://github.example.com',
getRunnerLogger());
t.assert(spyGetContents.called);
});
});
test("Remote config handles the case where a directory is provided", async t => {
return await util.withTmpDir(async tmpDir => {
process.env['RUNNER_TEMP'] = tmpDir;
process.env['GITHUB_WORKSPACE'] = tmpDir;
const dummyResponse = []; // directories are returned as arrays
mockGetContents(dummyResponse);
const repoReference = 'octo-org/codeql-config/config.yaml@main';
setInput('config-file', repoReference);
try {
await configUtils.initConfig(tmpDir, tmpDir, getCachedCodeQL());
await configUtils.initConfig(
undefined,
undefined,
repoReference,
tmpDir,
tmpDir,
getCachedCodeQL(),
tmpDir,
'token',
'https://github.example.com',
getRunnerLogger());
throw new Error('initConfig did not throw error');
} catch (err) {
t.deepEqual(err, new Error(configUtils.getConfigFileDirectoryGivenMessage(repoReference)));
@@ -530,18 +616,24 @@ test("Remote config handles the case where a directory is provided", async t =>
test("Invalid format of remote config handled correctly", async t => {
return await util.withTmpDir(async tmpDir => {
process.env['RUNNER_TEMP'] = tmpDir;
process.env['GITHUB_WORKSPACE'] = tmpDir;
const dummyResponse = {
// note no "content" property here
};
mockGetContents(dummyResponse);
const repoReference = 'octo-org/codeql-config/config.yaml@main';
setInput('config-file', repoReference);
try {
await configUtils.initConfig(tmpDir, tmpDir, getCachedCodeQL());
await configUtils.initConfig(
undefined,
undefined,
repoReference,
tmpDir,
tmpDir,
getCachedCodeQL(),
tmpDir,
'token',
'https://github.example.com',
getRunnerLogger());
throw new Error('initConfig did not throw error');
} catch (err) {
t.deepEqual(err, new Error(configUtils.getConfigFileFormatInvalidMessage(repoReference)));
@@ -551,13 +643,20 @@ test("Invalid format of remote config handled correctly", async t => {
test("No detected languages", async t => {
return await util.withTmpDir(async tmpDir => {
process.env['RUNNER_TEMP'] = tmpDir;
process.env['GITHUB_WORKSPACE'] = tmpDir;
mockListLanguages([]);
try {
await configUtils.initConfig(tmpDir, tmpDir, getCachedCodeQL());
await configUtils.initConfig(
undefined,
undefined,
undefined,
tmpDir,
tmpDir,
getCachedCodeQL(),
tmpDir,
'token',
'https://github.example.com',
getRunnerLogger());
throw new Error('initConfig did not throw error');
} catch (err) {
t.deepEqual(err, new Error(configUtils.getNoLanguagesError()));
@@ -567,13 +666,20 @@ test("No detected languages", async t => {
test("Unknown languages", async t => {
return await util.withTmpDir(async tmpDir => {
process.env['RUNNER_TEMP'] = tmpDir;
process.env['GITHUB_WORKSPACE'] = tmpDir;
setInput('languages', 'ruby,english');
const languages = 'ruby,english';
try {
await configUtils.initConfig(tmpDir, tmpDir, getCachedCodeQL());
await configUtils.initConfig(
languages,
undefined,
undefined,
tmpDir,
tmpDir,
getCachedCodeQL(),
tmpDir,
'token',
'https://github.example.com',
getRunnerLogger());
throw new Error('initConfig did not throw error');
} catch (err) {
t.deepEqual(err, new Error(configUtils.getUnknownLanguagesError(['ruby', 'english'])));
@@ -588,9 +694,6 @@ function doInvalidInputTest(
test("load invalid input - " + testName, async t => {
return await util.withTmpDir(async tmpDir => {
process.env['RUNNER_TEMP'] = tmpDir;
process.env['GITHUB_WORKSPACE'] = tmpDir;
const codeQL = setCodeQL({
resolveQueries: async function() {
return {
@@ -601,13 +704,23 @@ function doInvalidInputTest(
},
});
const inputFile = path.join(tmpDir, 'input');
const languages = 'javascript';
const configFile = 'input';
const inputFile = path.join(tmpDir, configFile);
fs.writeFileSync(inputFile, inputFileContents, 'utf8');
setInput('config-file', 'input');
setInput('languages', 'javascript');
try {
await configUtils.initConfig(tmpDir, tmpDir, codeQL);
await configUtils.initConfig(
languages,
undefined,
configFile,
tmpDir,
tmpDir,
codeQL,
tmpDir,
'token',
'https://github.example.com',
getRunnerLogger());
throw new Error('initConfig did not throw error');
} catch (err) {
t.deepEqual(err, new Error(expectedErrorMessageGenerator(inputFile)));
@@ -715,10 +828,10 @@ test('path validations', t => {
const configFile = './.github/codeql/config.yml';
for (const path of validPaths) {
t.truthy(configUtils.validateAndSanitisePath(path, propertyName, configFile));
t.truthy(configUtils.validateAndSanitisePath(path, propertyName, configFile, getRunnerLogger()));
}
for (const path of invalidPaths) {
t.throws(() => configUtils.validateAndSanitisePath(path, propertyName, configFile));
t.throws(() => configUtils.validateAndSanitisePath(path, propertyName, configFile, getRunnerLogger()));
}
});
@@ -729,11 +842,11 @@ test('path sanitisation', t => {
// Valid paths are not modified
t.deepEqual(
configUtils.validateAndSanitisePath('foo/bar', propertyName, configFile),
configUtils.validateAndSanitisePath('foo/bar', propertyName, configFile, getRunnerLogger()),
'foo/bar');
// Trailing stars are stripped
t.deepEqual(
configUtils.validateAndSanitisePath('foo/**', propertyName, configFile),
configUtils.validateAndSanitisePath('foo/**', propertyName, configFile, getRunnerLogger()),
'foo/');
});

224
src/config-utils.ts
View File

@@ -1,4 +1,3 @@
import * as core from '@actions/core';
import * as fs from 'fs';
import * as yaml from 'js-yaml';
import * as path from 'path';
@@ -7,7 +6,7 @@ import * as api from './api-client';
import { CodeQL, ResolveQueriesOutput } from './codeql';
import * as externalQueries from "./external-queries";
import { Language, parseLanguage } from "./languages";
import * as util from './util';
import { Logger } from './logging';
// Property names from the user-supplied config file.
const NAME_PROPERTY = 'name';
@@ -182,12 +181,12 @@ async function addLocalQueries(
codeQL: CodeQL,
resultMap: { [language: string]: string[] },
localQueryPath: string,
checkoutPath: string,
configFile?: string) {
// Resolve the local path against the workspace so that when this is
// passed to codeql it resolves to exactly the path we expect it to resolve to.
const workspacePath = fs.realpathSync(util.getRequiredEnvParam('GITHUB_WORKSPACE'));
let absoluteQueryPath = path.join(workspacePath, localQueryPath);
let absoluteQueryPath = path.join(checkoutPath, localQueryPath);
// Check the file exists
if (!fs.existsSync(absoluteQueryPath)) {
@@ -198,14 +197,11 @@ async function addLocalQueries(
absoluteQueryPath = fs.realpathSync(absoluteQueryPath);
// Check the local path doesn't jump outside the repo using '..' or symlinks
if (!(absoluteQueryPath + path.sep).startsWith(workspacePath + path.sep)) {
if (!(absoluteQueryPath + path.sep).startsWith(fs.realpathSync(checkoutPath) + path.sep)) {
throw new Error(getLocalPathOutsideOfRepository(configFile, localQueryPath));
}
// Get the root of the current repo to use when resolving query dependencies
const rootOfRepo = util.getRequiredEnvParam('GITHUB_WORKSPACE');
await runResolveQueries(codeQL, resultMap, [absoluteQueryPath], rootOfRepo, true);
await runResolveQueries(codeQL, resultMap, [absoluteQueryPath], checkoutPath, true);
}
/**
@@ -216,6 +212,7 @@ async function addRemoteQueries(
resultMap: { [language: string]: string[] },
queryUses: string,
tempDir: string,
githubUrl: string,
configFile?: string) {
let tok = queryUses.split('@');
@@ -239,13 +236,17 @@ async function addRemoteQueries(
const nwo = tok[0] + '/' + tok[1];
// Checkout the external repository
const rootOfRepo = await externalQueries.checkoutExternalRepository(nwo, ref, tempDir);
const checkoutPath = await externalQueries.checkoutExternalRepository(
nwo,
ref,
githubUrl,
tempDir);
const queryPath = tok.length > 2
? path.join(rootOfRepo, tok.slice(2).join('/'))
: rootOfRepo;
? path.join(checkoutPath, tok.slice(2).join('/'))
: checkoutPath;
await runResolveQueries(codeQL, resultMap, [queryPath], rootOfRepo, true);
await runResolveQueries(codeQL, resultMap, [queryPath], checkoutPath, true);
}
/**
@@ -262,6 +263,8 @@ async function parseQueryUses(
resultMap: { [language: string]: string[] },
queryUses: string,
tempDir: string,
checkoutPath: string,
githubUrl: string,
configFile?: string) {
queryUses = queryUses.trim();
@@ -271,7 +274,7 @@ async function parseQueryUses(
// Check for the local path case before we start trying to parse the repository name
if (queryUses.startsWith("./")) {
await addLocalQueries(codeQL, resultMap, queryUses.slice(2), configFile);
await addLocalQueries(codeQL, resultMap, queryUses.slice(2), checkoutPath, configFile);
return;
}
@@ -282,7 +285,7 @@ async function parseQueryUses(
}
// Otherwise, must be a reference to another repo
await addRemoteQueries(codeQL, resultMap, queryUses, tempDir, configFile);
await addRemoteQueries(codeQL, resultMap, queryUses, tempDir, githubUrl, configFile);
}
// Regex validating stars in paths or paths-ignore entries.
@@ -299,7 +302,8 @@ const filterPatternCharactersRegex = /.*[\?\+\[\]!].*/;
export function validateAndSanitisePath(
originalPath: string,
propertyName: string,
configFile: string): string {
configFile: string,
logger: Logger): string {
// Take a copy so we don't modify the original path, so we can still construct error messages
let path = originalPath;
@@ -335,7 +339,7 @@ export function validateAndSanitisePath(
// Check for other regex characters that we don't support.
// Output a warning so the user knows, but otherwise continue normally.
if (path.match(filterPatternCharactersRegex)) {
core.warning(getConfigFilePropertyError(
logger.warning(getConfigFilePropertyError(
configFile,
propertyName,
'"' + originalPath + '" contains an unsupported character. ' +
@@ -445,19 +449,23 @@ export function getUnknownLanguagesError(languages: string[]): string {
/**
* Gets the set of languages in the current repository
*/
async function getLanguagesInRepo(): Promise<Language[]> {
async function getLanguagesInRepo(
githubAuth: string,
githubUrl: string,
logger: Logger): Promise<Language[]> {
let repo_nwo = process.env['GITHUB_REPOSITORY']?.split("/");
if (repo_nwo) {
let owner = repo_nwo[0];
let repo = repo_nwo[1];
core.debug(`GitHub repo ${owner} ${repo}`);
const response = await api.getActionsApiClient(true).repos.listLanguages({
logger.debug(`GitHub repo ${owner} ${repo}`);
const response = await api.getApiClient(githubAuth, githubUrl, true).repos.listLanguages({
owner,
repo
});
core.debug("Languages API response: " + JSON.stringify(response));
logger.debug("Languages API response: " + JSON.stringify(response));
// The GitHub API is going to return languages in order of popularity,
// When we pick a language to autobuild we want to pick the most popular traced language
@@ -486,19 +494,26 @@ async function getLanguagesInRepo(): Promise<Language[]> {
* If no languages could be detected from either the workflow or the repository
* then throw an error.
*/
async function getLanguages(): Promise<Language[]> {
async function getLanguages(
languagesInput: string | undefined,
githubAuth: string,
githubUrl: string,
logger: Logger): Promise<Language[]> {
// Obtain from action input 'languages' if set
let languages = core.getInput('languages', { required: false })
let languages = (languagesInput || "")
.split(',')
.map(x => x.trim())
.filter(x => x.length > 0);
core.info("Languages from configuration: " + JSON.stringify(languages));
logger.info("Languages from configuration: " + JSON.stringify(languages));
if (languages.length === 0) {
// Obtain languages as all languages in the repo that can be analysed
languages = await getLanguagesInRepo();
core.info("Automatically detected languages: " + JSON.stringify(languages));
languages = await getLanguagesInRepo(
githubAuth,
githubUrl,
logger);
logger.info("Automatically detected languages: " + JSON.stringify(languages));
}
// If the languages parameter was not given and no languages were
@@ -529,31 +544,44 @@ async function getLanguages(): Promise<Language[]> {
* Returns true if queries were provided in the workflow file
* (and thus added), otherwise false
*/
async function addQueriesFromWorkflowIfRequired(
async function addQueriesFromWorkflow(
codeQL: CodeQL,
queriesInput: string,
languages: string[],
resultMap: { [language: string]: string[] },
tempDir: string
): Promise<boolean> {
const queryUses = core.getInput('queries');
if (queryUses) {
for (const query of queryUses.split(',')) {
await parseQueryUses(languages, codeQL, resultMap, query, tempDir);
}
return true;
}
tempDir: string,
checkoutPath: string,
githubUrl: string) {
return false;
for (const query of queriesInput.split(',')) {
await parseQueryUses(languages, codeQL, resultMap, query, tempDir, checkoutPath, githubUrl);
}
}
/**
* Get the default config for when the user has not supplied one.
*/
export async function getDefaultConfig(tempDir: string, toolCacheDir: string, codeQL: CodeQL): Promise<Config> {
const languages = await getLanguages();
export async function getDefaultConfig(
languagesInput: string | undefined,
queriesInput: string | undefined,
tempDir: string,
toolCacheDir: string,
codeQL: CodeQL,
checkoutPath: string,
githubAuth: string,
githubUrl: string,
logger: Logger): Promise<Config> {
const languages = await getLanguages(
languagesInput,
githubAuth,
githubUrl,
logger);
const queries = {};
await addDefaultQueries(codeQL, languages, queries);
await addQueriesFromWorkflowIfRequired(codeQL, languages, queries, tempDir);
if (queriesInput) {
await addQueriesFromWorkflow(codeQL, queriesInput, languages, queries, tempDir, checkoutPath, githubUrl);
}
return {
languages: languages,
@@ -570,17 +598,29 @@ export async function getDefaultConfig(tempDir: string, toolCacheDir: string, co
/**
* Load the config from the given file.
*/
async function loadConfig(configFile: string, tempDir: string, toolCacheDir: string, codeQL: CodeQL): Promise<Config> {
async function loadConfig(
languagesInput: string | undefined,
queriesInput: string | undefined,
configFile: string,
tempDir: string,
toolCacheDir: string,
codeQL: CodeQL,
checkoutPath: string,
githubAuth: string,
githubUrl: string,
logger: Logger): Promise<Config> {
let parsedYAML: UserConfig;
if (isLocal(configFile)) {
// Treat the config file as relative to the workspace
const workspacePath = util.getRequiredEnvParam('GITHUB_WORKSPACE');
configFile = path.resolve(workspacePath, configFile);
parsedYAML = getLocalConfig(configFile, workspacePath);
configFile = path.resolve(checkoutPath, configFile);
parsedYAML = getLocalConfig(configFile, checkoutPath);
} else {
parsedYAML = await getRemoteConfig(configFile);
parsedYAML = await getRemoteConfig(
configFile,
githubAuth,
githubUrl);
}
// Validate that the 'name' property is syntactically correct,
@@ -594,7 +634,11 @@ async function loadConfig(configFile: string, tempDir: string, toolCacheDir: str
}
}
const languages = await getLanguages();
const languages = await getLanguages(
languagesInput,
githubAuth,
githubUrl,
logger);
const queries = {};
const pathsIgnore: string[] = [];
@@ -613,8 +657,9 @@ async function loadConfig(configFile: string, tempDir: string, toolCacheDir: str
// If queries were provided using `with` in the action configuration,
// they should take precedence over the queries in the config file
const addedQueriesFromAction = await addQueriesFromWorkflowIfRequired(codeQL, languages, queries, tempDir);
if (!addedQueriesFromAction && QUERIES_PROPERTY in parsedYAML) {
if (queriesInput) {
await addQueriesFromWorkflow(codeQL, queriesInput, languages, queries, tempDir, checkoutPath, githubUrl);
} else if (QUERIES_PROPERTY in parsedYAML) {
if (!(parsedYAML[QUERIES_PROPERTY] instanceof Array)) {
throw new Error(getQueriesInvalid(configFile));
}
@@ -622,7 +667,15 @@ async function loadConfig(configFile: string, tempDir: string, toolCacheDir: str
if (!(QUERIES_USES_PROPERTY in query) || typeof query[QUERIES_USES_PROPERTY] !== "string") {
throw new Error(getQueryUsesInvalid(configFile));
}
await parseQueryUses(languages, codeQL, queries, query[QUERIES_USES_PROPERTY], tempDir, configFile);
await parseQueryUses(
languages,
codeQL,
queries,
query[QUERIES_USES_PROPERTY],
tempDir,
checkoutPath,
githubUrl,
configFile);
}
}
@@ -634,7 +687,7 @@ async function loadConfig(configFile: string, tempDir: string, toolCacheDir: str
if (typeof path !== "string" || path === '') {
throw new Error(getPathsIgnoreInvalid(configFile));
}
pathsIgnore.push(validateAndSanitisePath(path, PATHS_IGNORE_PROPERTY, configFile));
pathsIgnore.push(validateAndSanitisePath(path, PATHS_IGNORE_PROPERTY, configFile, logger));
});
}
@@ -646,7 +699,7 @@ async function loadConfig(configFile: string, tempDir: string, toolCacheDir: str
if (typeof path !== "string" || path === '') {
throw new Error(getPathsInvalid(configFile));
}
paths.push(validateAndSanitisePath(path, PATHS_PROPERTY, configFile));
paths.push(validateAndSanitisePath(path, PATHS_PROPERTY, configFile, logger));
});
}
@@ -677,20 +730,49 @@ async function loadConfig(configFile: string, tempDir: string, toolCacheDir: str
* This will parse the config from the user input if present, or generate
* a default config. The parsed config is then stored to a known location.
*/
export async function initConfig(tempDir: string, toolCacheDir: string, codeQL: CodeQL): Promise<Config> {
const configFile = core.getInput('config-file');
export async function initConfig(
languagesInput: string | undefined,
queriesInput: string | undefined,
configFile: string | undefined,
tempDir: string,
toolCacheDir: string,
codeQL: CodeQL,
checkoutPath: string,
githubAuth: string,
githubUrl: string,
logger: Logger): Promise<Config> {
let config: Config;
// If no config file was provided create an empty one
if (configFile === '') {
core.debug('No configuration file was provided');
config = await getDefaultConfig(tempDir, toolCacheDir, codeQL);
if (!configFile) {
logger.debug('No configuration file was provided');
config = await getDefaultConfig(
languagesInput,
queriesInput,
tempDir,
toolCacheDir,
codeQL,
checkoutPath,
githubAuth,
githubUrl,
logger);
} else {
config = await loadConfig(configFile, tempDir, toolCacheDir, codeQL);
config = await loadConfig(
languagesInput,
queriesInput,
configFile,
tempDir,
toolCacheDir,
codeQL,
checkoutPath,
githubAuth,
githubUrl,
logger);
}
// Save the config so we can easily access it again in the future
await saveConfig(config);
await saveConfig(config, logger);
return config;
}
@@ -703,9 +785,9 @@ function isLocal(configPath: string): boolean {
return (configPath.indexOf("@") === -1);
}
function getLocalConfig(configFile: string, workspacePath: string): UserConfig {
function getLocalConfig(configFile: string, checkoutPath: string): UserConfig {
// Error if the config file is now outside of the workspace
if (!(configFile + path.sep).startsWith(workspacePath + path.sep)) {
if (!(configFile + path.sep).startsWith(checkoutPath + path.sep)) {
throw new Error(getConfigFileOutsideWorkspaceErrorMessage(configFile));
}
@@ -717,7 +799,11 @@ function getLocalConfig(configFile: string, workspacePath: string): UserConfig {
return yaml.safeLoad(fs.readFileSync(configFile, 'utf8'));
}
async function getRemoteConfig(configFile: string): Promise<UserConfig> {
async function getRemoteConfig(
configFile: string,
githubAuth: string,
githubUrl: string): Promise<UserConfig> {
// retrieve the various parts of the config location, and ensure they're present
const format = new RegExp('(?<owner>[^/]+)/(?<repo>[^/]+)/(?<path>[^@]+)@(?<ref>.*)');
const pieces = format.exec(configFile);
@@ -726,7 +812,7 @@ async function getRemoteConfig(configFile: string): Promise<UserConfig> {
throw new Error(getConfigFileRepoFormatInvalidMessage(configFile));
}
const response = await api.getActionsApiClient(true).repos.getContents({
const response = await api.getApiClient(githubAuth, githubUrl, true).repos.getContents({
owner: pieces.groups.owner,
repo: pieces.groups.repo,
path: pieces.groups.path,
@@ -755,13 +841,13 @@ export function getPathToParsedConfigFile(tempDir: string): string {
/**
* Store the given config to the path returned from getPathToParsedConfigFile.
*/
async function saveConfig(config: Config) {
async function saveConfig(config: Config, logger: Logger) {
const configString = JSON.stringify(config);
const configFile = getPathToParsedConfigFile(config.tempDir);
fs.mkdirSync(path.dirname(configFile), { recursive: true });
fs.writeFileSync(configFile, configString, 'utf8');
core.debug('Saved config:');
core.debug(configString);
logger.debug('Saved config:');
logger.debug(configString);
}
/**
@@ -772,13 +858,13 @@ async function saveConfig(config: Config) {
* stored to a known location. On the second and further calls, this will
* return the contents of the parsed config from the known location.
*/
export async function getConfig(tempDir: string): Promise<Config> {
export async function getConfig(tempDir: string, logger: Logger): Promise<Config> {
const configFile = getPathToParsedConfigFile(tempDir);
if (!fs.existsSync(configFile)) {
throw new Error("Config file could not be found at expected location. Has the 'init' action been called?");
}
const configString = fs.readFileSync(configFile, 'utf8');
core.debug('Loaded config:');
core.debug(configString);
logger.debug('Loaded config:');
logger.debug(configString);
return JSON.parse(configString);
}

1
src/external-queries.test.ts
View File

@@ -14,6 +14,7 @@ test("checkoutExternalQueries", async t => {
await externalQueries.checkoutExternalRepository(
"github/codeql-go",
ref,
'https://github.com',
tmpDir);
// COPYRIGHT file existed in df4c6869212341b601005567381944ed90906b6b but not in the default branch

9
src/external-queries.ts
View File

@@ -6,7 +6,12 @@ import * as path from 'path';
/**
* Check out repository at the given ref, and return the directory of the checkout.
*/
export async function checkoutExternalRepository(repository: string, ref: string, tempDir: string): Promise<string> {
export async function checkoutExternalRepository(
repository: string,
ref: string,
githubUrl: string,
tempDir: string): Promise<string> {
core.info('Checking out ' + repository);
const checkoutLocation = path.join(tempDir, repository, ref);
@@ -17,7 +22,7 @@ export async function checkoutExternalRepository(repository: string, ref: string
}
if (!fs.existsSync(checkoutLocation)) {
const repoURL = 'https://github.com/' + repository + '.git';
const repoURL = githubUrl + '/' + repository + '.git';
await exec.exec('git', ['clone', repoURL, checkoutLocation]);
await exec.exec('git', [
'--work-tree=' + checkoutLocation,

12
src/fingerprints.test.ts
View File

@@ -116,7 +116,7 @@ test('hash', (t: ava.Assertions) => {
function testResolveUriToFile(uri: any, index: any, artifactsURIs: any[]) {
const location = { "uri": uri, "index": index };
const artifacts = artifactsURIs.map(uri => ({ "location": { "uri": uri } }));
return fingerprints.resolveUriToFile(location, artifacts, getRunnerLogger());
return fingerprints.resolveUriToFile(location, artifacts, process.cwd(), getRunnerLogger());
}
test('resolveUriToFile', t => {
@@ -129,8 +129,6 @@ test('resolveUriToFile', t => {
t.true(filepath.startsWith(cwd + '/'));
const relativeFilepaht = filepath.substring(cwd.length + 1);
process.env['GITHUB_WORKSPACE'] = cwd;
// Absolute paths are unmodified
t.is(testResolveUriToFile(filepath, undefined, []), filepath);
t.is(testResolveUriToFile('file://' + filepath, undefined, []), filepath);
@@ -173,9 +171,9 @@ test('addFingerprints', t => {
expected = JSON.stringify(JSON.parse(expected));
// The URIs in the SARIF files resolve to files in the testdata directory
process.env['GITHUB_WORKSPACE'] = path.normalize(__dirname + '/../src/testdata');
const checkoutPath = path.normalize(__dirname + '/../src/testdata');
t.deepEqual(fingerprints.addFingerprints(input, getRunnerLogger()), expected);
t.deepEqual(fingerprints.addFingerprints(input, checkoutPath, getRunnerLogger()), expected);
});
test('missingRegions', t => {
@@ -188,7 +186,7 @@ test('missingRegions', t => {
expected = JSON.stringify(JSON.parse(expected));
// The URIs in the SARIF files resolve to files in the testdata directory
process.env['GITHUB_WORKSPACE'] = path.normalize(__dirname + '/../src/testdata');
const checkoutPath = path.normalize(__dirname + '/../src/testdata');
t.deepEqual(fingerprints.addFingerprints(input, getRunnerLogger()), expected);
t.deepEqual(fingerprints.addFingerprints(input, checkoutPath, getRunnerLogger()), expected);
});

17
src/fingerprints.ts
View File

@@ -161,7 +161,12 @@ function locationUpdateCallback(result: any, location: any, logger: Logger): has
// the source file so we can hash it.
// If possible returns a absolute file path for the source file,
// or if not possible then returns undefined.
export function resolveUriToFile(location: any, artifacts: any[], logger: Logger): string | undefined {
export function resolveUriToFile(
location: any,
artifacts: any[],
checkoutPath: string,
logger: Logger): string | undefined {
// This may be referencing an artifact
if (!location.uri && location.index !== undefined) {
if (typeof location.index !== 'number' ||
@@ -192,7 +197,7 @@ export function resolveUriToFile(location: any, artifacts: any[], logger: Logger
}
// Discard any absolute paths that aren't in the src root
const srcRootPrefix = process.env['GITHUB_WORKSPACE'] + '/';
const srcRootPrefix = checkoutPath + '/';
if (uri.startsWith('/') && !uri.startsWith(srcRootPrefix)) {
logger.debug(`Ignoring location URI "${uri}" as it is outside of the src root`);
return undefined;
@@ -216,7 +221,7 @@ export function resolveUriToFile(location: any, artifacts: any[], logger: Logger
// Compute fingerprints for results in the given sarif file
// and return an updated sarif file contents.
export function addFingerprints(sarifContents: string, logger: Logger): string {
export function addFingerprints(sarifContents: string, checkoutPath: string, logger: Logger): string {
let sarif = JSON.parse(sarifContents);
// Gather together results for the same file and construct
@@ -234,7 +239,11 @@ export function addFingerprints(sarifContents: string, logger: Logger): string {
continue;
}
const filepath = resolveUriToFile(primaryLocation.physicalLocation.artifactLocation, artifacts, logger);
const filepath = resolveUriToFile(
primaryLocation.physicalLocation.artifactLocation,
artifacts,
checkoutPath,
logger);
if (!filepath) {
continue;
}

61
src/init-action.ts
View File

@@ -1,12 +1,9 @@
import * as core from '@actions/core';
import * as exec from '@actions/exec';
import * as fs from 'fs';
import * as path from 'path';
import * as analysisPaths from './analysis-paths';
import { CodeQL, setupCodeQL } from './codeql';
import { CodeQL } from './codeql';
import * as configUtils from './config-utils';
import { getCombinedTracerConfig } from './tracer-config';
import { initCodeQL, initConfig, runInit } from './init';
import { getActionsLogger } from './logging';
import * as util from './util';
interface InitSuccessStatusReport extends util.StatusReportBase {
@@ -49,8 +46,8 @@ async function sendSuccessStatusReport(startedAt: Date, config: configUtils.Conf
}
async function run() {
const startedAt = new Date();
const logger = getActionsLogger();
let config: configUtils.Config;
let codeql: CodeQL;
@@ -60,18 +57,25 @@ async function run() {
return;
}
core.startGroup('Setup CodeQL tools');
codeql = await setupCodeQL();
await codeql.printVersion();
core.endGroup();
core.startGroup('Load language configuration');
config = await configUtils.initConfig(
codeql = await initCodeQL(
core.getInput('tools'),
core.getInput('token'),
util.getRequiredEnvParam('GITHUB_API_URL'),
util.getRequiredEnvParam('RUNNER_TEMP'),
util.getRequiredEnvParam('RUNNER_TOOL_CACHE'),
codeql);
analysisPaths.includeAndExcludeAnalysisPaths(config);
core.endGroup();
'actions',
logger);
config = await initConfig(
core.getInput('languages'),
core.getInput('queries'),
core.getInput('config-file'),
util.getRequiredEnvParam('RUNNER_TEMP'),
util.getRequiredEnvParam('RUNNER_TOOL_CACHE'),
codeql,
util.getRequiredEnvParam('GITHUB_WORKSPACE'),
core.getInput('token'),
util.getRequiredEnvParam('GITHUB_API_URL'),
logger);
} catch (e) {
core.setFailed(e.message);
@@ -82,8 +86,6 @@ async function run() {
try {
const sourceRoot = path.resolve();
// Forward Go flags
const goFlags = process.env['GOFLAGS'];
if (goFlags) {
@@ -95,27 +97,8 @@ async function run() {
const codeqlRam = process.env['CODEQL_RAM'] || '6500';
core.exportVariable('CODEQL_RAM', codeqlRam);
fs.mkdirSync(util.getCodeQLDatabasesDir(config.tempDir), { recursive: true });
// TODO: replace this code once CodeQL supports multi-language tracing
for (let language of config.languages) {
// Init language database
await codeql.databaseInit(util.getCodeQLDatabasePath(config.tempDir, language), language, sourceRoot);
}
const tracerConfig = await getCombinedTracerConfig(config, codeql);
const tracerConfig = await runInit(codeql, config);
if (tracerConfig !== undefined) {
if (process.platform === 'win32') {
await exec.exec(
'powershell',
[
path.resolve(__dirname, '..', 'src', 'inject-tracer.ps1'),
path.resolve(path.dirname(codeql.getPath()), 'tools', 'win64', 'tracer.exe'),
],
{ env: { 'ODASA_TRACER_CONFIGURATION': tracerConfig.spec } });
}
// NB: in CLI mode these will be output to a file rather than exported with core.exportVariable
Object.entries(tracerConfig.env).forEach(([key, value]) => core.exportVariable(key, value));
}

91
src/init.ts Normal file
View File

@@ -0,0 +1,91 @@
import * as exec from '@actions/exec';
import * as fs from 'fs';
import * as path from 'path';
import * as analysisPaths from './analysis-paths';
import { CodeQL, setupCodeQL } from './codeql';
import * as configUtils from './config-utils';
import { Logger } from './logging';
import { getCombinedTracerConfig, TracerConfig } from './tracer-config';
import * as util from './util';
export async function initCodeQL(
codeqlURL: string | undefined,
githubAuth: string,
githubUrl: string,
tempDir: string,
toolsDir: string,
mode: util.Mode,
logger: Logger): Promise<CodeQL> {
logger.startGroup('Setup CodeQL tools');
const codeql = await setupCodeQL(
codeqlURL,
githubAuth,
githubUrl,
tempDir,
toolsDir,
mode,
logger);
await codeql.printVersion();
logger.endGroup();
return codeql;
}
export async function initConfig(
languagesInput: string | undefined,
queriesInput: string | undefined,
configFile: string | undefined,
tempDir: string,
toolCacheDir: string,
codeQL: CodeQL,
checkoutPath: string,
githubAuth: string,
githubUrl: string,
logger: Logger): Promise<configUtils.Config> {
logger.startGroup('Load language configuration');
const config = await configUtils.initConfig(
languagesInput,
queriesInput,
configFile,
tempDir,
toolCacheDir,
codeQL,
checkoutPath,
githubAuth,
githubUrl,
logger);
analysisPaths.printPathFiltersWarning(config, logger);
logger.endGroup();
return config;
}
export async function runInit(
codeql: CodeQL,
config: configUtils.Config): Promise<TracerConfig | undefined> {
const sourceRoot = path.resolve();
fs.mkdirSync(util.getCodeQLDatabasesDir(config.tempDir), { recursive: true });
// TODO: replace this code once CodeQL supports multi-language tracing
for (let language of config.languages) {
// Init language database
await codeql.databaseInit(util.getCodeQLDatabasePath(config.tempDir, language), language, sourceRoot);
}
const tracerConfig = await getCombinedTracerConfig(config, codeql);
if (tracerConfig !== undefined) {
if (process.platform === 'win32') {
await exec.exec(
'powershell',
[
path.resolve(__dirname, '..', 'src', 'inject-tracer.ps1'),
path.resolve(path.dirname(codeql.getPath()), 'tools', 'win64', 'tracer.exe'),
],
{ env: { 'ODASA_TRACER_CONFIGURATION': tracerConfig.spec } });
}
}
return tracerConfig;
}

219
src/runner.ts
View File

@@ -1,6 +1,13 @@
import { Command } from 'commander';
import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
import { runAnalyze } from './analyze';
import { runAutobuild } from './autobuild';
import { CodeQL, getCodeQL } from './codeql';
import { initCodeQL, initConfig, runInit } from './init';
import { parseLanguage } from './languages';
import { getRunnerLogger } from './logging';
import { parseRepositoryNwo } from './repository';
import * as upload_lib from './upload-lib';
@@ -8,6 +15,192 @@ import * as upload_lib from './upload-lib';
const program = new Command();
program.version('0.0.1');
function parseGithubUrl(inputUrl: string): string {
try {
const url = new URL(inputUrl);
// If we detect this is trying to be to github.com
// then return with a fixed canonical URL.
if (url.hostname === 'github.com' || url.hostname === 'api.github.com') {
return 'https://github.com';
}
// Remove the API prefix if it's present
if (url.pathname.indexOf('/api/v3') !== -1) {
url.pathname = url.pathname.substring(0, url.pathname.indexOf('/api/v3'));
}
return url.toString();
} catch (e) {
throw new Error(`"${inputUrl}" is not a valid URL`);
}
}
function getTempDir(userInput: string | undefined): string {
const tempDir = path.join(userInput || os.tmpdir(), 'codeql-runner-temp');
if (!fs.existsSync(tempDir)) {
fs.mkdirSync(tempDir, { recursive: true });
}
return tempDir;
}
function getToolsDir(userInput: string | undefined, tmpDir: string): string {
const toolsDir = path.join(userInput || path.dirname(tmpDir), 'codeql-runner-tools');
if (!fs.existsSync(toolsDir)) {
fs.mkdirSync(toolsDir, { recursive: true });
}
return toolsDir;
}
const logger = getRunnerLogger();
interface InitArgs {
languages: string | undefined;
queries: string | undefined;
configFile: string | undefined;
codeqlPath: string | undefined;
tempDir: string | undefined;
toolsDir: string | undefined;
checkoutPath: string | undefined;
githubUrl: string;
githubAuth: string;
}
program
.command('init')
.description('Initializes CodeQL')
.requiredOption('--github-url <url>', 'URL of GitHub instance')
.requiredOption('--github-auth <auth>', 'GitHub Apps token, or of the form "username:token" if using a personal access token')
.option('--languages <languages>', 'Comma-separated list of languages to analyze. Defaults to trying to detect languages from the repo.')
.option('--queries <queries>', 'Comma-separated list of additional queries to run. By default, this overrides the same setting in a configuration file.')
.option('--config-file <file>', 'Path to config file')
.option('--codeql-path <path>', 'Path to a copy of the CodeQL CLI executable to use. Otherwise downloads a copy.')
.option('--temp-dir <dir>', 'Directory to use for temporary files. Defaults to OS temp dir.')
.option('--tools-dir <dir>', 'Directory to use for CodeQL tools and other files to store between runs. Defaults to same as temp dir.')
.option('--checkout-path <path>', 'Checkout path (default: current working directory)')
.action(async (cmd: InitArgs) => {
try {
const tempDir = getTempDir(cmd.tempDir);
const toolsDir = getToolsDir(cmd.toolsDir, tempDir);
// Wipe the temp dir
fs.rmdirSync(tempDir, { recursive: true });
fs.mkdirSync(tempDir, { recursive: true });
let codeql: CodeQL;
if (cmd.codeqlPath !== undefined) {
codeql = getCodeQL(cmd.codeqlPath);
} else {
codeql = await initCodeQL(
undefined,
cmd.githubAuth,
parseGithubUrl(cmd.githubUrl),
tempDir,
toolsDir,
'runner',
logger);
}
const config = await initConfig(
cmd.languages,
cmd.queries,
cmd.configFile,
tempDir,
toolsDir,
codeql,
cmd.checkoutPath || process.cwd(),
cmd.githubAuth,
parseGithubUrl(cmd.githubUrl),
logger);
await runInit(codeql, config);
} catch (e) {
logger.error('Init failed');
logger.error(e);
process.exitCode = 1;
}
});
interface AutobuildArgs {
language: string;
tempDir: string | undefined;
}
program
.command('autobuild')
.description('Attempts to automatically build code')
.requiredOption('--language <language>', 'The language to build')
.option('--temp-dir <dir>', 'Directory to use for temporary files. Defaults to OS temp dir.')
.action(async (cmd: AutobuildArgs) => {
try {
const language = parseLanguage(cmd.language);
if (language === undefined) {
throw new Error(`"${cmd.language}" is not a recognised language`);
}
await runAutobuild(
language,
getTempDir(cmd.tempDir),
logger);
} catch (e) {
logger.error('Autobuild failed');
logger.error(e);
process.exitCode = 1;
}
});
interface AnalyzeArgs {
repository: string;
commit: string;
ref: string;
githubUrl: string;
githubAuth: string;
checkoutPath: string | undefined;
upload: boolean;
outputDir: string | undefined;
tempDir: string | undefined;
}
program
.command('analyze')
.description('Finishes extracting code and runs CodeQL queries')
.requiredOption('--repository <repository>', 'Repository name')
.requiredOption('--commit <commit>', 'SHA of commit that was analyzed')
.requiredOption('--ref <ref>', 'Name of ref that was analyzed')
.requiredOption('--github-url <url>', 'URL of GitHub instance')
.requiredOption('--github-auth <auth>', 'GitHub Apps token, or of the form "username:token" if using a personal access token')
.option('--checkout-path <path>', 'Checkout path (default: current working directory)')
.option('--no-upload', 'Do not upload results after analysis', false)
.option('--output-dir <dir>', 'Directory to output SARIF files to. By default will use temp directory.')
.option('--temp-dir <dir>', 'Directory to use for temporary files. Defaults to OS temp dir.')
.action(async (cmd: AnalyzeArgs) => {
try {
const tempDir = getTempDir(cmd.tempDir);
const outputDir = cmd.outputDir || path.join(tempDir, 'codeql-sarif');
await runAnalyze(
parseRepositoryNwo(cmd.repository),
cmd.commit,
cmd.ref,
undefined,
undefined,
undefined,
cmd.checkoutPath || process.cwd(),
undefined,
cmd.githubAuth,
parseGithubUrl(cmd.githubUrl),
cmd.upload,
'runner',
outputDir,
tempDir,
logger);
} catch (e) {
logger.error('Upload failed');
logger.error(e);
process.exitCode = 1;
}
});
interface UploadArgs {
sarifFile: string;
repository: string;
@@ -18,30 +211,6 @@ interface UploadArgs {
checkoutPath: string | undefined;
}
function parseGithubApiUrl(inputUrl: string): string {
try {
const url = new URL(inputUrl);
// If we detect this is trying to be to github.com
// then return with a fixed canonical URL.
if (url.hostname === 'github.com' || url.hostname === 'api.github.com') {
return 'https://api.github.com';
}
// Add the API path if it's not already present.
if (url.pathname.indexOf('/api/v3') === -1) {
url.pathname = path.join(url.pathname, 'api', 'v3');
}
return url.toString();
} catch (e) {
throw new Error(`"${inputUrl}" is not a valid URL`);
}
}
const logger = getRunnerLogger();
program
.command('upload')
.description('Uploads a SARIF file, or all SARIF files from a directory, to code scanning')
@@ -65,7 +234,7 @@ program
cmd.checkoutPath || process.cwd(),
undefined,
cmd.githubAuth,
parseGithubApiUrl(cmd.githubUrl),
parseGithubUrl(cmd.githubUrl),
'runner',
logger);
} catch (e) {

22
src/upload-lib.ts
View File

@@ -12,8 +12,6 @@ import { RepositoryNwo } from './repository';
import * as sharedEnv from './shared-environment';
import * as util from './util';
type UploadMode = 'actions' | 'runner';
// Takes a list of paths to sarif files and combines them together,
// returning the contents of the combined sarif file.
export function combineSarifFiles(sarifFiles: string[]): string {
@@ -43,8 +41,8 @@ async function uploadPayload(
payload: any,
repositoryNwo: RepositoryNwo,
githubAuth: string,
githubApiUrl: string,
mode: UploadMode,
githubUrl: string,
mode: util.Mode,
logger: Logger) {
logger.info('Uploading results');
@@ -61,7 +59,7 @@ async function uploadPayload(
// minutes, but just waiting a little bit could maybe help.
const backoffPeriods = [1, 5, 15];
const client = api.getApiClient(githubAuth, githubApiUrl);
const client = api.getApiClient(githubAuth, githubUrl);
for (let attempt = 0; attempt <= backoffPeriods.length; attempt++) {
const reqURL = mode === 'actions'
@@ -134,8 +132,8 @@ export async function upload(
checkoutPath: string,
environment: string | undefined,
githubAuth: string,
githubApiUrl: string,
mode: UploadMode,
githubUrl: string,
mode: util.Mode,
logger: Logger): Promise<UploadStatusReport> {
const sarifFiles: string[] = [];
@@ -165,7 +163,7 @@ export async function upload(
checkoutPath,
environment,
githubAuth,
githubApiUrl,
githubUrl,
mode,
logger);
}
@@ -214,8 +212,8 @@ async function uploadFiles(
checkoutPath: string,
environment: string | undefined,
githubAuth: string,
githubApiUrl: string,
mode: UploadMode,
githubUrl: string,
mode: util.Mode,
logger: Logger): Promise<UploadStatusReport> {
logger.info("Uploading sarif files: " + JSON.stringify(sarifFiles));
@@ -235,7 +233,7 @@ async function uploadFiles(
}
let sarifPayload = combineSarifFiles(sarifFiles);
sarifPayload = fingerprints.addFingerprints(sarifPayload, logger);
sarifPayload = fingerprints.addFingerprints(sarifPayload, checkoutPath, logger);
const zipped_sarif = zlib.gzipSync(sarifPayload).toString('base64');
let checkoutURI = fileUrl(checkoutPath);
@@ -275,7 +273,7 @@ async function uploadFiles(
logger.debug("Number of results in upload: " + numResultInSarif);
// Make the upload
await uploadPayload(payload, repositoryNwo, githubAuth, githubApiUrl, mode, logger);
await uploadPayload(payload, repositoryNwo, githubAuth, githubUrl, mode, logger);
return {
raw_upload_size_bytes: rawUploadSizeBytes,

27
src/util.ts
View File

@@ -8,26 +8,21 @@ import * as api from './api-client';
import { Language } from './languages';
import * as sharedEnv from './shared-environment';
/**
* Are we running on actions, or not.
*/
export type Mode = 'actions' | 'runner';
/**
* The URL for github.com.
*/
export const GITHUB_DOTCOM_URL = "https://github.com";
/**
* The API URL for github.com.
*/
export const GITHUB_DOTCOM_API_URL = "https://api.github.com";
/**
* Get the API URL for the GitHub instance we are connected to.
* May be for github.com or for an enterprise instance.
*/
export function getInstanceAPIURL(): string {
return process.env["GITHUB_API_URL"] || GITHUB_DOTCOM_API_URL;
}
/**
* Are we running against a GitHub Enterpise instance, as opposed to github.com.
*/
export function isEnterprise(): boolean {
return getInstanceAPIURL() !== GITHUB_DOTCOM_API_URL;
}
/**
* Get an environment parameter, but throw an error if it is not set.
*/
@@ -297,7 +292,7 @@ export async function sendStatusReport<S extends StatusReportBase>(
statusReport: S,
ignoreFailures?: boolean): Promise<boolean> {
if (isEnterprise()) {
if (getRequiredEnvParam("GITHUB_API_URL") !== GITHUB_DOTCOM_API_URL) {
core.debug("Not sending status report to GitHub Enterprise");
return true;
}