Running lint-fix

This commit is contained in:
Chris Raynor
2020-09-14 10:44:43 +01:00
parent c96f84308a
commit a184d50a26
89 changed files with 3646 additions and 2809 deletions

View File

@@ -1,21 +1,21 @@
import { Command } from 'commander';
import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
import { Command } from "commander";
import * as fs from "fs";
import * as os from "os";
import * as path from "path";
import { runAnalyze } from './analyze';
import { determineAutobuildLanguage, runAutobuild } from './autobuild';
import { CodeQL, getCodeQL } from './codeql';
import { Config, getConfig } from './config-utils';
import { initCodeQL, initConfig, injectWindowsTracer, runInit } from './init';
import { Language, parseLanguage } from './languages';
import { getRunnerLogger } from './logging';
import { parseRepositoryNwo } from './repository';
import * as upload_lib from './upload-lib';
import { getAddSnippetsFlag, getMemoryFlag, getThreadsFlag } from './util';
import { runAnalyze } from "./analyze";
import { determineAutobuildLanguage, runAutobuild } from "./autobuild";
import { CodeQL, getCodeQL } from "./codeql";
import { Config, getConfig } from "./config-utils";
import { initCodeQL, initConfig, injectWindowsTracer, runInit } from "./init";
import { Language, parseLanguage } from "./languages";
import { getRunnerLogger } from "./logging";
import { parseRepositoryNwo } from "./repository";
import * as upload_lib from "./upload-lib";
import { getAddSnippetsFlag, getMemoryFlag, getThreadsFlag } from "./util";
const program = new Command();
program.version('0.0.1');
program.version("0.0.1");
function parseGithubUrl(inputUrl: string): string {
try {
@@ -23,24 +23,23 @@ function parseGithubUrl(inputUrl: string): string {
// If we detect this is trying to be to github.com
// then return with a fixed canonical URL.
if (url.hostname === 'github.com' || url.hostname === 'api.github.com') {
return 'https://github.com';
if (url.hostname === "github.com" || url.hostname === "api.github.com") {
return "https://github.com";
}
// Remove the API prefix if it's present
if (url.pathname.indexOf('/api/v3') !== -1) {
url.pathname = url.pathname.substring(0, url.pathname.indexOf('/api/v3'));
if (url.pathname.indexOf("/api/v3") !== -1) {
url.pathname = url.pathname.substring(0, url.pathname.indexOf("/api/v3"));
}
return url.toString();
} catch (e) {
throw new Error(`"${inputUrl}" is not a valid URL`);
}
}
function getTempDir(userInput: string | undefined): string {
const tempDir = path.join(userInput || process.cwd(), 'codeql-runner');
const tempDir = path.join(userInput || process.cwd(), "codeql-runner");
if (!fs.existsSync(tempDir)) {
fs.mkdirSync(tempDir, { recursive: true });
}
@@ -48,38 +47,38 @@ function getTempDir(userInput: string | undefined): string {
}
function getToolsDir(userInput: string | undefined): string {
const toolsDir = userInput || path.join(os.homedir(), 'codeql-runner-tools');
const toolsDir = userInput || path.join(os.homedir(), "codeql-runner-tools");
if (!fs.existsSync(toolsDir)) {
fs.mkdirSync(toolsDir, { recursive: true });
}
return toolsDir;
}
const codeqlEnvJsonFilename = 'codeql-env.json';
const codeqlEnvJsonFilename = "codeql-env.json";
// Imports the environment from codeqlEnvJsonFilename if not already present
function importTracerEnvironment(config: Config) {
if (!('ODASA_TRACER_CONFIGURATION' in process.env)) {
if (!("ODASA_TRACER_CONFIGURATION" in process.env)) {
const jsonEnvFile = path.join(config.tempDir, codeqlEnvJsonFilename);
const env = JSON.parse(fs.readFileSync(jsonEnvFile).toString('utf-8'));
Object.keys(env).forEach(key => process.env[key] = env[key]);
const env = JSON.parse(fs.readFileSync(jsonEnvFile).toString("utf-8"));
Object.keys(env).forEach((key) => (process.env[key] = env[key]));
}
}
// Allow the user to specify refs in full refs/heads/branch format
// or just the short branch name and prepend "refs/heads/" to it.
function parseRef(userInput: string): string {
if (userInput.startsWith('refs/')) {
if (userInput.startsWith("refs/")) {
return userInput;
} else {
return 'refs/heads/' + userInput;
return `refs/heads/${userInput}`;
}
}
// Parses the --trace-process-name arg from process.argv, or returns undefined
function parseTraceProcessName(): string | undefined {
for (let i = 0; i < process.argv.length - 1; i++) {
if (process.argv[i] === '--trace-process-name') {
if (process.argv[i] === "--trace-process-name") {
return process.argv[i + 1];
}
}
@@ -89,7 +88,7 @@ function parseTraceProcessName(): string | undefined {
// Parses the --trace-process-level arg from process.argv, or returns undefined
function parseTraceProcessLevel(): number | undefined {
for (let i = 0; i < process.argv.length - 1; i++) {
if (process.argv[i] === '--trace-process-level') {
if (process.argv[i] === "--trace-process-level") {
const v = parseInt(process.argv[i + 1], 10);
return isNaN(v) ? undefined : v;
}
@@ -112,19 +111,40 @@ interface InitArgs {
}
program
.command('init')
.description('Initializes CodeQL')
.requiredOption('--repository <repository>', 'Repository name. (Required)')
.requiredOption('--github-url <url>', 'URL of GitHub instance. (Required)')
.requiredOption('--github-auth <auth>', 'GitHub Apps token or personal access token. (Required)')
.option('--languages <languages>', 'Comma-separated list of languages to analyze. Otherwise detects and analyzes all supported languages from the repo.')
.option('--queries <queries>', 'Comma-separated list of additional queries to run. This overrides the same setting in a configuration file.')
.option('--config-file <file>', 'Path to config file.')
.option('--codeql-path <path>', 'Path to a copy of the CodeQL CLI executable to use. Otherwise downloads a copy.')
.option('--temp-dir <dir>', 'Directory to use for temporary files. Default is "./codeql-runner".')
.option('--tools-dir <dir>', 'Directory to use for CodeQL tools and other files to store between runs. Default is a subdirectory of the home directory.')
.option('--checkout-path <path>', 'Checkout path. Default is the current working directory.')
.option('--debug', 'Print more verbose output', false)
.command("init")
.description("Initializes CodeQL")
.requiredOption("--repository <repository>", "Repository name. (Required)")
.requiredOption("--github-url <url>", "URL of GitHub instance. (Required)")
.requiredOption(
"--github-auth <auth>",
"GitHub Apps token or personal access token. (Required)"
)
.option(
"--languages <languages>",
"Comma-separated list of languages to analyze. Otherwise detects and analyzes all supported languages from the repo."
)
.option(
"--queries <queries>",
"Comma-separated list of additional queries to run. This overrides the same setting in a configuration file."
)
.option("--config-file <file>", "Path to config file.")
.option(
"--codeql-path <path>",
"Path to a copy of the CodeQL CLI executable to use. Otherwise downloads a copy."
)
.option(
"--temp-dir <dir>",
'Directory to use for temporary files. Default is "./codeql-runner".'
)
.option(
"--tools-dir <dir>",
"Directory to use for CodeQL tools and other files to store between runs. Default is a subdirectory of the home directory."
)
.option(
"--checkout-path <path>",
"Checkout path. Default is the current working directory."
)
.option("--debug", "Print more verbose output", false)
// This prevents a message like: error: unknown option '--trace-process-level'
// Remove this if commander.js starts supporting hidden options.
.allowUnknownOption()
@@ -149,8 +169,9 @@ program
parseGithubUrl(cmd.githubUrl),
tempDir,
toolsDir,
'runner',
logger);
"runner",
logger
);
}
const config = await initConfig(
@@ -164,60 +185,64 @@ program
cmd.checkoutPath || process.cwd(),
cmd.githubAuth,
parseGithubUrl(cmd.githubUrl),
logger);
logger
);
const tracerConfig = await runInit(codeql, config);
if (tracerConfig === undefined) {
return;
}
if (process.platform === 'win32') {
if (process.platform === "win32") {
await injectWindowsTracer(
parseTraceProcessName(),
parseTraceProcessLevel(),
config,
codeql,
tracerConfig);
tracerConfig
);
}
// Always output a json file of the env that can be consumed programatically
const jsonEnvFile = path.join(config.tempDir, codeqlEnvJsonFilename);
fs.writeFileSync(jsonEnvFile, JSON.stringify(tracerConfig.env));
if (process.platform === 'win32') {
const batEnvFile = path.join(config.tempDir, 'codeql-env.bat');
if (process.platform === "win32") {
const batEnvFile = path.join(config.tempDir, "codeql-env.bat");
const batEnvFileContents = Object.entries(tracerConfig.env)
.map(([key, value]) => `Set ${key}=${value}`)
.join('\n');
.join("\n");
fs.writeFileSync(batEnvFile, batEnvFileContents);
const powershellEnvFile = path.join(config.tempDir, 'codeql-env.sh');
const powershellEnvFile = path.join(config.tempDir, "codeql-env.sh");
const powershellEnvFileContents = Object.entries(tracerConfig.env)
.map(([key, value]) => `$env:${key}="${value}"`)
.join('\n');
.join("\n");
fs.writeFileSync(powershellEnvFile, powershellEnvFileContents);
logger.info(`\nCodeQL environment output to "${jsonEnvFile}", "${batEnvFile}" and "${powershellEnvFile}". ` +
`Please export these variables to future processes so the build can be traced. ` +
`If using cmd/batch run "call ${batEnvFile}" ` +
`or if using PowerShell run "cat ${powershellEnvFile} | Invoke-Expression".`);
logger.info(
`\nCodeQL environment output to "${jsonEnvFile}", "${batEnvFile}" and "${powershellEnvFile}". ` +
`Please export these variables to future processes so the build can be traced. ` +
`If using cmd/batch run "call ${batEnvFile}" ` +
`or if using PowerShell run "cat ${powershellEnvFile} | Invoke-Expression".`
);
} else {
// Assume that anything that's not windows is using a unix-style shell
const shEnvFile = path.join(config.tempDir, 'codeql-env.sh');
const shEnvFile = path.join(config.tempDir, "codeql-env.sh");
const shEnvFileContents = Object.entries(tracerConfig.env)
// Some vars contain ${LIB} that we do not want to be expanded when executing this script
.map(([key, value]) => `export ${key}="${value.replace('$', '\\$')}"`)
.join('\n');
.map(([key, value]) => `export ${key}="${value.replace("$", "\\$")}"`)
.join("\n");
fs.writeFileSync(shEnvFile, shEnvFileContents);
logger.info(`\nCodeQL environment output to "${jsonEnvFile}" and "${shEnvFile}". ` +
`Please export these variables to future processes so the build can be traced, ` +
`for example by running ". ${shEnvFile}".`);
logger.info(
`\nCodeQL environment output to "${jsonEnvFile}" and "${shEnvFile}". ` +
`Please export these variables to future processes so the build can be traced, ` +
`for example by running ". ${shEnvFile}".`
);
}
} catch (e) {
logger.error('Init failed');
logger.error("Init failed");
logger.error(e);
process.exitCode = 1;
}
@@ -230,26 +255,38 @@ interface AutobuildArgs {
}
program
.command('autobuild')
.description('Attempts to automatically build code')
.option('--language <language>', 'The language to build. Otherwise will detect the dominant compiled language.')
.option('--temp-dir <dir>', 'Directory to use for temporary files. Default is "./codeql-runner".')
.option('--debug', 'Print more verbose output', false)
.command("autobuild")
.description("Attempts to automatically build code")
.option(
"--language <language>",
"The language to build. Otherwise will detect the dominant compiled language."
)
.option(
"--temp-dir <dir>",
'Directory to use for temporary files. Default is "./codeql-runner".'
)
.option("--debug", "Print more verbose output", false)
.action(async (cmd: AutobuildArgs) => {
const logger = getRunnerLogger(cmd.debug);
try {
const config = await getConfig(getTempDir(cmd.tempDir), logger);
if (config === undefined) {
throw new Error("Config file could not be found at expected location. " +
"Was the 'init' command run with the same '--temp-dir' argument as this command.");
throw new Error(
"Config file could not be found at expected location. " +
"Was the 'init' command run with the same '--temp-dir' argument as this command."
);
}
importTracerEnvironment(config);
let language: Language | undefined = undefined;
if (cmd.language !== undefined) {
language = parseLanguage(cmd.language);
if (language === undefined || !config.languages.includes(language)) {
throw new Error(`"${cmd.language}" is not a recognised language. ` +
`Known languages in this project are ${config.languages.join(', ')}.`);
throw new Error(
`"${cmd.language}" is not a recognised language. ` +
`Known languages in this project are ${config.languages.join(
", "
)}.`
);
}
} else {
language = determineAutobuildLanguage(config, logger);
@@ -258,7 +295,7 @@ program
await runAutobuild(language, config, logger);
}
} catch (e) {
logger.error('Autobuild failed');
logger.error("Autobuild failed");
logger.error(e);
process.exitCode = 1;
}
@@ -281,31 +318,57 @@ interface AnalyzeArgs {
}
program
.command('analyze')
.description('Finishes extracting code and runs CodeQL queries')
.requiredOption('--repository <repository>', 'Repository name. (Required)')
.requiredOption('--commit <commit>', 'SHA of commit that was analyzed. (Required)')
.requiredOption('--ref <ref>', 'Name of ref that was analyzed. (Required)')
.requiredOption('--github-url <url>', 'URL of GitHub instance. (Required)')
.requiredOption('--github-auth <auth>', 'GitHub Apps token or personal access token. (Required)')
.option('--checkout-path <path>', 'Checkout path. Default is the current working directory.')
.option('--no-upload', 'Do not upload results after analysis.')
.option('--output-dir <dir>', 'Directory to output SARIF files to. Default is in the temp directory.')
.option('--ram <ram>', 'Amount of memory to use when running queries. Default is to use all available memory.')
.option('--no-add-snippets', 'Specify whether to include code snippets in the sarif output.')
.option('--threads <threads>', 'Number of threads to use when running queries. ' +
'Default is to use all available cores.')
.option('--temp-dir <dir>', 'Directory to use for temporary files. Default is "./codeql-runner".')
.option('--debug', 'Print more verbose output', false)
.command("analyze")
.description("Finishes extracting code and runs CodeQL queries")
.requiredOption("--repository <repository>", "Repository name. (Required)")
.requiredOption(
"--commit <commit>",
"SHA of commit that was analyzed. (Required)"
)
.requiredOption("--ref <ref>", "Name of ref that was analyzed. (Required)")
.requiredOption("--github-url <url>", "URL of GitHub instance. (Required)")
.requiredOption(
"--github-auth <auth>",
"GitHub Apps token or personal access token. (Required)"
)
.option(
"--checkout-path <path>",
"Checkout path. Default is the current working directory."
)
.option("--no-upload", "Do not upload results after analysis.")
.option(
"--output-dir <dir>",
"Directory to output SARIF files to. Default is in the temp directory."
)
.option(
"--ram <ram>",
"Amount of memory to use when running queries. Default is to use all available memory."
)
.option(
"--no-add-snippets",
"Specify whether to include code snippets in the sarif output."
)
.option(
"--threads <threads>",
"Number of threads to use when running queries. " +
"Default is to use all available cores."
)
.option(
"--temp-dir <dir>",
'Directory to use for temporary files. Default is "./codeql-runner".'
)
.option("--debug", "Print more verbose output", false)
.action(async (cmd: AnalyzeArgs) => {
const logger = getRunnerLogger(cmd.debug);
try {
const tempDir = getTempDir(cmd.tempDir);
const outputDir = cmd.outputDir || path.join(tempDir, 'codeql-sarif');
const outputDir = cmd.outputDir || path.join(tempDir, "codeql-sarif");
const config = await getConfig(getTempDir(cmd.tempDir), logger);
if (config === undefined) {
throw new Error("Config file could not be found at expected location. " +
"Was the 'init' command run with the same '--temp-dir' argument as this command.");
throw new Error(
"Config file could not be found at expected location. " +
"Was the 'init' command run with the same '--temp-dir' argument as this command."
);
}
await runAnalyze(
parseRepositoryNwo(cmd.repository),
@@ -319,15 +382,16 @@ program
cmd.githubAuth,
parseGithubUrl(cmd.githubUrl),
cmd.upload,
'runner',
"runner",
outputDir,
getMemoryFlag(cmd.ram),
getAddSnippetsFlag(cmd.addSnippets),
getThreadsFlag(cmd.threads, logger),
config,
logger);
logger
);
} catch (e) {
logger.error('Analyze failed');
logger.error("Analyze failed");
logger.error(e);
process.exitCode = 1;
}
@@ -345,16 +409,30 @@ interface UploadArgs {
}
program
.command('upload')
.description('Uploads a SARIF file, or all SARIF files from a directory, to code scanning')
.requiredOption('--sarif-file <file>', 'SARIF file to upload, or a directory containing multiple SARIF files. (Required)')
.requiredOption('--repository <repository>', 'Repository name. (Required)')
.requiredOption('--commit <commit>', 'SHA of commit that was analyzed. (Required)')
.requiredOption('--ref <ref>', 'Name of ref that was analyzed. (Required)')
.requiredOption('--github-url <url>', 'URL of GitHub instance. (Required)')
.requiredOption('--github-auth <auth>', 'GitHub Apps token or personal access token. (Required)')
.option('--checkout-path <path>', 'Checkout path. Default is the current working directory.')
.option('--debug', 'Print more verbose output', false)
.command("upload")
.description(
"Uploads a SARIF file, or all SARIF files from a directory, to code scanning"
)
.requiredOption(
"--sarif-file <file>",
"SARIF file to upload, or a directory containing multiple SARIF files. (Required)"
)
.requiredOption("--repository <repository>", "Repository name. (Required)")
.requiredOption(
"--commit <commit>",
"SHA of commit that was analyzed. (Required)"
)
.requiredOption("--ref <ref>", "Name of ref that was analyzed. (Required)")
.requiredOption("--github-url <url>", "URL of GitHub instance. (Required)")
.requiredOption(
"--github-auth <auth>",
"GitHub Apps token or personal access token. (Required)"
)
.option(
"--checkout-path <path>",
"Checkout path. Default is the current working directory."
)
.option("--debug", "Print more verbose output", false)
.action(async (cmd: UploadArgs) => {
const logger = getRunnerLogger(cmd.debug);
try {
@@ -370,10 +448,11 @@ program
undefined,
cmd.githubAuth,
parseGithubUrl(cmd.githubUrl),
'runner',
logger);
"runner",
logger
);
} catch (e) {
logger.error('Upload failed');
logger.error("Upload failed");
logger.error(e);
process.exitCode = 1;
}