github/codeql-action
1
0
Fork 0
mirror of https://github.com/github/codeql-action.git synced 2025-12-09 17:28:06 +08:00
Code Issues Packages Projects Releases Wiki Activity

Implement removing duplicate locations from a SARIF file

Browse Source
This commit is contained in:
Henry Mercer
2023-03-24 20:01:35 +00:00
parent 097ab4665f
commit 6f852eeb38
12 changed files with 288 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
lib/shared-environment.js generated
View File

@@ -1,6 +1,6 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.ODASA_TRACER_CONFIGURATION = exports.CODEQL_WORKFLOW_STARTED_AT = exports.CODEQL_ACTION_TEST_MODE = exports.CODEQL_ACTION_TESTING_ENVIRONMENT = exports.CODEQL_ACTION_ANALYZE_DID_COMPLETE_SUCCESSFULLY = exports.CODEQL_ACTION_DID_AUTOBUILD_GOLANG = void 0;
exports.ODASA_TRACER_CONFIGURATION = exports.CODEQL_WORKFLOW_STARTED_AT = exports.CODEQL_ACTION_DISABLE_DUPLICATE_LOCATION_FIX = exports.CODEQL_ACTION_TEST_MODE = exports.CODEQL_ACTION_TESTING_ENVIRONMENT = exports.CODEQL_ACTION_ANALYZE_DID_COMPLETE_SUCCESSFULLY = exports.CODEQL_ACTION_DID_AUTOBUILD_GOLANG = void 0;
/**
* Environment variable that is set to true when the CodeQL Action has invoked
* the Go autobuilder.
@@ -14,6 +14,11 @@ exports.CODEQL_ACTION_ANALYZE_DID_COMPLETE_SUCCESSFULLY = "CODEQL_ACTION_ANALYZE
exports.CODEQL_ACTION_TESTING_ENVIRONMENT = "CODEQL_ACTION_TESTING_ENVIRONMENT";
/** Used to disable uploading SARIF results or status reports to the GitHub API */
exports.CODEQL_ACTION_TEST_MODE = "CODEQL_ACTION_TEST_MODE";
/**
* Used to disable the SARIF post-processing in the Action that removes duplicate locations from
* notifications in the `run[].invocations[].toolExecutionNotifications` SARIF property.
*/
exports.CODEQL_ACTION_DISABLE_DUPLICATE_LOCATION_FIX = "CODEQL_ACTION_DISABLE_DUPLICATE_LOCATION_FIX";
/**
* The time at which the first action (normally init) started executing.
* If a workflow invokes a different action without first invoking the init

2
lib/shared-environment.js.map
View File

@@ -1 +1 @@
{"version":3,"file":"shared-environment.js","sourceRoot":"","sources":["../src/shared-environment.ts"],"names":[],"mappings":";;;AAAA;;;GAGG;AACU,QAAA,kCAAkC,GAC7C,oCAAoC,CAAC;AAEvC;;;GAGG;AACU,QAAA,+CAA+C,GAC1D,iDAAiD,CAAC;AAEvC,QAAA,iCAAiC,GAC5C,mCAAmC,CAAC;AAEtC,kFAAkF;AACrE,QAAA,uBAAuB,GAAG,yBAAyB,CAAC;AAEjE;;;;;;GAMG;AACU,QAAA,0BAA0B,GAAG,4BAA4B,CAAC;AAE1D,QAAA,0BAA0B,GAAG,4BAA4B,CAAC"}
{"version":3,"file":"shared-environment.js","sourceRoot":"","sources":["../src/shared-environment.ts"],"names":[],"mappings":";;;AAAA;;;GAGG;AACU,QAAA,kCAAkC,GAC7C,oCAAoC,CAAC;AAEvC;;;GAGG;AACU,QAAA,+CAA+C,GAC1D,iDAAiD,CAAC;AAEvC,QAAA,iCAAiC,GAC5C,mCAAmC,CAAC;AAEtC,kFAAkF;AACrE,QAAA,uBAAuB,GAAG,yBAAyB,CAAC;AAEjE;;;GAGG;AACU,QAAA,4CAA4C,GACvD,8CAA8C,CAAC;AAEjD;;;;;;GAMG;AACU,QAAA,0BAA0B,GAAG,4BAA4B,CAAC;AAE1D,QAAA,0BAA0B,GAAG,4BAA4B,CAAC"}

4
lib/upload-lib.js generated
View File

@@ -38,7 +38,7 @@ const actionsUtil = __importStar(require("./actions-util"));
const api = __importStar(require("./api-client"));
const fingerprints = __importStar(require("./fingerprints"));
const repository_1 = require("./repository");
const sharedEnv = __importStar(require("./shared-environment"));
const shared_environment_1 = require("./shared-environment");
const util = __importStar(require("./util"));
const workflow = __importStar(require("./workflow"));
// Takes a list of paths to sarif files and combines them together,
@@ -206,7 +206,7 @@ function buildPayload(commitOid, ref, analysisKey, analysisName, zippedSarif, wo
workflow_run_id: workflowRunID,
checkout_uri: checkoutURI,
environment,
started_at: process.env[sharedEnv.CODEQL_WORKFLOW_STARTED_AT],
started_at: process.env[shared_environment_1.CODEQL_WORKFLOW_STARTED_AT],
tool_names: toolNames,
base_ref: undefined,
base_sha: undefined,

2
lib/upload-lib.js.map
View File

File diff suppressed because one or more lines are too long

65
lib/util.js generated
View File

@@ -26,7 +26,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.parseMatrixInput = exports.isHostedRunner = exports.checkForTimeout = exports.withTimeout = exports.tryGetFolderBytes = exports.listFolder = exports.doesDirectoryExist = exports.logCodeScanningConfigInCli = exports.useCodeScanningConfigInCli = exports.isInTestMode = exports.getMlPoweredJsQueriesStatus = exports.getMlPoweredJsQueriesPack = exports.ML_POWERED_JS_QUERIES_PACK_NAME = exports.supportExpectDiscardedCache = exports.isGoodVersion = exports.delay = exports.bundleDb = exports.codeQlVersionAbove = exports.getCachedCodeQlVersion = exports.cacheCodeQlVersion = exports.isHTTPError = exports.UserError = exports.HTTPError = exports.getRequiredEnvParam = exports.enrichEnvironment = exports.initializeEnvironment = exports.EnvVar = exports.assertNever = exports.apiVersionInRange = exports.DisallowedAPIVersionReason = exports.checkGitHubVersionInRange = exports.getGitHubVersion = exports.GitHubVariant = exports.parseGitHubUrl = exports.getCodeQLDatabasePath = exports.getThreadsFlag = exports.getThreadsFlagValue = exports.getAddSnippetsFlag = exports.getMemoryFlag = exports.getMemoryFlagValue = exports.withTmpDir = exports.getToolNames = exports.getExtraOptionsEnvParam = exports.DEFAULT_DEBUG_DATABASE_NAME = exports.DEFAULT_DEBUG_ARTIFACT_NAME = exports.GITHUB_DOTCOM_URL = void 0;
exports.fixInvalidNotifications = exports.parseMatrixInput = exports.isHostedRunner = exports.checkForTimeout = exports.withTimeout = exports.tryGetFolderBytes = exports.listFolder = exports.doesDirectoryExist = exports.logCodeScanningConfigInCli = exports.useCodeScanningConfigInCli = exports.isInTestMode = exports.getMlPoweredJsQueriesStatus = exports.getMlPoweredJsQueriesPack = exports.ML_POWERED_JS_QUERIES_PACK_NAME = exports.supportExpectDiscardedCache = exports.isGoodVersion = exports.delay = exports.bundleDb = exports.codeQlVersionAbove = exports.getCachedCodeQlVersion = exports.cacheCodeQlVersion = exports.isHTTPError = exports.UserError = exports.HTTPError = exports.getRequiredEnvParam = exports.enrichEnvironment = exports.initializeEnvironment = exports.EnvVar = exports.assertNever = exports.apiVersionInRange = exports.DisallowedAPIVersionReason = exports.checkGitHubVersionInRange = exports.getGitHubVersion = exports.GitHubVariant = exports.parseGitHubUrl = exports.getCodeQLDatabasePath = exports.getThreadsFlag = exports.getThreadsFlagValue = exports.getAddSnippetsFlag = exports.getMemoryFlag = exports.getMemoryFlagValue = exports.withTmpDir = exports.getToolNames = exports.getExtraOptionsEnvParam = exports.DEFAULT_DEBUG_DATABASE_NAME = exports.DEFAULT_DEBUG_ARTIFACT_NAME = exports.GITHUB_DOTCOM_URL = void 0;
const fs = __importStar(require("fs"));
const os = __importStar(require("os"));
const path = __importStar(require("path"));
@@ -707,4 +707,67 @@ function parseMatrixInput(matrixInput) {
return JSON.parse(matrixInput);
}
exports.parseMatrixInput = parseMatrixInput;
function removeDuplicateLocations(locations) {
const newJsonLocations = new Set();
return locations.filter((location) => {
const jsonLocation = JSON.stringify(location);
if (!newJsonLocations.has(jsonLocation)) {
newJsonLocations.add(jsonLocation);
return true;
}
return false;
});
}
function fixInvalidNotifications(sarif, logger) {
if (process.env[shared_environment_1.CODEQL_ACTION_DISABLE_DUPLICATE_LOCATION_FIX] === "true") {
logger.info("SARIF notification object duplicate location fix disabled by the " +
`${shared_environment_1.CODEQL_ACTION_DISABLE_DUPLICATE_LOCATION_FIX} environment variable.`);
return sarif;
}
if (!(sarif.runs instanceof Array)) {
return sarif;
}
// Ensure that the array of locations for each SARIF notification contains unique locations.
// This is a workaround for a bug in the CodeQL CLI that causes duplicate locations to be
// emitted in some cases.
let numDuplicateLocationsRemoved = 0;
const newSarif = {
...sarif,
runs: sarif.runs.map((run) => {
if (run.tool?.driver?.name !== "CodeQL" ||
!(run.invocations instanceof Array)) {
return run;
}
return {
...run,
invocations: run.invocations.map((invocation) => {
if (!(invocation.toolExecutionNotifications instanceof Array)) {
return invocation;
}
return {
...invocation,
toolExecutionNotifications: invocation.toolExecutionNotifications.map((notification) => {
if (!(notification.locations instanceof Array)) {
return notification;
}
const newLocations = removeDuplicateLocations(notification.locations);
numDuplicateLocationsRemoved +=
notification.locations.length - newLocations.length;
return {
...notification,
locations: newLocations,
};
}),
};
}),
};
}),
};
if (numDuplicateLocationsRemoved > 0) {
logger.info(`Removed ${numDuplicateLocationsRemoved} duplicate locations from SARIF notification ` +
"objects.");
}
return newSarif;
}
exports.fixInvalidNotifications = fixInvalidNotifications;
//# sourceMappingURL=util.js.map

2
lib/util.js.map
View File

File diff suppressed because one or more lines are too long

45
lib/util.test.js generated
View File

@@ -330,4 +330,49 @@ const shortTime = 10;
t.deepEqual(shortTaskTimedOut, false);
t.deepEqual(result, 99);
});
function createMockSarifWithNotification(locations) {
return {
runs: [
{
tool: {
driver: {
name: "CodeQL",
},
},
invocations: [
{
toolExecutionNotifications: [
{
locations,
},
],
},
],
},
],
};
}
const stubLocation = {
physicalLocation: {
artifactLocation: {
uri: "file1",
},
},
};
(0, ava_1.default)("fixInvalidNotifications leaves notifications with unique locations alone", (t) => {
const messages = [];
const result = util.fixInvalidNotifications(createMockSarifWithNotification([stubLocation]), (0, testing_utils_1.getRecordingLogger)(messages));
t.deepEqual(result, createMockSarifWithNotification([stubLocation]));
t.is(messages.length, 0);
});
(0, ava_1.default)("fixInvalidNotifications removes duplicate locations", (t) => {
const messages = [];
const result = util.fixInvalidNotifications(createMockSarifWithNotification([stubLocation, stubLocation]), (0, testing_utils_1.getRecordingLogger)(messages));
t.deepEqual(result, createMockSarifWithNotification([stubLocation]));
t.is(messages.length, 1);
t.deepEqual(messages[0], {
type: "info",
message: "Removed 1 duplicate locations from SARIF notification objects.",
});
});
//# sourceMappingURL=util.test.js.map

2
lib/util.test.js.map
View File

File diff suppressed because one or more lines are too long

7
src/shared-environment.ts
View File

@@ -18,6 +18,13 @@ export const CODEQL_ACTION_TESTING_ENVIRONMENT =
/** Used to disable uploading SARIF results or status reports to the GitHub API */
export const CODEQL_ACTION_TEST_MODE = "CODEQL_ACTION_TEST_MODE";
/**
* Used to disable the SARIF post-processing in the Action that removes duplicate locations from
* notifications in the `run[].invocations[].toolExecutionNotifications` SARIF property.
*/
export const CODEQL_ACTION_DISABLE_DUPLICATE_LOCATION_FIX =
"CODEQL_ACTION_DISABLE_DUPLICATE_LOCATION_FIX";
/**
* The time at which the first action (normally init) started executing.
* If a workflow invokes a different action without first invoking the init

4
src/upload-lib.ts
View File

@@ -13,7 +13,7 @@ import * as api from "./api-client";
import * as fingerprints from "./fingerprints";
import { Logger } from "./logging";
import { parseRepositoryNwo, RepositoryNwo } from "./repository";
import * as sharedEnv from "./shared-environment";
import { CODEQL_WORKFLOW_STARTED_AT } from "./shared-environment";
import * as util from "./util";
import { SarifFile, SarifResult, SarifRun } from "./util";
import * as workflow from "./workflow";
@@ -272,7 +272,7 @@ export function buildPayload(
workflow_run_id: workflowRunID,
checkout_uri: checkoutURI,
environment,
started_at: process.env[sharedEnv.CODEQL_WORKFLOW_STARTED_AT],
started_at: process.env[CODEQL_WORKFLOW_STARTED_AT],
tool_names: toolNames,
base_ref: undefined as undefined | string,
base_sha: undefined as undefined | string,

59
src/util.test.ts
View File

@@ -9,7 +9,7 @@ import * as sinon from "sinon";
import * as api from "./api-client";
import { Config } from "./config-utils";
import { getRunnerLogger } from "./logging";
import { setupTests } from "./testing-utils";
import { getRecordingLogger, LoggedMessage, setupTests } from "./testing-utils";
import * as util from "./util";
setupTests(test);
@@ -400,3 +400,60 @@ test("withTimeout doesn't call callback if promise resolves", async (t) => {
t.deepEqual(shortTaskTimedOut, false);
t.deepEqual(result, 99);
});
function createMockSarifWithNotification(
locations: util.SarifLocation[]
): util.SarifFile {
return {
runs: [
{
tool: {
driver: {
name: "CodeQL",
},
},
invocations: [
{
toolExecutionNotifications: [
{
locations,
},
],
},
],
},
],
};
}
const stubLocation: util.SarifLocation = {
physicalLocation: {
artifactLocation: {
uri: "file1",
},
},
};
test("fixInvalidNotifications leaves notifications with unique locations alone", (t) => {
const messages: LoggedMessage[] = [];
const result = util.fixInvalidNotifications(
createMockSarifWithNotification([stubLocation]),
getRecordingLogger(messages)
);
t.deepEqual(result, createMockSarifWithNotification([stubLocation]));
t.is(messages.length, 0);
});
test("fixInvalidNotifications removes duplicate locations", (t) => {
const messages: LoggedMessage[] = [];
const result = util.fixInvalidNotifications(
createMockSarifWithNotification([stubLocation, stubLocation]),
getRecordingLogger(messages)
);
t.deepEqual(result, createMockSarifWithNotification([stubLocation]));
t.is(messages.length, 1);
t.deepEqual(messages[0], {
type: "info",
message: "Removed 1 duplicate locations from SARIF notification objects.",
});
});

101
src/util.ts
View File

@@ -19,7 +19,10 @@ import {
import { Feature, FeatureEnablement } from "./feature-flags";
import { Language } from "./languages";
import { Logger } from "./logging";
import { CODEQL_ACTION_TEST_MODE } from "./shared-environment";
import {
CODEQL_ACTION_DISABLE_DUPLICATE_LOCATION_FIX,
CODEQL_ACTION_TEST_MODE,
} from "./shared-environment";
/**
* Specifies bundle versions that are known to be broken
@@ -58,9 +61,14 @@ export interface SarifRun {
id?: string;
};
artifacts?: string[];
invocations?: SarifInvocation[];
results?: SarifResult[];
}
export interface SarifInvocation {
toolExecutionNotifications?: SarifNotification[];
}
export interface SarifResult {
ruleId?: string;
message?: {
@@ -81,6 +89,18 @@ export interface SarifResult {
};
}
export interface SarifNotification {
locations?: SarifLocation[];
}
export interface SarifLocation {
physicalLocation?: {
artifactLocation?: {
uri?: string;
};
};
}
/**
* Get the extra options for the codeql commands.
*/
@@ -828,3 +848,82 @@ export function parseMatrixInput(
}
return JSON.parse(matrixInput);
}
function removeDuplicateLocations(locations: SarifLocation[]): SarifLocation[] {
const newJsonLocations = new Set<string>();
return locations.filter((location) => {
const jsonLocation = JSON.stringify(location);
if (!newJsonLocations.has(jsonLocation)) {
newJsonLocations.add(jsonLocation);
return true;
}
return false;
});
}
export function fixInvalidNotifications(
sarif: SarifFile,
logger: Logger
): SarifFile {
if (process.env[CODEQL_ACTION_DISABLE_DUPLICATE_LOCATION_FIX] === "true") {
logger.info(
"SARIF notification object duplicate location fix disabled by the " +
`${CODEQL_ACTION_DISABLE_DUPLICATE_LOCATION_FIX} environment variable.`
);
return sarif;
}
if (!(sarif.runs instanceof Array)) {
return sarif;
}
// Ensure that the array of locations for each SARIF notification contains unique locations.
// This is a workaround for a bug in the CodeQL CLI that causes duplicate locations to be
// emitted in some cases.
let numDuplicateLocationsRemoved = 0;
const newSarif = {
...sarif,
runs: sarif.runs.map((run) => {
if (
run.tool?.driver?.name !== "CodeQL" ||
!(run.invocations instanceof Array)
) {
return run;
}
return {
...run,
invocations: run.invocations.map((invocation) => {
if (!(invocation.toolExecutionNotifications instanceof Array)) {
return invocation;
}
return {
...invocation,
toolExecutionNotifications:
invocation.toolExecutionNotifications.map((notification) => {
if (!(notification.locations instanceof Array)) {
return notification;
}
const newLocations = removeDuplicateLocations(
notification.locations
);
numDuplicateLocationsRemoved +=
notification.locations.length - newLocations.length;
return {
...notification,
locations: newLocations,
};
}),
};
}),
};
}),
};
if (numDuplicateLocationsRemoved > 0) {
logger.info(
`Removed ${numDuplicateLocationsRemoved} duplicate locations from SARIF notification ` +
"objects."
);
}
return newSarif;
}