diff --git a/lib/analyze-action.js b/lib/analyze-action.js
index 77db2a754..2b79889f0 100644
--- a/lib/analyze-action.js
+++ b/lib/analyze-action.js
@@ -92294,16 +92294,27 @@ async function getCodeQLSource(toolsInput, defaultCliVersion, apiDetails, varian
       );
     }
   } else if (toolsInput !== void 0 && toolsInput === CODEQL_TOOLCACHE_INPUT) {
-    logger.info(
-      `Attempting to use the latest CodeQL CLI version in the toolcache, as requested by 'tools: ${toolsInput}'.`
-    );
-    const latestToolcacheVersion = getLatestToolcacheVersion(logger);
-    if (latestToolcacheVersion) {
-      cliVersion2 = latestToolcacheVersion;
-    } else {
+    let latestToolcacheVersion;
+    const allowToolcacheValue = isDynamicWorkflow() || isInTestMode();
+    if (allowToolcacheValue) {
       logger.info(
-        `Found no CodeQL CLI in the toolcache, ignoring 'tools: ${toolsInput}'...`
+        `Attempting to use the latest CodeQL CLI version in the toolcache, as requested by 'tools: ${toolsInput}'.`
       );
+      latestToolcacheVersion = getLatestToolcacheVersion(logger);
+      if (latestToolcacheVersion) {
+        cliVersion2 = latestToolcacheVersion;
+      }
+    }
+    if (latestToolcacheVersion === void 0) {
+      if (allowToolcacheValue) {
+        logger.info(
+          `Found no CodeQL CLI in the toolcache, ignoring 'tools: ${toolsInput}'...`
+        );
+      } else {
+        logger.warning(
+          `Ignoring 'tools: ${toolsInput}' because the workflow was not triggered dynamically.`
+        );
+      }
       cliVersion2 = defaultCliVersion.cliVersion;
       tagName = defaultCliVersion.tagName;
     }
diff --git a/lib/init-action-post.js b/lib/init-action-post.js
index d2b206bdf..865a932b8 100644
--- a/lib/init-action-post.js
+++ b/lib/init-action-post.js
@@ -130276,16 +130276,27 @@ async function getCodeQLSource(toolsInput, defaultCliVersion, apiDetails, varian
       );
     }
   } else if (toolsInput !== void 0 && toolsInput === CODEQL_TOOLCACHE_INPUT) {
-    logger.info(
-      `Attempting to use the latest CodeQL CLI version in the toolcache, as requested by 'tools: ${toolsInput}'.`
-    );
-    const latestToolcacheVersion = getLatestToolcacheVersion(logger);
-    if (latestToolcacheVersion) {
-      cliVersion2 = latestToolcacheVersion;
-    } else {
+    let latestToolcacheVersion;
+    const allowToolcacheValue = isDynamicWorkflow() || isInTestMode();
+    if (allowToolcacheValue) {
       logger.info(
-        `Found no CodeQL CLI in the toolcache, ignoring 'tools: ${toolsInput}'...`
+        `Attempting to use the latest CodeQL CLI version in the toolcache, as requested by 'tools: ${toolsInput}'.`
       );
+      latestToolcacheVersion = getLatestToolcacheVersion(logger);
+      if (latestToolcacheVersion) {
+        cliVersion2 = latestToolcacheVersion;
+      }
+    }
+    if (latestToolcacheVersion === void 0) {
+      if (allowToolcacheValue) {
+        logger.info(
+          `Found no CodeQL CLI in the toolcache, ignoring 'tools: ${toolsInput}'...`
+        );
+      } else {
+        logger.warning(
+          `Ignoring 'tools: ${toolsInput}' because the workflow was not triggered dynamically.`
+        );
+      }
       cliVersion2 = defaultCliVersion.cliVersion;
       tagName = defaultCliVersion.tagName;
     }
diff --git a/lib/init-action.js b/lib/init-action.js
index b22dbec7a..4484568a4 100644
--- a/lib/init-action.js
+++ b/lib/init-action.js
@@ -89069,16 +89069,27 @@ async function getCodeQLSource(toolsInput, defaultCliVersion, apiDetails, varian
       );
     }
   } else if (toolsInput !== void 0 && toolsInput === CODEQL_TOOLCACHE_INPUT) {
-    logger.info(
-      `Attempting to use the latest CodeQL CLI version in the toolcache, as requested by 'tools: ${toolsInput}'.`
-    );
-    const latestToolcacheVersion = getLatestToolcacheVersion(logger);
-    if (latestToolcacheVersion) {
-      cliVersion2 = latestToolcacheVersion;
-    } else {
+    let latestToolcacheVersion;
+    const allowToolcacheValue = isDynamicWorkflow() || isInTestMode();
+    if (allowToolcacheValue) {
       logger.info(
-        `Found no CodeQL CLI in the toolcache, ignoring 'tools: ${toolsInput}'...`
+        `Attempting to use the latest CodeQL CLI version in the toolcache, as requested by 'tools: ${toolsInput}'.`
       );
+      latestToolcacheVersion = getLatestToolcacheVersion(logger);
+      if (latestToolcacheVersion) {
+        cliVersion2 = latestToolcacheVersion;
+      }
+    }
+    if (latestToolcacheVersion === void 0) {
+      if (allowToolcacheValue) {
+        logger.info(
+          `Found no CodeQL CLI in the toolcache, ignoring 'tools: ${toolsInput}'...`
+        );
+      } else {
+        logger.warning(
+          `Ignoring 'tools: ${toolsInput}' because the workflow was not triggered dynamically.`
+        );
+      }
       cliVersion2 = defaultCliVersion.cliVersion;
       tagName = defaultCliVersion.tagName;
     }
diff --git a/lib/upload-lib.js b/lib/upload-lib.js
index 95316fbe6..7d3d294d4 100644
--- a/lib/upload-lib.js
+++ b/lib/upload-lib.js
@@ -90110,16 +90110,27 @@ async function getCodeQLSource(toolsInput, defaultCliVersion, apiDetails, varian
       );
     }
   } else if (toolsInput !== void 0 && toolsInput === CODEQL_TOOLCACHE_INPUT) {
-    logger.info(
-      `Attempting to use the latest CodeQL CLI version in the toolcache, as requested by 'tools: ${toolsInput}'.`
-    );
-    const latestToolcacheVersion = getLatestToolcacheVersion(logger);
-    if (latestToolcacheVersion) {
-      cliVersion2 = latestToolcacheVersion;
-    } else {
+    let latestToolcacheVersion;
+    const allowToolcacheValue = isDynamicWorkflow() || isInTestMode();
+    if (allowToolcacheValue) {
       logger.info(
-        `Found no CodeQL CLI in the toolcache, ignoring 'tools: ${toolsInput}'...`
+        `Attempting to use the latest CodeQL CLI version in the toolcache, as requested by 'tools: ${toolsInput}'.`
       );
+      latestToolcacheVersion = getLatestToolcacheVersion(logger);
+      if (latestToolcacheVersion) {
+        cliVersion2 = latestToolcacheVersion;
+      }
+    }
+    if (latestToolcacheVersion === void 0) {
+      if (allowToolcacheValue) {
+        logger.info(
+          `Found no CodeQL CLI in the toolcache, ignoring 'tools: ${toolsInput}'...`
+        );
+      } else {
+        logger.warning(
+          `Ignoring 'tools: ${toolsInput}' because the workflow was not triggered dynamically.`
+        );
+      }
       cliVersion2 = defaultCliVersion.cliVersion;
       tagName = defaultCliVersion.tagName;
     }
diff --git a/lib/upload-sarif-action.js b/lib/upload-sarif-action.js
index 7fa50945e..23c046616 100644
--- a/lib/upload-sarif-action.js
+++ b/lib/upload-sarif-action.js
@@ -90782,16 +90782,27 @@ async function getCodeQLSource(toolsInput, defaultCliVersion, apiDetails, varian
       );
     }
   } else if (toolsInput !== void 0 && toolsInput === CODEQL_TOOLCACHE_INPUT) {
-    logger.info(
-      `Attempting to use the latest CodeQL CLI version in the toolcache, as requested by 'tools: ${toolsInput}'.`
-    );
-    const latestToolcacheVersion = getLatestToolcacheVersion(logger);
-    if (latestToolcacheVersion) {
-      cliVersion2 = latestToolcacheVersion;
-    } else {
+    let latestToolcacheVersion;
+    const allowToolcacheValue = isDynamicWorkflow() || isInTestMode();
+    if (allowToolcacheValue) {
       logger.info(
-        `Found no CodeQL CLI in the toolcache, ignoring 'tools: ${toolsInput}'...`
+        `Attempting to use the latest CodeQL CLI version in the toolcache, as requested by 'tools: ${toolsInput}'.`
       );
+      latestToolcacheVersion = getLatestToolcacheVersion(logger);
+      if (latestToolcacheVersion) {
+        cliVersion2 = latestToolcacheVersion;
+      }
+    }
+    if (latestToolcacheVersion === void 0) {
+      if (allowToolcacheValue) {
+        logger.info(
+          `Found no CodeQL CLI in the toolcache, ignoring 'tools: ${toolsInput}'...`
+        );
+      } else {
+        logger.warning(
+          `Ignoring 'tools: ${toolsInput}' because the workflow was not triggered dynamically.`
+        );
+      }
       cliVersion2 = defaultCliVersion.cliVersion;
       tagName = defaultCliVersion.tagName;
     }
diff --git a/src/setup-codeql.test.ts b/src/setup-codeql.test.ts
index f2bb7efb4..4e65c6edf 100644
--- a/src/setup-codeql.test.ts
+++ b/src/setup-codeql.test.ts
@@ -259,6 +259,8 @@ test("getCodeQLSource correctly returns latest version from toolcache when tools
   const loggedMessages: LoggedMessage[] = [];
   const logger = getRecordingLogger(loggedMessages);
 
+  process.env["GITHUB_EVENT_NAME"] = "dynamic";
+
   const latestToolcacheVersion = "3.2.1";
   const latestVersionPath = "/path/to/latest";
   const testVersions = ["2.3.1", latestToolcacheVersion, "1.2.3"];
@@ -318,6 +320,8 @@ test("getCodeQLSource falls back to downloading the CLI if the toolcache doesn't
   const loggedMessages: LoggedMessage[] = [];
   const logger = getRecordingLogger(loggedMessages);
 
+  process.env["GITHUB_EVENT_NAME"] = "dynamic";
+
   const testVersions = [];
   const findAllVersionsStub = sinon
     .stub(toolcache, "findAllVersions")
diff --git a/src/setup-codeql.ts b/src/setup-codeql.ts
index 43868f253..e01037223 100644
--- a/src/setup-codeql.ts
+++ b/src/setup-codeql.ts
@@ -7,7 +7,7 @@ import { default as deepEqual } from "fast-deep-equal";
 import * as semver from "semver";
 import { v4 as uuidV4 } from "uuid";
 
-import { isRunningLocalAction } from "./actions-util";
+import { isDynamicWorkflow, isRunningLocalAction } from "./actions-util";
 import * as api from "./api-client";
 import * as defaults from "./defaults.json";
 import {
@@ -351,20 +351,37 @@ export async function getCodeQLSource(
     toolsInput !== undefined &&
     toolsInput === CODEQL_TOOLCACHE_INPUT
   ) {
-    // If `toolsInput === "toolcache"`, try to find the latest version of the CLI that's available in the toolcache
-    // and use that. We perform this check here since we can set `cliVersion` directly and don't want to default to
-    // the linked version.
-    logger.info(
-      `Attempting to use the latest CodeQL CLI version in the toolcache, as requested by 'tools: ${toolsInput}'.`,
-    );
+    let latestToolcacheVersion: string | undefined;
 
-    const latestToolcacheVersion = getLatestToolcacheVersion(logger);
-    if (latestToolcacheVersion) {
-      cliVersion = latestToolcacheVersion;
-    } else {
+    // We only allow `toolsInput === "toolcache"` for `dynamic` events. In general, using `toolsInput === "toolcache"`
+    // can lead to alert wobble and so it shouldn't be used for an analysis where results are intended to be uploaded.
+    // We also allow this in test mode.
+    const allowToolcacheValue = isDynamicWorkflow() || util.isInTestMode();
+    if (allowToolcacheValue) {
+      // If `toolsInput === "toolcache"`, try to find the latest version of the CLI that's available in the toolcache
+      // and use that. We perform this check here since we can set `cliVersion` directly and don't want to default to
+      // the linked version.
       logger.info(
-        `Found no CodeQL CLI in the toolcache, ignoring 'tools: ${toolsInput}'...`,
+        `Attempting to use the latest CodeQL CLI version in the toolcache, as requested by 'tools: ${toolsInput}'.`,
       );
+
+      latestToolcacheVersion = getLatestToolcacheVersion(logger);
+      if (latestToolcacheVersion) {
+        cliVersion = latestToolcacheVersion;
+      }
+    }
+
+    if (latestToolcacheVersion === undefined) {
+      if (allowToolcacheValue) {
+        logger.info(
+          `Found no CodeQL CLI in the toolcache, ignoring 'tools: ${toolsInput}'...`,
+        );
+      } else {
+        logger.warning(
+          `Ignoring 'tools: ${toolsInput}' because the workflow was not triggered dynamically.`,
+        );
+      }
+
       cliVersion = defaultCliVersion.cliVersion;
       tagName = defaultCliVersion.tagName;
     }