Tools telemetry: accurately report when feature flags were inaccessible (#1532)

* Cache whether feature flags are accessible

* Small comment fixup from linting change

lib/feature-flags.js

@@ -118,7 +118,7 @@ class GitHubFeatureFlags {
         this.repositoryNwo = repositoryNwo;
         this.featureFlagsFile = featureFlagsFile;
         this.logger = logger;
-        /**/
+        this.hasAccessedRemoteFeatureFlags = false; // Not accessed by default.
     }
     getCliVersionFromFeatureFlag(f) {
         if (!f.startsWith(DEFAULT_VERSION_FEATURE_FLAG_PREFIX) ||
@@ -139,7 +139,9 @@ class GitHubFeatureFlags {
             const defaultDotComCliVersion = await this.getDefaultDotcomCliVersion();
             return {
                 cliVersion: defaultDotComCliVersion.version,
-                toolsFeatureFlagsValid: defaultDotComCliVersion.toolsFeatureFlagsValid,
+                toolsFeatureFlagsValid: this.hasAccessedRemoteFeatureFlags
+                    ? defaultDotComCliVersion.toolsFeatureFlagsValid
+                    : undefined,
                 variant,
             };
         }
@@ -170,7 +172,9 @@ class GitHubFeatureFlags {
                 `shipped with the Action. This is ${defaults.cliVersion}.`);
             return {
                 version: defaults.cliVersion,
-                toolsFeatureFlagsValid: false,
+                toolsFeatureFlagsValid: this.hasAccessedRemoteFeatureFlags
+                    ? false
+                    : undefined,
             };
         }
         const maxCliVersion = enabledFeatureFlagCliVersions.reduce((maxVersion, currentVersion) => currentVersion > maxVersion ? currentVersion : maxVersion, enabledFeatureFlagCliVersions[0]);
@@ -237,6 +241,7 @@ class GitHubFeatureFlags {
         // Do nothing when not running against github.com
         if (this.gitHubVersion.type !== util.GitHubVariant.DOTCOM) {
             this.logger.debug("Not running against github.com. Disabling all toggleable features.");
+            this.hasAccessedRemoteFeatureFlags = false;
             return {};
         }
         try {
@@ -247,6 +252,7 @@ class GitHubFeatureFlags {
             const remoteFlags = response.data;
             this.logger.debug("Loaded the following default values for the feature flags from the Code Scanning API: " +
                 `${JSON.stringify(remoteFlags)}`);
+            this.hasAccessedRemoteFeatureFlags = true;
             return remoteFlags;
         }
         catch (e) {
@@ -255,6 +261,7 @@ class GitHubFeatureFlags {
                     "As a result, it will not be opted into any experimental features. " +
                     "This could be because the Action is running on a pull request from a fork. If not, " +
                     `please ensure the Action has the 'security-events: write' permission. Details: ${e}`);
+                this.hasAccessedRemoteFeatureFlags = false;
                 return {};
             }
             else {
@@ -265,7 +272,6 @@ class GitHubFeatureFlags {
                 throw new Error(`Encountered an error while trying to determine feature enablement: ${e}`);
             }
         }
-        return {};
     }
 }
 //# sourceMappingURL=feature-flags.js.map

src/feature-flags.ts

@@ -153,13 +153,17 @@ export class Features implements FeatureEnablement {
 class GitHubFeatureFlags implements FeatureEnablement {
   private cachedApiResponse: GitHubFeatureFlagsApiResponse | undefined;
 
+  // We cache whether the feature flags were accessed or not in order to accurately report whether flags were
+  // incorrectly configured vs. inaccessible in our telemetry.
+  private hasAccessedRemoteFeatureFlags: boolean;
+
   constructor(
     private readonly gitHubVersion: util.GitHubVersion,
     private readonly repositoryNwo: RepositoryNwo,
     private readonly featureFlagsFile: string,
     private readonly logger: Logger
   ) {
-    /**/
+    this.hasAccessedRemoteFeatureFlags = false; // Not accessed by default.
   }
 
   private getCliVersionFromFeatureFlag(f: string): string | undefined {
@@ -192,7 +196,9 @@ class GitHubFeatureFlags implements FeatureEnablement {
       const defaultDotComCliVersion = await this.getDefaultDotcomCliVersion();
       return {
         cliVersion: defaultDotComCliVersion.version,
-        toolsFeatureFlagsValid: defaultDotComCliVersion.toolsFeatureFlagsValid,
+        toolsFeatureFlagsValid: this.hasAccessedRemoteFeatureFlags
+          ? defaultDotComCliVersion.toolsFeatureFlagsValid
+          : undefined,
         variant,
       };
     }
@@ -205,7 +211,7 @@ class GitHubFeatureFlags implements FeatureEnablement {
 
   async getDefaultDotcomCliVersion(): Promise<{
     version: string;
-    toolsFeatureFlagsValid: boolean;
+    toolsFeatureFlagsValid: boolean | undefined;
   }> {
     const response = await this.getAllFeatures();
 
@@ -233,7 +239,9 @@ class GitHubFeatureFlags implements FeatureEnablement {
       );
       return {
         version: defaults.cliVersion,
-        toolsFeatureFlagsValid: false,
+        toolsFeatureFlagsValid: this.hasAccessedRemoteFeatureFlags
+          ? false
+          : undefined,
       };
     }
 
@@ -331,6 +339,7 @@ class GitHubFeatureFlags implements FeatureEnablement {
       this.logger.debug(
         "Not running against github.com. Disabling all toggleable features."
       );
+      this.hasAccessedRemoteFeatureFlags = false;
       return {};
     }
     try {
@@ -346,6 +355,7 @@ class GitHubFeatureFlags implements FeatureEnablement {
         "Loaded the following default values for the feature flags from the Code Scanning API: " +
           `${JSON.stringify(remoteFlags)}`
       );
+      this.hasAccessedRemoteFeatureFlags = true;
       return remoteFlags;
     } catch (e) {
       if (util.isHTTPError(e) && e.status === 403) {
@@ -355,6 +365,7 @@ class GitHubFeatureFlags implements FeatureEnablement {
             "This could be because the Action is running on a pull request from a fork. If not, " +
             `please ensure the Action has the 'security-events: write' permission. Details: ${e}`
         );
+        this.hasAccessedRemoteFeatureFlags = false;
         return {};
       } else {
         // Some features, such as `ml_powered_queries_enabled` affect the produced alerts.
@@ -366,6 +377,5 @@ class GitHubFeatureFlags implements FeatureEnablement {
         );
       }
     }
-    return {};
   }
 }

src/init-action.ts

@@ -86,7 +86,8 @@ interface InitWithConfigStatusReport extends InitStatusReport {
 interface InitToolsDownloadFields {
   /** Time taken to download the bundle, in milliseconds. */
   tools_download_duration_ms?: number;
-  /** Whether the relevant tools dotcom feature flags have been misconfigured.
+  /**
+   * Whether the relevant tools dotcom feature flags have been misconfigured.
    * Only populated if we attempt to determine the default version based on the dotcom feature flags. */
   tools_feature_flags_valid?: boolean;
 }