Add telemetry around signature validation (#3410) (#3415)

* Added helper class and methods to track signature validation telemetry. Added tests

* Added a counter for signature validation. Updated ITelemetryClient interface and TelemetryClient to support the new telemetry. Added null telemetry client for no op

* Added telemetry to JWT and SAML/SAML2 handlers. Expanded catching of exceptions to identify the stage at which the signature validation failed.

* Added tests

* Addressed Copilot's feedback

* Reverted readonly changes to allow setting the telemetry client in tests

* Replaced lock with volatile modifier for immutable array, removed the hashset in favour of the array to avoid converting in the getter.

* Updated CryptoTelemetry's public API and tests

* Added benchmarks for signature validation telemetry

* Removed issuer caching from CryptoTelemetry signature validation telemetry. Updated tests

* Updated public API to match the updated enable telemetry method

* Addressed PR feedback

- Rename _telemetryClient to TelemetryClient in all token handlers and update all references accordingly.
- Simplify CryptoTelemetry.GetTrackedIssuerOrOther: now matches tracked issuers by substring (case-insensitive) instead of parsing host; remove ExtractHostFromIssuer.
- Update comments to clarify substring matching and its limitations.
- Remove TelemetryConfiguration enum from benchmarks; update benchmark attributes and tracked issuer values for consistency.
- Refactor and expand CryptoTelemetry tests: remove host extraction tests, consolidate key algorithm ID tests, and add more scenarios for tracked issuer matching and allowlist filtering.
- Update API documentation to reflect field renaming.
- Overall, unify telemetry client usage and streamline issuer tracking logic, with tests updated to match new behavior.

* Removed case insensitive comparison for telemetry issuer extraction based on PR feedback

(cherry picked from commit 61449b8)

Author: iNinja

benchmark/Microsoft.IdentityModel.Benchmarks/ValidateTokenAsyncTests.cs

@@ -10,6 +10,7 @@
 using BenchmarkDotNet.Attributes;
 using BenchmarkDotNet.Configs;
 using Microsoft.IdentityModel.JsonWebTokens;
+using Microsoft.IdentityModel.Telemetry;
 using Microsoft.IdentityModel.Tokens;
 using Microsoft.IdentityModel.Tokens.Experimental;
 
@@ -193,4 +194,75 @@ public async Task<List<Claim>> JsonWebTokenHandler_ValidateTokenAsyncWithVP_Crea
             return claims.ToList();
         }
     }
+
+    // ===== Telemetry Impact Benchmarks =====
+    // "Tracking" in this context refers to tracking signature validation for specific issuer hosts.
+    // When enabled, telemetry collects data for the configured hosts (e.g., "contoso.com").
+    // Every other host is reported as "other" to avoid excessive cardinality in telemetry.
+    // These benchmarks measure the performance impact of enabling telemetry overall, and of tracking specific hosts.
+    [GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory)]
+    public class ValidateTokenAsyncTests_TelemetryImpact
+    {
+        private const int IterationCount = 10000;
+        private JsonWebTokenHandler _jsonWebTokenHandler;
+        private string _jwsClaims;
+        private TokenValidationParameters _tokenValidationParameters;
+
+        [GlobalSetup]
+        public void GlobalSetup()
+        {
+            var tokenDescriptorClaims = new SecurityTokenDescriptor
+            {
+                Claims = BenchmarkUtils.Claims,
+                SigningCredentials = BenchmarkUtils.SigningCredentialsRsaSha256,
+            };
+
+            _jsonWebTokenHandler = new JsonWebTokenHandler();
+            _jwsClaims = _jsonWebTokenHandler.CreateToken(tokenDescriptorClaims);
+
+            _tokenValidationParameters = new TokenValidationParameters()
+            {
+                ValidAudience = BenchmarkUtils.Audience,
+                ValidateLifetime = true,
+                ValidIssuer = BenchmarkUtils.Issuer,
+                IssuerSigningKey = BenchmarkUtils.SigningCredentialsRsaSha256.Key,
+            };
+        }
+
+        [IterationSetup(Target = nameof(JsonWebTokenHandler_ValidateTokenAsync_TelemetryDisabled))]
+        public void Setup_TelemetryDisabled()
+        {
+            CryptoTelemetry.EnableSignatureValidationTelemetry(false, null);
+        }
+
+        [BenchmarkCategory("ValidateTokenAsync_TelemetryImpact"), Benchmark(Baseline = true, OperationsPerInvoke = IterationCount)]
+        public async Task<TokenValidationResult> JsonWebTokenHandler_ValidateTokenAsync_TelemetryDisabled()
+        {
+            return await _jsonWebTokenHandler.ValidateTokenAsync(_jwsClaims, _tokenValidationParameters).ConfigureAwait(false);
+        }
+
+        [IterationSetup(Target = nameof(JsonWebTokenHandler_ValidateTokenAsync_TelemetryEnabledNoTracking))]
+        public void Setup_TelemetryEnabledNoTracking()
+        {
+            CryptoTelemetry.EnableSignatureValidationTelemetry(true, null);
+        }
+
+        [BenchmarkCategory("ValidateTokenAsync_TelemetryImpact"), Benchmark(OperationsPerInvoke = IterationCount)]
+        public async Task<TokenValidationResult> JsonWebTokenHandler_ValidateTokenAsync_TelemetryEnabledNoTracking()
+        {
+            return await _jsonWebTokenHandler.ValidateTokenAsync(_jwsClaims, _tokenValidationParameters).ConfigureAwait(false);
+        }
+
+        [IterationSetup(Target = nameof(JsonWebTokenHandler_ValidateTokenAsync_TelemetryEnabledWithTracking))]
+        public void Setup_TelemetryEnabledWithTracking()
+        {
+            CryptoTelemetry.EnableSignatureValidationTelemetry(true, new[] { "contoso.com" });
+        }
+
+        [BenchmarkCategory("ValidateTokenAsync_TelemetryImpact"), Benchmark(OperationsPerInvoke = IterationCount)]
+        public async Task<TokenValidationResult> JsonWebTokenHandler_ValidateTokenAsync_TelemetryEnabledWithTracking()
+        {
+            return await _jsonWebTokenHandler.ValidateTokenAsync(_jwsClaims, _tokenValidationParameters).ConfigureAwait(false);
+        }
+    }
 }

src/Microsoft.IdentityModel.JsonWebTokens/Experimental/JsonWebTokenHandler.ValidateSignature.cs

@@ -5,6 +5,7 @@
 using System.Collections.Generic;
 using System.Text;
 using Microsoft.IdentityModel.Logging;
+using Microsoft.IdentityModel.Telemetry;
 using Microsoft.IdentityModel.Tokens;
 using Microsoft.IdentityModel.Tokens.Experimental;
 using TokenLogMessages = Microsoft.IdentityModel.Tokens.LogMessages;
@@ -23,7 +24,7 @@ public partial class JsonWebTokenHandler : TokenHandler
         /// <param name="configuration">The optional configuration used for validation.</param>
         /// <param name="callContext">The context in which the method is called.</param>
         /// <returns>A <see cref="ValidationResult{SecurityKey, ValidationError}"/> with the <see cref="SecurityKey"/> that signed the tokenif valid or a <see cref="ValidationError"/>.</returns>
-        internal static ValidationResult<SecurityKey, ValidationError> ValidateSignature(
+        internal ValidationResult<SecurityKey, ValidationError> ValidateSignature(
             JsonWebToken jwtToken,
             ValidationParameters validationParameters,
             BaseConfiguration? configuration,
@@ -101,6 +102,12 @@ internal static ValidationResult<SecurityKey, ValidationError> ValidateSignature
             if (validationParameters.TryAllSigningKeys)
                 return ValidateSignatureUsingAllKeys(jwtToken, validationParameters, configuration, callContext);
 
+            RecordSignatureValidationTelemetry(
+                TelemetryClient,
+                TelemetryConstants.SignatureValidationErrors.SigningKeyNotFound,
+                jwtToken,
+                key: null);
+
             // kid was NOT found, no matching keys available.
             if (string.IsNullOrEmpty(jwtToken.Kid))
             {
@@ -126,7 +133,7 @@ internal static ValidationResult<SecurityKey, ValidationError> ValidateSignature
                 ValidationError.GetCurrentStackFrame());
         }
 
-        private static ValidationResult<SecurityKey, ValidationError> ValidateSignatureUsingAllKeys(
+        private ValidationResult<SecurityKey, ValidationError> ValidateSignatureUsingAllKeys(
             JsonWebToken jwtToken,
             ValidationParameters validationParameters,
             BaseConfiguration? configuration,
@@ -218,7 +225,7 @@ private static ValidationResult<SecurityKey, ValidationError> ValidateSignatureU
                 ValidationError.GetCurrentStackFrame());
         }
 
-        private static ValidationResult<SecurityKey, ValidationError> ValidateSignatureWithKey(
+        private ValidationResult<SecurityKey, ValidationError> ValidateSignatureWithKey(
             JsonWebToken jsonWebToken,
             ValidationParameters validationParameters,
             SecurityKey key,
@@ -229,6 +236,12 @@ private static ValidationResult<SecurityKey, ValidationError> ValidateSignatureW
             CryptoProviderFactory cryptoProviderFactory = validationParameters.CryptoProviderFactory ?? key.CryptoProviderFactory;
             if (!cryptoProviderFactory.IsSupportedAlgorithm(jsonWebToken.Alg, key))
             {
+                RecordSignatureValidationTelemetry(
+                    TelemetryClient,
+                    TelemetryConstants.SignatureValidationErrors.AlgorithmNotSupported,
+                    jsonWebToken,
+                    key);
+
                 return new SignatureValidationError(
                     new MessageDetail(
                         TokenLogMessages.IDX10652,
@@ -242,13 +255,21 @@ private static ValidationResult<SecurityKey, ValidationError> ValidateSignatureW
             try
             {
                 if (signatureProvider == null)
+                {
+                    RecordSignatureValidationTelemetry(
+                        TelemetryClient,
+                        TelemetryConstants.SignatureValidationErrors.SignatureProviderCreationFailed,
+                        jsonWebToken,
+                        key);
+
                     return new SignatureValidationError(
                         new MessageDetail(
                             TokenLogMessages.IDX10636,
                             LogHelper.MarkAsNonPII(key?.KeyId ?? "Null"),
                             LogHelper.MarkAsNonPII(jsonWebToken.Alg)),
                         ValidationFailureType.CryptoProviderReturnedNull,
                         ValidationError.GetCurrentStackFrame());
+                }
 
                 bool valid = EncodingUtils.PerformEncodingDependentOperation<bool, string, int, SignatureProvider>(
                     jsonWebToken.EncodedToken,
@@ -262,21 +283,41 @@ private static ValidationResult<SecurityKey, ValidationError> ValidateSignatureW
 
                 if (valid)
                 {
+                    RecordSignatureValidationTelemetry(
+                        TelemetryClient,
+                        TelemetryConstants.SignatureValidationErrors.None,
+                        jsonWebToken,
+                        key);
+
                     jsonWebToken.SigningKey = key;
                     return key;
                 }
                 else
+                {
+                    RecordSignatureValidationTelemetry(
+                        TelemetryClient,
+                        TelemetryConstants.SignatureValidationErrors.SignatureVerificationFailed,
+                        jsonWebToken,
+                        key);
+
                     return new SignatureValidationError(
                         new MessageDetail(
                             TokenLogMessages.IDX10520,
                             LogHelper.MarkAsNonPII(key.ToString())),
                         SignatureValidationFailure.ValidationFailed,
                         ValidationError.GetCurrentStackFrame());
+                }
             }
 #pragma warning disable CA1031 // Do not catch general exception types
             catch (Exception ex)
 #pragma warning restore CA1031 // Do not catch general exception types
             {
+                RecordSignatureValidationTelemetry(
+                    TelemetryClient,
+                    TelemetryConstants.SignatureValidationErrors.SignatureVerificationFailed,
+                    jsonWebToken,
+                    key);
+
                 return new SignatureValidationError(
                     new MessageDetail(
                         TokenLogMessages.IDX10521,

src/Microsoft.IdentityModel.JsonWebTokens/InternalAPI.Shipped.txt

@@ -105,7 +105,7 @@ Microsoft.IdentityModel.JsonWebTokens.JsonWebToken.ReadToken(System.ReadOnlyMemo
 Microsoft.IdentityModel.JsonWebTokens.JsonWebToken.Typ.set -> void
 Microsoft.IdentityModel.JsonWebTokens.JsonWebToken.ValidFromNullable.get -> System.DateTime?
 Microsoft.IdentityModel.JsonWebTokens.JsonWebToken.ValidToNullable.get -> System.DateTime?
-Microsoft.IdentityModel.JsonWebTokens.JsonWebTokenHandler._telemetryClient -> Microsoft.IdentityModel.Telemetry.ITelemetryClient
+Microsoft.IdentityModel.JsonWebTokens.JsonWebTokenHandler.TelemetryClient -> Microsoft.IdentityModel.Telemetry.ITelemetryClient
 Microsoft.IdentityModel.JsonWebTokens.JsonWebTokenHandler.GetContentEncryptionKeys(Microsoft.IdentityModel.JsonWebTokens.JsonWebToken jwtToken, Microsoft.IdentityModel.Tokens.TokenValidationParameters validationParameters, Microsoft.IdentityModel.Tokens.BaseConfiguration configuration) -> System.Collections.Generic.IEnumerable<Microsoft.IdentityModel.Tokens.SecurityKey>
 Microsoft.IdentityModel.JsonWebTokens.JsonWebTokenHandler.GetContentEncryptionKeys(Microsoft.IdentityModel.JsonWebTokens.JsonWebToken jwtToken, Microsoft.IdentityModel.Tokens.ValidationParameters validationParameters, Microsoft.IdentityModel.Tokens.BaseConfiguration configuration, Microsoft.IdentityModel.Tokens.CallContext callContext) -> (System.Collections.Generic.IList<Microsoft.IdentityModel.Tokens.SecurityKey>, Microsoft.IdentityModel.Tokens.ValidationError)
 Microsoft.IdentityModel.JsonWebTokens.JsonWebTokenHandler.StackFrames