Allow excluding traffic by matching headers (#5)

Author: pboos

README.md

@@ -88,6 +88,9 @@ openapi.validation.specification-file-path=folder/my-spec.json
 
 # Comma separated list of paths to be excluded from validation. Default is no excluded paths
 openapi.validation.excluded-paths=/_readiness,/_liveness,/_metrics
+# Allows to exclude requests based on headers. Default is no excluded headers.
+# Each entry is the header plus a matching regex. The regex is case insensitive.
+openapi.validation.excluded-headers[0]=User-Agent: .*(bingbot|googlebot).*
 
 # Throttle the validation reporting (logs & metrics) to a maximum of 1 log/metric per 10 seconds.
 # Default is null which results in no throttling.

openapi-validation-api/src/main/java/com/getyourguide/openapi/validation/api/exclusions/ExcludedHeader.java

@@ -0,0 +1,5 @@
+package com.getyourguide.openapi.validation.api.exclusions;
+
+import java.util.regex.Pattern;
+
+public record ExcludedHeader(String headerName, Pattern headerValuePattern) { }

openapi-validation-api/src/main/java/com/getyourguide/openapi/validation/api/selector/DefaultTrafficSelector.java

@@ -1,7 +1,10 @@
 package com.getyourguide.openapi.validation.api.selector;
 
+import com.getyourguide.openapi.validation.api.exclusions.ExcludedHeader;
 import com.getyourguide.openapi.validation.api.model.RequestMetaData;
 import com.getyourguide.openapi.validation.api.model.ResponseMetaData;
+import java.util.Collections;
+import java.util.List;
 import java.util.Set;
 import java.util.concurrent.ThreadLocalRandom;
 
@@ -11,21 +14,24 @@ public class DefaultTrafficSelector implements TrafficSelector {
 
     private final double sampleRate;
     private final Set<String> excludedPaths;
+    private final List<ExcludedHeader> excludedHeaders;
     private final Boolean shouldFailOnRequestViolation;
     private final Boolean shouldFailOnResponseViolation;
 
-    public DefaultTrafficSelector(Double sampleRate, Set<String> excludedPaths) {
-        this(sampleRate, excludedPaths, null, null);
+    public DefaultTrafficSelector(Double sampleRate, Set<String> excludedPaths, List<ExcludedHeader> excludedHeaders) {
+        this(sampleRate, excludedPaths, excludedHeaders, null, null);
     }
 
     public DefaultTrafficSelector(
         Double sampleRate,
         Set<String> excludedPaths,
+        List<ExcludedHeader> excludedHeaders,
         Boolean shouldFailOnRequestViolation,
         Boolean shouldFailOnResponseViolation
     ) {
         this.sampleRate = sampleRate != null ? sampleRate : SAMPLE_RATE_DEFAULT;
         this.excludedPaths = excludedPaths != null ? excludedPaths : Set.of();
+        this.excludedHeaders = excludedHeaders != null ? excludedHeaders : Collections.emptyList();
         this.shouldFailOnRequestViolation = shouldFailOnRequestViolation != null ? shouldFailOnRequestViolation : false;
         this.shouldFailOnResponseViolation =
             shouldFailOnResponseViolation != null ? shouldFailOnResponseViolation : false;
@@ -65,6 +71,17 @@ public boolean shouldFailOnResponseViolation(RequestMetaData request) {
     }
 
     private boolean isExcludedRequest(RequestMetaData request) {
+        return isRequestExcludedByHeader(request) || isRequestExcludedByPath(request);
+    }
+
+    private boolean isRequestExcludedByHeader(RequestMetaData request) {
+        return excludedHeaders.stream().anyMatch(excludedHeader -> {
+            var headerValue = request.getHeaders().get(excludedHeader.headerName());
+            return headerValue != null && excludedHeader.headerValuePattern().matcher(headerValue).matches();
+        });
+    }
+
+    private boolean isRequestExcludedByPath(RequestMetaData request) {
         return excludedPaths.contains(request.getUri().getPath());
     }
 

openapi-validation-api/src/test/java/com/getyourguide/openapi/validation/api/selector/DefaultTrafficSelectorTest.java

@@ -0,0 +1,56 @@
+package com.getyourguide.openapi.validation.api.selector;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import com.getyourguide.openapi.validation.api.exclusions.ExcludedHeader;
+import com.getyourguide.openapi.validation.api.model.RequestMetaData;
+import java.net.URI;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.regex.Pattern;
+import org.junit.jupiter.api.Test;
+
+class DefaultTrafficSelectorTest {
+    private final TrafficSelector trafficSelector = new DefaultTrafficSelector(
+        1.0,
+        null,
+        List.of(
+            new ExcludedHeader("User-Agent", Pattern.compile(".*(bingbot|googlebot).*", Pattern.CASE_INSENSITIVE)),
+            new ExcludedHeader("x-is-bot", Pattern.compile("true", Pattern.CASE_INSENSITIVE))
+        )
+    );
+
+    @Test
+    public void testIsExcludedByHeaderPattern() {
+        assertHeaderIsExcluded(true,
+            "user-Agent", "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)");
+        assertHeaderIsExcluded(false,
+            "User-Agent",
+            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36");
+
+        assertHeaderIsExcluded(true, "x-is-bot", "true");
+        assertHeaderIsExcluded(false, "x-is-bot", "truebot");
+
+    }
+
+    private void assertHeaderIsExcluded(boolean expectedExclusion, String headerName, String headerValue) {
+
+        var request = new RequestMetaData(
+            "GET",
+            URI.create("https://api.example.com/v1/path"),
+            toCaseInsensitiveMap(Map.of(
+                "Content-Type", "application/json",
+                "Content-Length", "10",
+                headerName, headerValue
+            ))
+        );
+        assertEquals(!expectedExclusion, trafficSelector.shouldRequestBeValidated(request));
+    }
+
+    private Map<String, String> toCaseInsensitiveMap(Map<String, String> map) {
+        var newMap = new TreeMap<String, String>(String.CASE_INSENSITIVE_ORDER);
+        newMap.putAll(map);
+        return newMap;
+    }
+}

spring-boot-starter/spring-boot-starter-core/src/main/java/com/getyourguide/openapi/validation/OpenApiValidationApplicationProperties.java

@@ -2,11 +2,15 @@
 
 import static com.getyourguide.openapi.validation.OpenApiValidationApplicationProperties.PROPERTY_PREFIX;
 
+import com.getyourguide.openapi.validation.api.exclusions.ExcludedHeader;
 import com.getyourguide.openapi.validation.api.metrics.MetricTag;
 import com.getyourguide.openapi.validation.util.CommaSeparatedStringsUtil;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
+import java.util.Objects;
 import java.util.Set;
+import java.util.regex.Pattern;
 import lombok.AllArgsConstructor;
 import lombok.Getter;
 import lombok.NoArgsConstructor;
@@ -27,6 +31,7 @@ public class OpenApiValidationApplicationProperties {
     private String validationReportMetricName;
     private String validationReportMetricAdditionalTags;
     private String excludedPaths;
+    private List<String> excludedHeaders;
     private Boolean shouldFailOnRequestViolation;
     private Boolean shouldFailOnResponseViolation;
 
@@ -49,4 +54,21 @@ public List<MetricTag> getValidationReportMetricAdditionalTags() {
     public Set<String> getExcludedPathsAsSet() {
         return CommaSeparatedStringsUtil.convertCommaSeparatedStringToSet(excludedPaths);
     }
+
+    public List<ExcludedHeader> getExcludedHeaders() {
+        if (excludedHeaders == null) {
+            return Collections.emptyList();
+        }
+
+        return excludedHeaders.stream()
+            .map(header -> {
+                var parts = header.split(":", 2);
+                if (parts.length != 2) {
+                    return null;
+                }
+                return new ExcludedHeader(parts[0].trim(), Pattern.compile(parts[1].trim(), Pattern.CASE_INSENSITIVE));
+            })
+            .filter(Objects::nonNull)
+            .toList();
+    }
 }

spring-boot-starter/spring-boot-starter-core/src/main/java/com/getyourguide/openapi/validation/autoconfigure/FallbackLibraryAutoConfiguration.java

@@ -24,6 +24,7 @@ public TrafficSelector defaultTrafficSelector() {
         return new DefaultTrafficSelector(
             properties.getSampleRate(),
             properties.getExcludedPathsAsSet(),
+            properties.getExcludedHeaders(),
             properties.getShouldFailOnRequestViolation(),
             properties.getShouldFailOnResponseViolation()
         );

spring-boot-starter/spring-boot-starter-core/src/main/resources/META-INF/spring-configuration-metadata.json

@@ -15,6 +15,11 @@
       "type": "java.lang.String",
       "description": "Comma separated list of paths to be excluded from validation. Default is no excluded paths."
     },
+    {
+      "name": "openapi.validation.excluded-headers",
+      "type": "java.util.List<java.lang.String>",
+      "description": "Headers with patterns to be excluded. e.g. `User-Agent: .*(bingbot|googlebot).*`. Default is no excluded paths."
+    },
     {
       "name": "openapi.validation.validation-report-throttle-wait-seconds",
       "type": "java.lang.Integer",

spring-boot-starter/spring-boot-starter-core/src/test/java/com/getyourguide/openapi/validation/OpenApiValidationApplicationPropertiesTest.java

@@ -4,6 +4,7 @@
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
+import com.getyourguide.openapi.validation.api.exclusions.ExcludedHeader;
 import com.getyourguide.openapi.validation.api.metrics.MetricTag;
 import java.util.List;
 import java.util.Set;
@@ -17,6 +18,7 @@ class OpenApiValidationApplicationPropertiesTest {
     private static final String VALIDATION_REPORT_METRIC_NAME = "openapi_validation_error";
     private static final String VALIDATION_REPORT_METRIC_ADDITONAL_TAGS_STRING = "service=payment,team=chk";
     private static final String EXCLUDED_PATHS = "/_readiness,/_liveness,/_metrics";
+    private static final List<String> EXCLUDED_HEADERS = List.of("User-Agent: .*(bingbot|googlebot).*", "x-is-bot: true");
 
     @Test
     void getters() {
@@ -27,21 +29,38 @@ void getters() {
             VALIDATION_REPORT_METRIC_NAME,
             VALIDATION_REPORT_METRIC_ADDITONAL_TAGS_STRING,
             EXCLUDED_PATHS,
+            EXCLUDED_HEADERS,
             true,
             false
         );
 
         assertEquals(SAMPLE_RATE, loggingConfiguration.getSampleRate());
         assertEquals(SPECIFICATION_FILE_PATH, loggingConfiguration.getSpecificationFilePath());
-        assertEquals(VALIDATION_REPORT_THROTTLE_WAIT_SECONDS, loggingConfiguration.getValidationReportThrottleWaitSeconds());
+        assertEquals(VALIDATION_REPORT_THROTTLE_WAIT_SECONDS,
+            loggingConfiguration.getValidationReportThrottleWaitSeconds());
         assertEquals(VALIDATION_REPORT_METRIC_NAME, loggingConfiguration.getValidationReportMetricName());
         assertEquals(
             List.of(new MetricTag("service", "payment"), new MetricTag("team", "chk")),
             loggingConfiguration.getValidationReportMetricAdditionalTags()
         );
         assertEquals(EXCLUDED_PATHS, loggingConfiguration.getExcludedPaths());
-        assertEquals(Set.of("/_readiness","/_liveness","/_metrics"), loggingConfiguration.getExcludedPathsAsSet());
+        assertExcludedHeaders(loggingConfiguration.getExcludedHeaders());
+        assertEquals(Set.of("/_readiness", "/_liveness", "/_metrics"), loggingConfiguration.getExcludedPathsAsSet());
         assertTrue(loggingConfiguration.getShouldFailOnRequestViolation());
         assertFalse(loggingConfiguration.getShouldFailOnResponseViolation());
     }
+
+    private void assertExcludedHeaders(List<ExcludedHeader> excludedHeaders) {
+        assertEquals(EXCLUDED_HEADERS.size(), excludedHeaders.size());
+        for (int i = 0; i < EXCLUDED_HEADERS.size(); i++) {
+            assertExcludedHeader(excludedHeaders, i);
+        }
+    }
+
+    private static void assertExcludedHeader(List<ExcludedHeader> excludedHeaders, int index) {
+        var excludedHeader = EXCLUDED_HEADERS.get(index);
+        var parts = excludedHeader.split(":");
+        assertEquals(parts[0].trim(), excludedHeaders.get(index).headerName());
+        assertEquals(parts[1].trim(), excludedHeaders.get(index).headerValuePattern().pattern());
+    }
 }