From aba526cd1795ee75e36e38f50ccdf3ccc0f1e78a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mario=20Vidal=20Dom=C3=ADnguez?= Date: Wed, 12 Feb 2025 16:12:04 +0100 Subject: [PATCH 1/3] Upgrade libddwaf and add InputTruncated metric --- dd-java-agent/appsec/build.gradle | 2 +- .../datadog/appsec/gateway/GatewayBridge.java | 18 +++++++++++++++ .../trace/api/telemetry/TruncatedType.java | 22 ++++++++++++++++++ .../api/telemetry/WafMetricCollector.java | 23 +++++++++++++++++++ 4 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 internal-api/src/main/java/datadog/trace/api/telemetry/TruncatedType.java diff --git a/dd-java-agent/appsec/build.gradle b/dd-java-agent/appsec/build.gradle index f501d453cec..afb3aea1659 100644 --- a/dd-java-agent/appsec/build.gradle +++ b/dd-java-agent/appsec/build.gradle @@ -15,7 +15,7 @@ dependencies { implementation project(':internal-api') implementation project(':communication') implementation project(':telemetry') - implementation group: 'io.sqreen', name: 'libsqreen', version: '11.2.0' + implementation group: 'io.sqreen', name: 'libsqreen', version: '11.3.0' implementation libs.moshi testImplementation libs.bytebuddy diff --git a/dd-java-agent/appsec/src/main/java/com/datadog/appsec/gateway/GatewayBridge.java b/dd-java-agent/appsec/src/main/java/com/datadog/appsec/gateway/GatewayBridge.java index 9f1285649b5..b7e2e9a4247 100644 --- a/dd-java-agent/appsec/src/main/java/com/datadog/appsec/gateway/GatewayBridge.java +++ b/dd-java-agent/appsec/src/main/java/com/datadog/appsec/gateway/GatewayBridge.java @@ -38,11 +38,14 @@ import datadog.trace.api.internal.TraceSegment; import datadog.trace.api.telemetry.LoginEvent; import datadog.trace.api.telemetry.RuleType; +import datadog.trace.api.telemetry.TruncatedType; import datadog.trace.api.telemetry.WafMetricCollector; import datadog.trace.bootstrap.instrumentation.api.Tags; import datadog.trace.bootstrap.instrumentation.api.URIDataAdapter; import datadog.trace.util.stacktrace.StackTraceEvent; import datadog.trace.util.stacktrace.StackUtils; +import io.sqreen.powerwaf.PowerwafMetrics; +import io.sqreen.powerwaf.metrics.InputTruncatedType; import java.net.URI; import java.net.URISyntaxException; import java.nio.charset.Charset; @@ -822,6 +825,21 @@ private NoopFlow onRequestEnded(RequestContext ctx_, IGSpanInfo spanInfo) { log.debug("Unable to commit, derivatives will be skipped {}", ctx.getDerivativeKeys()); } + PowerwafMetrics wafMetrics = ctx.getWafMetrics(); + if (wafMetrics != null) { + final long stringTooLong = + wafMetrics.getWafInputsTruncatedCount(InputTruncatedType.STRING_TOO_LONG); + final long listMapTooLarge = + wafMetrics.getWafInputsTruncatedCount(InputTruncatedType.LIST_MAP_TOO_LARGE); + final long objectTooDeep = + wafMetrics.getWafInputsTruncatedCount(InputTruncatedType.OBJECT_TOO_DEEP); + + WafMetricCollector.get().wafInputTruncated(TruncatedType.STRING_TOO_LONG, stringTooLong); + WafMetricCollector.get() + .wafInputTruncated(TruncatedType.LIST_MAP_TOO_LARGE, listMapTooLarge); + WafMetricCollector.get().wafInputTruncated(TruncatedType.OBJECT_TOO_DEEP, objectTooDeep); + } + if (ctx.isBlocked()) { WafMetricCollector.get().wafRequestBlocked(); } else if (!collectedEvents.isEmpty()) { diff --git a/internal-api/src/main/java/datadog/trace/api/telemetry/TruncatedType.java b/internal-api/src/main/java/datadog/trace/api/telemetry/TruncatedType.java new file mode 100644 index 00000000000..eb865f61827 --- /dev/null +++ b/internal-api/src/main/java/datadog/trace/api/telemetry/TruncatedType.java @@ -0,0 +1,22 @@ +package datadog.trace.api.telemetry; + +public enum TruncatedType { + STRING_TOO_LONG(1), + LIST_MAP_TOO_LARGE(2), + OBJECT_TOO_DEEP(4); + + private final int value; + private static final int numValues = RuleType.values().length; + + TruncatedType(int value) { + this.value = value; + } + + public int getValue() { + return this.value; + } + + public static int getNumValues() { + return numValues; + } +} diff --git a/internal-api/src/main/java/datadog/trace/api/telemetry/WafMetricCollector.java b/internal-api/src/main/java/datadog/trace/api/telemetry/WafMetricCollector.java index 9266b74691c..c4b795a8d7d 100644 --- a/internal-api/src/main/java/datadog/trace/api/telemetry/WafMetricCollector.java +++ b/internal-api/src/main/java/datadog/trace/api/telemetry/WafMetricCollector.java @@ -34,6 +34,8 @@ private WafMetricCollector() { private static final AtomicRequestCounter wafTriggeredRequestCounter = new AtomicRequestCounter(); private static final AtomicRequestCounter wafBlockedRequestCounter = new AtomicRequestCounter(); private static final AtomicRequestCounter wafTimeoutRequestCounter = new AtomicRequestCounter(); + private static final AtomicLongArray wafInputTruncatedCounter = + new AtomicLongArray(TruncatedType.getNumValues()); private static final AtomicLongArray raspRuleEvalCounter = new AtomicLongArray(RuleType.getNumValues()); private static final AtomicLongArray raspRuleMatchCounter = @@ -92,6 +94,10 @@ public void wafRequestTimeout() { wafTimeoutRequestCounter.increment(); } + public void wafInputTruncated(final TruncatedType truncatedType, long increment) { + wafInputTruncatedCounter.addAndGet(truncatedType.ordinal(), increment); + } + public void raspRuleEval(final RuleType ruleType) { raspRuleEvalCounter.incrementAndGet(ruleType.ordinal()); } @@ -183,6 +189,17 @@ public void prepareMetrics() { } } + // Input Truncated Type metric + for (TruncatedType truncatedType : TruncatedType.values()) { + long counter = wafInputTruncatedCounter.getAndSet(truncatedType.ordinal(), 0); + if (counter > 0) { + if (!rawMetricsQueue.offer( + new WafInputTruncatedRawMetric(counter, truncatedType.getValue()))) { + return; + } + } + } + // RASP rule eval per rule type for (RuleType ruleType : RuleType.values()) { long counter = raspRuleEvalCounter.getAndSet(ruleType.ordinal(), 0); @@ -319,6 +336,12 @@ public WafRequestsRawMetric( } } + public static class WafInputTruncatedRawMetric extends WafMetric { + public WafInputTruncatedRawMetric(final long counter, final int truncationReason) { + super("waf.input_truncated", counter, "truncation_reason:" + truncationReason); + } + } + public static class RaspRuleEval extends WafMetric { public RaspRuleEval(final long counter, final RuleType ruleType, final String wafVersion) { super( From 5a94e18fb29104f64dc5df1af45c8c9d6bcf980b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mario=20Vidal=20Dom=C3=ADnguez?= Date: Wed, 12 Feb 2025 16:33:13 +0100 Subject: [PATCH 2/3] Add tests --- .../PowerWAFModuleSpecification.groovy | 15 ++++++-- .../telemetry/WafMetricCollectorTest.groovy | 34 +++++++++++++++++-- 2 files changed, 43 insertions(+), 6 deletions(-) diff --git a/dd-java-agent/appsec/src/test/groovy/com/datadog/appsec/powerwaf/PowerWAFModuleSpecification.groovy b/dd-java-agent/appsec/src/test/groovy/com/datadog/appsec/powerwaf/PowerWAFModuleSpecification.groovy index b5f5403bbd1..c8fff877b0e 100644 --- a/dd-java-agent/appsec/src/test/groovy/com/datadog/appsec/powerwaf/PowerWAFModuleSpecification.groovy +++ b/dd-java-agent/appsec/src/test/groovy/com/datadog/appsec/powerwaf/PowerWAFModuleSpecification.groovy @@ -37,6 +37,7 @@ import spock.lang.Shared import spock.lang.Unroll import java.util.concurrent.CountDownLatch +import java.util.concurrent.atomic.AtomicLong import static datadog.trace.api.config.AppSecConfig.APPSEC_OBFUSCATION_PARAMETER_KEY_REGEXP import static datadog.trace.api.config.AppSecConfig.APPSEC_OBFUSCATION_PARAMETER_VALUE_REGEXP @@ -546,9 +547,9 @@ class PowerWAFModuleSpecification extends DDSpecification { 1 * ctx.getOrCreateAdditive(_, true, false) >> { pwafAdditive = it[0].openAdditive() } - 3 * tracer.activeSpan() + 2 * tracer.activeSpan() // we get two events: one for origin rule, and one for the custom one - 1 * ctx.reportEvents(hasSize(2)) + 1 * ctx.reportEvents(hasSize(1)) 1 * ctx.getWafMetrics() 1 * ctx.isAdditiveClosed() >> false 1 * ctx.closeAdditive() @@ -777,7 +778,15 @@ class PowerWAFModuleSpecification extends DDSpecification { pwafAdditive } 1 * ctx.closeAdditive() - 2 * ctx.getWafMetrics() >> { metrics.with { totalDdwafRunTimeNs = 1000; totalRunTimeNs = 2000; it} } + 2 * ctx.getWafMetrics() >> { + metrics.with { + totalDdwafRunTimeNs = new AtomicLong(1000) + totalRunTimeNs = new AtomicLong(2000) + wafInputsTruncatedStringTooLongCount = new AtomicLong(0) + wafInputsTruncatedListMapTooLargeCount = new AtomicLong(0) + wafInputsTruncatedObjectTooDeepCount = new AtomicLong(0) + it + } } 1 * segment.setTagTop('_dd.appsec.waf.duration', 1) 1 * segment.setTagTop('_dd.appsec.waf.duration_ext', 2) diff --git a/internal-api/src/test/groovy/datadog/trace/api/telemetry/WafMetricCollectorTest.groovy b/internal-api/src/test/groovy/datadog/trace/api/telemetry/WafMetricCollectorTest.groovy index 5f532b3ed8f..e6cc61ca829 100644 --- a/internal-api/src/test/groovy/datadog/trace/api/telemetry/WafMetricCollectorTest.groovy +++ b/internal-api/src/test/groovy/datadog/trace/api/telemetry/WafMetricCollectorTest.groovy @@ -27,6 +27,7 @@ class WafMetricCollectorTest extends DDSpecification { WafMetricCollector.get().wafRequestTriggered() WafMetricCollector.get().wafRequestBlocked() WafMetricCollector.get().wafRequestTimeout() + WafMetricCollector.get().wafInputTruncated(TruncatedType.STRING_TOO_LONG, 5) WafMetricCollector.get().raspRuleEval(RuleType.SQL_INJECTION) WafMetricCollector.get().raspRuleEval(RuleType.SQL_INJECTION) WafMetricCollector.get().raspRuleMatch(RuleType.SQL_INJECTION) @@ -111,21 +112,28 @@ class WafMetricCollectorTest extends DDSpecification { 'waf_timeout:true' ].toSet() - def raspRuleEvalSqli = (WafMetricCollector.RaspRuleEval)metrics[7] + def inputTruncatedStringTooLong = (WafMetricCollector.WafInputTruncatedRawMetric)metrics[7] + inputTruncatedStringTooLong.type == 'count' + inputTruncatedStringTooLong.value == 5 + inputTruncatedStringTooLong.namespace == 'appsec' + inputTruncatedStringTooLong.metricName == 'waf.input_truncated' + inputTruncatedStringTooLong.tags.toSet() == ['truncation_reason:1'].toSet() + + def raspRuleEvalSqli = (WafMetricCollector.RaspRuleEval)metrics[8] raspRuleEvalSqli.type == 'count' raspRuleEvalSqli.value == 3 raspRuleEvalSqli.namespace == 'appsec' raspRuleEvalSqli.metricName == 'rasp.rule.eval' raspRuleEvalSqli.tags.toSet() == ['rule_type:sql_injection', 'waf_version:waf_ver1'].toSet() - def raspRuleMatch = (WafMetricCollector.RaspRuleMatch)metrics[8] + def raspRuleMatch = (WafMetricCollector.RaspRuleMatch)metrics[9] raspRuleMatch.type == 'count' raspRuleMatch.value == 1 raspRuleMatch.namespace == 'appsec' raspRuleMatch.metricName == 'rasp.rule.match' raspRuleMatch.tags.toSet() == ['rule_type:sql_injection', 'waf_version:waf_ver1'].toSet() - def raspTimeout = (WafMetricCollector.RaspTimeout)metrics[9] + def raspTimeout = (WafMetricCollector.RaspTimeout)metrics[10] raspTimeout.type == 'count' raspTimeout.value == 1 raspTimeout.namespace == 'appsec' @@ -296,6 +304,26 @@ class WafMetricCollectorTest extends DDSpecification { } } + def "test WAF #inputTruncated metrics"() { + when: + WafMetricCollector.get().wafInit('waf_ver1', 'rules.1', true) + WafMetricCollector.get().wafInputTruncated(truncatedType, 5) + WafMetricCollector.get().prepareMetrics() + + then: + def metrics = WafMetricCollector.get().drain() + + def inputTruncated = (WafMetricCollector.WafInputTruncatedRawMetric)metrics[1] + inputTruncated.type == 'count' + inputTruncated.value == 5 + inputTruncated.namespace == 'appsec' + inputTruncated.metricName == 'waf.input_truncated' + inputTruncated.tags.toSet() == ['truncation_reason:' + truncatedType.value].toSet() + + where: + truncatedType << [TruncatedType.STRING_TOO_LONG, TruncatedType.LIST_MAP_TOO_LARGE, TruncatedType.OBJECT_TOO_DEEP] + } + def "test Rasp #ruleType metrics"() { when: WafMetricCollector.get().wafInit('waf_ver1', 'rules.1', true) From d3cbc7ef0c441fbb6fa5c60ba7a8a4b6654721e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mario=20Vidal=20Dom=C3=ADnguez?= Date: Wed, 12 Feb 2025 18:08:29 +0100 Subject: [PATCH 3/3] Adapt to new version of libddwaf-java --- .../datadog/appsec/gateway/GatewayBridge.java | 26 ++++++++++--------- .../trace/api/telemetry/TruncatedType.java | 5 ---- .../api/telemetry/WafMetricCollector.java | 2 +- .../telemetry/WafMetricCollectorTest.groovy | 2 +- 4 files changed, 16 insertions(+), 19 deletions(-) diff --git a/dd-java-agent/appsec/src/main/java/com/datadog/appsec/gateway/GatewayBridge.java b/dd-java-agent/appsec/src/main/java/com/datadog/appsec/gateway/GatewayBridge.java index b7e2e9a4247..f0221fc87c8 100644 --- a/dd-java-agent/appsec/src/main/java/com/datadog/appsec/gateway/GatewayBridge.java +++ b/dd-java-agent/appsec/src/main/java/com/datadog/appsec/gateway/GatewayBridge.java @@ -45,7 +45,6 @@ import datadog.trace.util.stacktrace.StackTraceEvent; import datadog.trace.util.stacktrace.StackUtils; import io.sqreen.powerwaf.PowerwafMetrics; -import io.sqreen.powerwaf.metrics.InputTruncatedType; import java.net.URI; import java.net.URISyntaxException; import java.nio.charset.Charset; @@ -827,17 +826,20 @@ private NoopFlow onRequestEnded(RequestContext ctx_, IGSpanInfo spanInfo) { PowerwafMetrics wafMetrics = ctx.getWafMetrics(); if (wafMetrics != null) { - final long stringTooLong = - wafMetrics.getWafInputsTruncatedCount(InputTruncatedType.STRING_TOO_LONG); - final long listMapTooLarge = - wafMetrics.getWafInputsTruncatedCount(InputTruncatedType.LIST_MAP_TOO_LARGE); - final long objectTooDeep = - wafMetrics.getWafInputsTruncatedCount(InputTruncatedType.OBJECT_TOO_DEEP); - - WafMetricCollector.get().wafInputTruncated(TruncatedType.STRING_TOO_LONG, stringTooLong); - WafMetricCollector.get() - .wafInputTruncated(TruncatedType.LIST_MAP_TOO_LARGE, listMapTooLarge); - WafMetricCollector.get().wafInputTruncated(TruncatedType.OBJECT_TOO_DEEP, objectTooDeep); + final long stringTooLong = wafMetrics.getWafInputsTruncatedStringTooLongCount(); + final long listMapTooLarge = wafMetrics.getWafInputsTruncatedListMapTooLargeCount(); + final long objectTooDeep = wafMetrics.getWafInputsTruncatedObjectTooDeepCount(); + + if (stringTooLong > 0) { + WafMetricCollector.get().wafInputTruncated(TruncatedType.STRING_TOO_LONG, stringTooLong); + } + if (listMapTooLarge > 0) { + WafMetricCollector.get() + .wafInputTruncated(TruncatedType.LIST_MAP_TOO_LARGE, listMapTooLarge); + } + if (objectTooDeep > 0) { + WafMetricCollector.get().wafInputTruncated(TruncatedType.OBJECT_TOO_DEEP, objectTooDeep); + } } if (ctx.isBlocked()) { diff --git a/internal-api/src/main/java/datadog/trace/api/telemetry/TruncatedType.java b/internal-api/src/main/java/datadog/trace/api/telemetry/TruncatedType.java index eb865f61827..a28f8b85bb1 100644 --- a/internal-api/src/main/java/datadog/trace/api/telemetry/TruncatedType.java +++ b/internal-api/src/main/java/datadog/trace/api/telemetry/TruncatedType.java @@ -6,7 +6,6 @@ public enum TruncatedType { OBJECT_TOO_DEEP(4); private final int value; - private static final int numValues = RuleType.values().length; TruncatedType(int value) { this.value = value; @@ -15,8 +14,4 @@ public enum TruncatedType { public int getValue() { return this.value; } - - public static int getNumValues() { - return numValues; - } } diff --git a/internal-api/src/main/java/datadog/trace/api/telemetry/WafMetricCollector.java b/internal-api/src/main/java/datadog/trace/api/telemetry/WafMetricCollector.java index c4b795a8d7d..76cabb35440 100644 --- a/internal-api/src/main/java/datadog/trace/api/telemetry/WafMetricCollector.java +++ b/internal-api/src/main/java/datadog/trace/api/telemetry/WafMetricCollector.java @@ -35,7 +35,7 @@ private WafMetricCollector() { private static final AtomicRequestCounter wafBlockedRequestCounter = new AtomicRequestCounter(); private static final AtomicRequestCounter wafTimeoutRequestCounter = new AtomicRequestCounter(); private static final AtomicLongArray wafInputTruncatedCounter = - new AtomicLongArray(TruncatedType.getNumValues()); + new AtomicLongArray(TruncatedType.values().length); private static final AtomicLongArray raspRuleEvalCounter = new AtomicLongArray(RuleType.getNumValues()); private static final AtomicLongArray raspRuleMatchCounter = diff --git a/internal-api/src/test/groovy/datadog/trace/api/telemetry/WafMetricCollectorTest.groovy b/internal-api/src/test/groovy/datadog/trace/api/telemetry/WafMetricCollectorTest.groovy index e6cc61ca829..44f6a7630f6 100644 --- a/internal-api/src/test/groovy/datadog/trace/api/telemetry/WafMetricCollectorTest.groovy +++ b/internal-api/src/test/groovy/datadog/trace/api/telemetry/WafMetricCollectorTest.groovy @@ -304,7 +304,7 @@ class WafMetricCollectorTest extends DDSpecification { } } - def "test WAF #inputTruncated metrics"() { + def "test WAF #truncatedType metrics"() { when: WafMetricCollector.get().wafInit('waf_ver1', 'rules.1', true) WafMetricCollector.get().wafInputTruncated(truncatedType, 5)