From 54f26c835ca5875710a59fff60d959c13f09e93c Mon Sep 17 00:00:00 2001 From: Mika Ayenson Date: Wed, 29 Apr 2026 11:33:37 -0500 Subject: [PATCH 1/3] add new unit test for process fields in non process events --- pyproject.toml | 2 +- tests/test_all_rules.py | 96 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8923f91a056..6f15cb4ac7a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "detection_rules" -version = "1.6.29" +version = "1.6.30" description = "Detection Rules is the home for rules used by Elastic Security. This repository is used for the development, maintenance, testing, validation, and release of rules for Elastic Security’s Detection Engine." readme = "README.md" requires-python = ">=3.12" diff --git a/tests/test_all_rules.py b/tests/test_all_rules.py index d89d3411390..e954881b74f 100644 --- a/tests/test_all_rules.py +++ b/tests/test_all_rules.py @@ -11,6 +11,7 @@ import uuid from collections import defaultdict from pathlib import Path +from typing import ClassVar import eql import kql @@ -1572,3 +1573,98 @@ def test_eql_non_sequence_support_only(self): # is_sequence method not yet available during schema validation # so we have to check in a unit test self.fail(f"{self.rule_str(rule)} Sequence rules cannot have alert suppression") + + +class TestEQLEventFieldUsage(BaseRuleTest): + """Test that EQL rules reference fields mapped in the endpoint integration schema.""" + + SAFE_EVENT_TYPES: ClassVar[set[str]] = {"process", "any"} + ALERT_INDEX_HINTS: ClassVar[tuple[str, ...]] = ( + ".alerts-security.", + "logs-endpoint.alerts", + "-alerts-", + ) + EVENT_TYPE_TO_DATASET: ClassVar[dict[str, str]] = { + "file": "file", + "network": "network", + "registry": "registry", + "library": "library", + "dns": "network", + } + + @classmethod + def _bad_fields_for_event_query( + cls, + event_query: eql.ast.EventQuery, + schema_by_dataset: dict[str, dict[str, str]], + ) -> tuple[str, list[str]] | None: + if event_query.event_type in cls.SAFE_EVENT_TYPES: + return None + dataset = cls.EVENT_TYPE_TO_DATASET.get(event_query.event_type) + if not dataset or dataset not in schema_by_dataset: + return None + schema_fields = schema_by_dataset[dataset] + bad_fields = sorted( + { + str(node) + for node in event_query + if isinstance(node, eql.ast.Field) + and str(node).startswith("process.") + and str(node) not in schema_fields + } + ) + return (dataset, bad_fields) if bad_fields else None + + def test_process_fields_present_in_endpoint_schema(self): + """Ensure process.* fields used in non-process EQL clauses exist in the endpoint integration schema.""" + load_integrations_schemas.clear() + schemas = load_integrations_schemas() + endpoint_versions = schemas.get("endpoint", {}) + if not endpoint_versions: + self.skipTest("endpoint integration schema not available") + latest = sorted(endpoint_versions, key=Version.parse)[-1] + schema_by_dataset = endpoint_versions[latest] + + offenders: list[str] = [] + for rule in self.all_rules: + data = rule.contents.data + if not isinstance(data, EQLRuleData): + continue + + indices = data.get("index") or [] + if not any("logs-endpoint.events." in idx for idx in indices): + continue + if any(any(hint in idx for hint in self.ALERT_INDEX_HINTS) for idx in indices): + continue + + try: + rule_ast = data.ast + except Exception: # noqa: BLE001, S112 + continue + if rule_ast is None: + continue + + first = rule_ast.first + event_queries = [sq.query for sq in first.queries] if isinstance(first, eql.ast.Sequence) else [first] + + for event_query in event_queries: + if not isinstance(event_query, eql.ast.EventQuery): + continue + result = self._bad_fields_for_event_query(event_query, schema_by_dataset) + if result is None: + continue + dataset, bad_fields = result + offenders.append( + f"{self.rule_str(rule, trailer=None)} references {bad_fields} " + f"inside `{event_query.event_type} where` but the endpoint integration " + f"schema (dataset `{dataset}`) does not include these fields" + ) + break + + if offenders: + self.fail( + "process.* fields below are not in the endpoint integration schema for the " + "targeted dataset, so the predicates referencing them never match. Move the " + "check into a `process where` clause (e.g. via an EQL sequence) or use a field " + "that the dataset actually populates.\nOffenders:\n - " + "\n - ".join(offenders) + ) From 786677dc084377856f073d9ea13c26a63f4ae14b Mon Sep 17 00:00:00 2001 From: Mika Ayenson Date: Fri, 1 May 2026 18:05:39 -0500 Subject: [PATCH 2/3] Address review: cover EQL Join nodes in event-query walk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per @eric-forte-elastic's suggestion, extend the Sequence-only check to also unwrap eql.ast.Join. Join.queries shares the same SubqueryBy shape as Sequence.queries, so the same .query iteration applies. No production EQL rule uses join today, but treating join symmetrically prevents a silent gap if one is added in the future — matching the test's intent of catching no-op predicates in non-process clauses. --- tests/test_all_rules.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_all_rules.py b/tests/test_all_rules.py index e954881b74f..0dd2723faa5 100644 --- a/tests/test_all_rules.py +++ b/tests/test_all_rules.py @@ -1645,7 +1645,9 @@ def test_process_fields_present_in_endpoint_schema(self): continue first = rule_ast.first - event_queries = [sq.query for sq in first.queries] if isinstance(first, eql.ast.Sequence) else [first] + event_queries = ( + [sq.query for sq in first.queries] if isinstance(first, (eql.ast.Sequence, eql.ast.Join)) else [first] + ) for event_query in event_queries: if not isinstance(event_query, eql.ast.EventQuery): From 23e4fd99be7ad98c35a804b6a729c499872b0257 Mon Sep 17 00:00:00 2001 From: Mika Ayenson Date: Mon, 4 May 2026 15:28:29 -0500 Subject: [PATCH 3/3] bump version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 6f15cb4ac7a..60d9411e47d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "detection_rules" -version = "1.6.30" +version = "1.6.31" description = "Detection Rules is the home for rules used by Elastic Security. This repository is used for the development, maintenance, testing, validation, and release of rules for Elastic Security’s Detection Engine." readme = "README.md" requires-python = ">=3.12"