elastic · mrodm · Feb 4, 2025 · Feb 4, 2025 · Feb 4, 2025 · Feb 5, 2025
@@ -30,6 +30,8 @@ env:
   ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI: "${ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI:-false}"
   # Disable checking for newer versions
   ELASTIC_PACKAGE_CHECK_UPDATE_DISABLED: "true"
+  # Select method to validate fields are documented
+  ELASTIC_PACKAGE_FIELD_VALIDATION_TEST_METHOD: "mappings"
 
 steps:
   - label: "Get reference from target branch"

@@ -757,7 +757,7 @@ teardown_test_package() {
 }
 
 list_all_directories() {
-    find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {} | sort
+    find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {} | sort |grep -E '^(box_events|claroty_ctd|crowdstrike|github|mimecast|sublime_security|teleport|ti_anomali|ti_custom|tychon|wiz)$'
 }
 
 check_package() {

@@ -231,3 +231,5 @@ require (
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect
 	sigs.k8s.io/yaml v1.4.0 // indirect
 )
+
+replace github.com/elastic/elastic-package => github.com/mrodm/elastic-package v0.53.1-0.20250205174526-5b3f7cdba246
@@ -125,8 +125,6 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/elastic/elastic-integration-corpus-generator-tool v0.10.0 h1:sx1lpZuTG5suJuvgix4FWQFCLFFbzkoOmPoHWYOPLCY=
 github.com/elastic/elastic-integration-corpus-generator-tool v0.10.0/go.mod h1:2/30n+2QRzRzus4TPVUV1T3U/j8g2ItUgvP0pcpjLGk=
-github.com/elastic/elastic-package v0.109.1 h1:ATZVgYOCI6L5Yr0NxjSX+MsuK4UvXkpu9tDkO4K2vgo=
-github.com/elastic/elastic-package v0.109.1/go.mod h1:vmVYISfxBrl0ejjKbm/AG0drjrmevysVg2ZIP7yewLo=
 github.com/elastic/go-elasticsearch/v7 v7.17.10 h1:TCQ8i4PmIJuBunvBS6bwT2ybzVFxxUhhltAs3Gyu1yo=
 github.com/elastic/go-elasticsearch/v7 v7.17.10/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4=
 github.com/elastic/go-licenser v0.4.2 h1:bPbGm8bUd8rxzSswFOqvQh1dAkKGkgAmrPxbUi+Y9+A=
@@ -372,6 +370,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0=
 github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4=
+github.com/mrodm/elastic-package v0.53.1-0.20250205174526-5b3f7cdba246 h1:EjWls8TjHBNk5E/caFYxZR7DkURTeDevDazWZgO1T7A=
+github.com/mrodm/elastic-package v0.53.1-0.20250205174526-5b3f7cdba246/go.mod h1:vmVYISfxBrl0ejjKbm/AG0drjrmevysVg2ZIP7yewLo=
 github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s=
 github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8=
 github.com/muesli/termenv v0.15.2 h1:GohcuySI0QmI3wN8Ok9PtKGkgkFIk7y6Vpb5PvrY+Wo=

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.11.1"
+  changes:
+    - description: Add missing ECS mappings
+      type: bugfix
+      link: http://github.com/elastic/integrations/pull/12624
 - version: "2.11.0"
   changes:
     - description: Add "preserve_original_event" tag to documents with `event.kind` set to "pipeline_error".

@@ -0,0 +1,7 @@
+# This definition could be removed once Kibana constraint is updated
+# to 8.15.2 or higher. "ecs@mappings" component template would define
+# the correct dynamic template for it.
+- external: ecs
+  name: threat.enrichments.indicator.first_seen
+- external: ecs
+  name: threat.enrichments.indicator.last_seen
@@ -270,4 +270,6 @@ Preserves a raw copy of the original event, added to the field `event.original`.
 | related.description | Array of `description` derived from `threat[.enrichments].indicator.description` | keyword |
 | related.indicator_type | Array of `indicator_type` derived from `threat[.enrichments].indicator.type` | keyword |
 | related.location | Array of `location` derived from `related.ip` | geo_point |
+| threat.enrichments.indicator.first_seen | The date and time when intelligence source first reported sighting this indicator. | date |
+| threat.enrichments.indicator.last_seen | The date and time when intelligence source last reported sighting this indicator. | date |
 
@@ -1,7 +1,7 @@
 format_version: "3.0.3"
 name: box_events
 title: Box Events
-version: "2.11.0"
+version: "2.11.1"
 description: "Collect logs from Box with Elastic Agent"
 type: integration
 categories:

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.4.1"
+  changes:
+    - description: Add missing ECS mappings
+      type: bugfix
+      link: http://github.com/elastic/integrations/pull/12624
 - version: "0.4.0"
   changes:
     - description: Add "preserve_original_event" tag to documents with `event.kind` manually set to "pipeline_error".

@@ -0,0 +1,6 @@
+# This definition could be removed once Kibana constraint is updated
+# to 8.15.2 or higher. "ecs@mappings" component template would define
+# the correct dynamic template for it.
+- external: ecs
+  name: threat.indicator.modified_at
+
@@ -717,6 +717,7 @@ An example event for `event` looks as following:
 | log.offset | Log offset. | long |
 | log.source.address | Source address from which the log event read/sent. | keyword |
 | tags | User defined tags. | keyword |
+| threat.indicator.modified_at | The date and time when intelligence source last modified information for this indicator. | date |
 
 
 ### Assets

@@ -1,7 +1,7 @@
 format_version: 3.1.4
 name: claroty_ctd
 title: Claroty CTD
-version: 0.4.0
+version: 0.4.1
 description: Collect logs from Claroty CTD using Elastic Agent.
 type: integration
 categories:

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.50.1"
+  changes:
+    - description: Avoid using dynamic template for flattened fields
+      type: bugfix
+      link: http://github.com/elastic/integrations/pull/12624
 - version: "1.50.0"
   changes:
     - description: Allow the usage of deprecated log input and support for stack 9.0

@@ -27,7 +27,7 @@
       type: long
     - name: AsepWrittenCount
       type: long
-    - name: assessments.*
+    - name: assessments
       type: flattened
     - name: AssociatedFile
       type: keyword

@@ -1662,7 +1662,7 @@ and/or `session_token`.
 | crowdstrike.__mv_aip |  | keyword |
 | crowdstrike.__mv_discoverer_aid |  | keyword |
 | crowdstrike.aipCount |  | integer |
-| crowdstrike.assessments.\* |  | flattened |
+| crowdstrike.assessments |  | flattened |
 | crowdstrike.cid |  | keyword |
 | crowdstrike.discovererCount |  | integer |
 | crowdstrike.discoverer_aid |  | keyword |

@@ -1,6 +1,6 @@
 name: crowdstrike
 title: CrowdStrike
-version: "1.50.0"
+version: "1.50.1"
 description: Collect logs from Crowdstrike with Elastic Agent.
 type: integration
 format_version: "3.0.3"

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.3.1"
+  changes:
+    - description: Add missing ECS field in latest_code_scanning transform
+      type: bugfix
+      link: http://github.com/elastic/integrations/pull/12624
 - version: "2.3.0"
   changes:
     - description: Do not remove `event.original` in main ingest pipeline.

@@ -38,3 +38,5 @@
   name: rule.name
 - external: ecs
   name: tags
+- external: ecs
+  name: message
@@ -10,7 +10,7 @@ source:
 # that ability in order to prevent having duplicate data and prevent query
 # time field type conflicts.
 dest:
-  index: "logs-github_latest.dest_code_scanning-1"
+  index: "logs-github_latest.dest_code_scanning-2"
   aliases:
     - alias: "logs-github_latest.code_scanning"
       move_on_creation: true

@@ -1,6 +1,6 @@
 name: github
 title: GitHub
-version: "2.3.0"
+version: "2.3.1"
 description: Collect logs from GitHub with Elastic Agent.
 type: integration
 format_version: "3.0.2"

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.4.4"
+  changes:
+    - description: Add missing ECS field mappings
+      type: bugfix
+      link: http://github.com/elastic/integrations/pull/12624
 - version: "2.4.3"
   changes:
     - description: Fix rendering of CEL programs in configuration.

@@ -0,0 +1,8 @@
+# This definition could be removed once Kibana constraint is updated
+# to 8.15.2 or higher. "ecs@mappings" component template would define
+# the correct dynamic template for it.
+- external: ecs
+  name: threat.indicator.modified_at
+- external: ecs
+  name: threat.indicator.first_seen
+
@@ -0,0 +1,8 @@
+# This definition could be removed once Kibana constraint is updated
+# to 8.15.2 or higher. "ecs@mappings" component template would define
+# the correct dynamic template for it.
+- external: ecs
+  name: threat.indicator.modified_at
+- external: ecs
+  name: threat.indicator.first_seen
+
@@ -1018,6 +1018,8 @@ An example event for `threat_intel_malware_customer` looks as following:
 | mimecast.type | The indicator type, can for example be "domain, email, FileHash-SHA256". | keyword |
 | mimecast.valid_from | The valid from date. | date |
 | mimecast.value | The value of the indicator. | keyword |
+| threat.indicator.first_seen | The date and time when intelligence source first reported sighting this indicator. | date |
+| threat.indicator.modified_at | The date and time when intelligence source last modified information for this indicator. | date |
 
 
 ### Threat Intel Feed Malware: Grid
@@ -1134,6 +1136,8 @@ An example event for `threat_intel_malware_grid` looks as following:
 | mimecast.type | The indicator type, can for example be "domain, email, FileHash-SHA256". | keyword |
 | mimecast.valid_from | The valid from date. | date |
 | mimecast.value | The value of the indicator. | keyword |
+| threat.indicator.first_seen | The date and time when intelligence source first reported sighting this indicator. | date |
+| threat.indicator.modified_at | The date and time when intelligence source last modified information for this indicator. | date |
 
 
 ### TTP Attachment Logs

@@ -1,7 +1,7 @@
 format_version: "3.0.2"
 name: mimecast
 title: "Mimecast"
-version: "2.4.3"
+version: "2.4.4"
 description: Collect logs from Mimecast with Elastic Agent.
 type: integration
 categories: ["security", "email_security"]

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.5.1"
+  changes:
+    - description: Fix sublime_security.email_message.headers.hops.fields group mappings
+      type: bugfix
+      link: http://github.com/elastic/integrations/pull/12624
 - version: "1.5.0"
   changes:
     - description: Improve `file_selectors` documentation.

@@ -641,9 +641,22 @@
                     - name: type
                       type: keyword
                       description: The type of authentication result, derived from the field name.
+                # https://github.com/elastic/kibana/pull/204104
+                # Option 1: generate all keys as keywords under fields
+                # - name: fields
+                #   type: object
+                #   object_type: keyword
+                #   object_type_mapping_type: "*"
+                # Option 2: keep position as long
                 - name: fields
-                  type: object
-                  object_type: keyword
+                  type: group
+                  fields:
+                    - name: "*"
+                      type: object
+                      object_type: keyword
+                    - name: position
+                      # description: ?
+                      type: long
                 - name: index
                   type: long
                   description: Index indicates the order in which a hop occurred from sender to recipient.

@@ -1222,7 +1222,8 @@ An example event for `email_message` looks as following:
 | sublime_security.email_message.headers.hops.authentication_results.spf_details.server.valid | Whether the domain is valid. | boolean |
 | sublime_security.email_message.headers.hops.authentication_results.spf_details.verdict | Verdict of the SPF. | keyword |
 | sublime_security.email_message.headers.hops.authentication_results.type | The type of authentication result, derived from the field name. | keyword |
-| sublime_security.email_message.headers.hops.fields |  | object |
+| sublime_security.email_message.headers.hops.fields.\* |  | object |
+| sublime_security.email_message.headers.hops.fields.position |  | long |
 | sublime_security.email_message.headers.hops.index | Index indicates the order in which a hop occurred from sender to recipient. | long |
 | sublime_security.email_message.headers.hops.received.additional.raw | The raw string for remaining additional clauses, such as transport information. | keyword |
 | sublime_security.email_message.headers.hops.received.id.raw | The raw string of 'id' section. | keyword |

@@ -1,7 +1,7 @@
 format_version: 3.2.1
 name: sublime_security
 title: Sublime Security
-version: 1.5.0
+version: 1.5.1
 description: Collect logs from Sublime Security with Elastic Agent.
 type: integration
 categories:

@@ -872,14 +872,20 @@ processors:
       field: teleport.audit.aws_region
       target_field: cloud.region
       ignore_missing: true
+      # This was failing due to `cloud.region` already existed
+      override: true # Should it be added an if condition? Should it be added a remove processor?
   - rename:
       field: teleport.audit.aws_service
       target_field: cloud.service.name
       ignore_missing: true
+      # This was failing due to `cloud.service.name` already existed
+      override: true # Should it be added an if condition? Should it be added a remove processor?
   - rename:
       field: teleport.audit.aws_host
       target_field: cloud.instance.id
       ignore_missing: true
+      # This was failing due to `cloud.instance.id` already existed
+      override: true # Should it be added an if condition? Should it be added a remove processor?
   - rename:
       field: teleport.audit.aws_assumed_role
       target_field: teleport.audit.app.aws.assumed_role
@@ -968,6 +974,8 @@ processors:
       field: teleport.audit.db_gcp_instance_id
       target_field: cloud.instance.id
       ignore_missing: true
+      # This was failing due to `cloud.instance.id` already existed
+      override: true # Should it be added an if condition? Should it be added a remove processor?
   - rename:
       field: teleport.audit.db_roles
       target_field: teleport.audit.database.roles
@@ -1407,6 +1415,8 @@ processors:
       field: teleport.audit.instance_id
       target_field: cloud.instance.id
       ignore_missing: true
+      # This was failing due to `cloud.instance.id` already existed
+      override: true # Should it be added an if condition? Should it be added a remove processor?
   - rename:
       field: teleport.audit.exit_code
       target_field: process.exit_code
@@ -1426,11 +1436,17 @@ processors:
       field: teleport.audit.account_id
       target_field: cloud.account.id
       ignore_missing: true
+      # This was failing due to `cloud.account.id` already existed
+      override: true # Should it be added an if condition? Should it be added a remove processor?
   - rename:
       field: teleport.audit.region
       target_field: cloud.region
       ignore_missing: true
-      ignore_failure: true
+      ignore_failure: true # it could already exist this field
+  # in case it fails previous rename processor, remove the field (not defined in the package)
+  - remove:
+      field: teleport.audit.region
+      ignore_missing: true
   - rename:
       field: teleport.audit.stdout
       target_field: teleport.audit.database.aws.ssm_run.stdout

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.25.1"
+  changes:
+    - description: Add missing ECS field in intelligence datastream
+      type: bugfix
+      link: http://github.com/elastic/integrations/pull/12624
 - version: "1.25.0"
   changes:
     - description: Do not remove `event.original` in main ingest pipeline.

@@ -0,0 +1,6 @@
+# This definition could be removed once Kibana constraint is updated
+# to 8.15.2 or higher. "ecs@mappings" component template would define
+# the correct dynamic template for it.
+- external: ecs
+  name: threat.indicator.modified_at
+
@@ -180,6 +180,7 @@ An example event for `intelligence` looks as following:
 | labels.is_ioc_transform_source | Indicates whether an IOC is in the raw source data stream, or the in latest destination index. | constant_keyword |
 | threat.feed.dashboard_id | Dashboard ID used for Kibana CTI UI | constant_keyword |
 | threat.feed.name | Display friendly feed name | constant_keyword |
+| threat.indicator.modified_at | The date and time when intelligence source last modified information for this indicator. | date |
 
 
 ### Anomali ThreatStream via the Elastic Extension

@@ -1,6 +1,6 @@
 name: ti_anomali
 title: Anomali
-version: "1.25.0"
+version: "1.25.1"
 description: Ingest threat intelligence indicators from Anomali with Elastic Agent.
 type: integration
 format_version: 3.0.2

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.7.1"
+  changes:
+    - description: Add mapping for threat.indicator.url.original in transform
+      type: bugfix
+      link: http://github.com/elastic/integrations/pull/12624
 - version: "0.7.0"
   changes:
     - description: Add mapping for log file fingerprint.

@@ -42,6 +42,8 @@
   type: keyword
 - name: threat.indicator.url.full
   type: keyword
+- name: threat.indicator.url.original
+  type: wildcard
 # Below fields to be moved into base-fields.yml after kibana.version changed to >= 8.14 
 # Related to fix: https://github.com/elastic/kibana/pull/177608
 - name: event.module