Adding health report to logstash integration

packages/logstash/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.5.0"
+  changes:
+    - description: Adding data and dashboards from Logstash Health Report
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/12464
 - version: "2.4.12"
   changes:
     - description: Update documentation for GA of agent based monitoring

packages/logstash/data_stream/health_report/agent/stream/stream.yml.hbs

@@ -0,0 +1,63 @@
+config_version: "1"
+interval: {{period}}
+resource.url: "{{url}}/_health_report"
+{{#if resource_ssl}}
+resource.ssl:
+  {{resource_ssl}}
+{{/if}}
+
+{{#if username}}
+auth.basic.user: {{escape_string username}}
+{{/if}}
+{{#if password}}
+auth.basic.password: {{escape_string password}}
+{{/if}}
+{{#if condition}}
+condition: {{ condition }}
+{{/if}}
+
+redact: 
+  fields: ~
+
+program: |
+  get(state.url)
+  .as(resp, bytes(resp.Body)
+  .decode_json().as(body,
+    [   
+        [{
+            "logstash.node": {
+            "name":body.name,
+            "address":body.http_address,
+            "uuid":body.id,
+            "version":body.version,
+            "status":body.status,
+            "symptom":body.symptom,
+            },
+        }],
+        body.indicators.pipelines.indicators.as(pipelines, // pipelines = body.indicators.pipelines.indicators
+              pipelines.map(pipeline_name, pipelines[pipeline_name].as(pipeline, { // pipeline = pipelines[pipeline_name]
+                    "logstash": {
+                        "node": {
+                            "name": body.name,
+                            "version": body.version,
+                            "address": body.http_address,
+                            "uuid": body.id
+                        },
+                        "pipeline": {
+                            "id":pipeline_name,
+                            "status":pipeline.status,
+                            "symptom":pipeline.symptom,
+                            "state":pipeline.details.status.state,
+                            "flow": pipeline.details.flow,
+                            "diagnosis": has(pipeline.diagnosis) ? pipeline.diagnosis[0] : {},
+                            "impacts": has(pipeline.impacts) ? pipeline.impacts[0] : {},
+                        }
+                    }
+                }
+            ))
+        )
+    ].as(entries, {
+        "events": entries.flatten()
+    })
+    )
+  )

packages/logstash/data_stream/health_report/fields/base-fields.yml

@@ -0,0 +1,9 @@
+- name: data_stream.type
+  external: ecs
+- name: data_stream.dataset
+  external: ecs
+- name: data_stream.namespace
+  external: ecs
+- name: service.hostname
+  type: keyword
+  description: Hostname of the service

packages/logstash/data_stream/health_report/fields/ecs.yml

@@ -0,0 +1,24 @@
+- name: '@timestamp'
+  external: ecs
+- name: service.id
+  external: ecs
+- name: service.type
+  external: ecs
+- name: service.version
+  external: ecs
+- name: service.address
+  external: ecs
+- name: service.name
+  external: ecs
+- name: process.pid
+  external: ecs
+- name: ecs.version
+  external: ecs
+- name: event.dataset
+  external: ecs
+- name: event.duration
+  external: ecs
+- name: event.module
+  external: ecs
+- name: error.message
+  external: ecs

packages/logstash/data_stream/health_report/fields/fields.yml

@@ -0,0 +1,90 @@
+- name: logstash
+  type: group
+  fields:
+  - name: node
+    type: group
+    fields:
+      - name: name
+        type: keyword
+      - name: version
+        type: keyword
+      - name: address
+        type: keyword
+      - name: symptom
+        type: keyword
+        multi_fields:
+        - name: text
+          type: match_only_text
+      - name: uuid
+        type: keyword
+      - name: status
+        type: keyword
+  - name: pipeline
+    type: group
+    fields:
+      - name: id
+        type: keyword
+      - name: status
+        type: keyword
+      - name: state
+        type: keyword
+      - name: symptom
+        type: keyword
+        multi_fields:
+        - name: text
+          type: match_only_text
+      - name: diagnosis
+        type: group
+        fields:
+          - name: id
+            type: keyword
+          - name: cause
+            type: keyword
+            multi_fields:
+            - name: text
+              type: match_only_text
+          - name: action
+            type: keyword
+            multi_fields:
+            - name: text
+              type: match_only_text
+          - name: help_url
+            type: keyword
+            format: url
+            multi_fields:
+            - name: text
+              type: match_only_text
+      - name: impacts
+        type: group
+        fields:
+          - name: id
+            type: keyword
+          - name: severity
+            type: short
+          - name: description
+            type: keyword
+            multi_fields:
+            - name: text
+              type: match_only_text
+          - name: impact_areas
+            type: keyword
+      - name: flow
+        type: group
+        fields:
+          - name: worker_utilization
+            type: group
+            fields:
+              - name: current
+                type: float
+              - name: last_1_hour
+                type: float
+              - name: last_5_minutes
+                type: float
+              - name: last_15_minutes
+                type: float
+              - name: lifetime
+                type: float
+              - name: last_1_minute
+                type: float
+              - name: last_24_hours
+                type: float

packages/logstash/data_stream/health_report/manifest.yml

@@ -0,0 +1,21 @@
+title: "Health Report"
+type: metrics
+dataset: logstash.health_report
+release: beta
+elasticsearch:
+  index_template:
+    mappings:
+      dynamic: false
+streams:
+  - input: cel
+    enabled: false
+    title: "Collect Health Report"
+    description: "Health Report for Logstash instance"
+    vars:
+      - name: period
+        type: text
+        title: Period
+        multi: false
+        required: true
+        show_user: true
+        default: 30s

packages/logstash/data_stream/health_report/sample_event.json

@@ -0,0 +1,88 @@
+{
+"logstash": {
+  "pipeline": {
+    "symptom": "The pipeline is concerning; 1 area is impacted and 1 diagnosis is available",
+    "diagnosis": {
+      "help_url": "https://www.elastic.co/guide/en/logstash/8.17/health-report-pipeline-status.html#finished",
+      "cause": "pipeline has finished running because its inputs have been closed and events have been processed",
+      "action": "if you expect this pipeline to run indefinitely, you will need to configure its inputs to continue receiving or fetching events",
+      "id": "logstash:health:pipeline:status:diagnosis:finished"
+    },
+    "id": "self-closer",
+    "state": "FINISHED",
+    "impacts": {
+      "severity": 10,
+      "impact_areas": [
+        "pipeline_execution"
+      ],
+      "description": "pipeline has finished running",
+      "id": "logstash:health:pipeline:status:impact:not_processing"
+    },
+    "flow": {
+      "worker_utilization": {
+        "current": 0.0009642,
+        "last_24_hours": 0.0009642,
+        "last_5_minutes": 0.0009642,
+        "last_1_hour": 0.0009642,
+        "last_15_minutes": 0.0009642,
+        "lifetime": 0.0009642,
+        "last_1_minute": 0.0009642
+      }
+    },
+    "status": "yellow"
+  },
+  "node": {
+    "address": "0.0.0.0:9600",
+    "name": "87f8aa570fcb",
+    "uuid": "8c2afc7e-a64f-42f3-9ab9-5e16dc95c9bc",
+    "version": "8.17.1"
+  }
+},
+"agent": {
+  "name": "a9f1b9c5936b",
+  "id": "af72217c-8c4d-427f-8c92-6b4566e9937f",
+  "type": "filebeat",
+  "ephemeral_id": "40ea6231-3856-4b90-8083-73b30558cfe7",
+  "version": "8.17.1"
+},
+"@timestamp": "2025-01-28T18:41:24.669Z",
+"ecs": {
+  "version": "8.0.0"
+},
+"data_stream": {
+  "namespace": "default",
+  "type": "metrics",
+  "dataset": "logstash.health_report"
+},
+"host": {
+  "hostname": "a9f1b9c5936b",
+  "os": {
+    "kernel": "6.10.14-linuxkit",
+    "codename": "focal",
+    "name": "Ubuntu",
+    "family": "debian",
+    "type": "linux",
+    "version": "20.04.6 LTS (Focal Fossa)",
+    "platform": "ubuntu"
+  },
+  "containerized": false,
+  "ip": [
+    "172.17.0.3"
+  ],
+  "name": "a9f1b9c5936b",
+  "mac": [
+    "02-42-AC-11-00-03"
+  ],
+  "architecture": "aarch64"
+},
+"elastic_agent": {
+  "id": "af72217c-8c4d-427f-8c92-6b4566e9937f",
+  "version": "8.17.1",
+  "snapshot": false
+},
+"event": {
+  "agent_id_status": "verified",
+  "ingested": "2025-01-28T18:41:28Z",
+  "dataset": "logstash.health_report"
+}
+}

packages/logstash/data_stream/node/manifest.yml

@@ -8,6 +8,7 @@ elasticsearch:
 streams:
   - input: logstash/metrics
     title: Logstash node metrics
+    enabled: false
     description: Collect Logstash node metrics
     vars:
       - name: period

packages/logstash/data_stream/node_cel/manifest.yml

@@ -7,7 +7,6 @@ elasticsearch:
       dynamic: false
 streams:
   - input: cel
-    enabled: false
     title: "Collect Node Metrics"
     description: "Collect Metrics related to Nodes running Logstash"
     template_path: cel.yml.hbs

packages/logstash/data_stream/node_stats/manifest.yml

@@ -8,6 +8,7 @@ elasticsearch:
 streams:
   - input: logstash/metrics
     title: Logstash node stats metrics
+    enabled: false
     description: Collect Logstash node stats metrics
     vars:
       - name: period

packages/logstash/data_stream/pipeline/manifest.yml

@@ -1,14 +1,12 @@
 type: metrics
 title: Logstash pipeline
-release: experimental
 elasticsearch:
   index_mode: "time_series"
   index_template:
     mappings:
       dynamic: false
 streams:
   - input: cel
-    enabled: false
     title: "Collect Pipeline Metrics"
     description: "Collect Metrics related to Logstash Pipeline usage"
     template_path: cel.yml.hbs

packages/logstash/data_stream/plugins/manifest.yml

@@ -1,14 +1,12 @@
 type: metrics
 title: Logstash plugins
-release: experimental
 elasticsearch:
   index_mode: "time_series"
   index_template:
     mappings:
       dynamic: false
 streams:
   - input: cel
-    enabled: false
     title: "Collect Plugin Metrics"
     description: "Collect metrics for Logstash plugin use. Note that large pipelines will increase the volume of plugin metrics, and a slower rate of collection may be appropriate"
     template_path: cel.yml.hbs