Skip to content

Commit bf8ab75

Browse files
flexitrevflexitrev
authored
[Logstash] Adding Health Report to Logstash Integration (#12677)
* third time is the charm
1 parent 6bc0dc8 commit bf8ab75

File tree

17 files changed

+5530
-2209
lines changed

17 files changed

+5530
-2209
lines changed

‎packages/logstash/_dev/build/docs/README.md

Lines changed: 45 additions & 1177 deletions
Large diffs are not rendered by default.

‎packages/logstash/changelog.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
# newer versions go on top
22
- version: "2.5.0"
33
changes:
4-
- description: Allow the usage of deprecated log input and support for stack 9.0
4+
- description: Adding data and dashboards from Logstash Health Report
55
type: enhancement
6-
link: https://github.com/elastic/integrations/pull/12503
6+
link: https://github.com/elastic/integrations/pull/12464
77
- version: "2.4.12"
88
changes:
99
- description: Update documentation for GA of agent based monitoring

‎packages/logstash/data_stream/health_report/agent/stream/stream.yml.hbs

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
config_version: "1"
2+
interval: {{period}}
3+
resource.url: "{{url}}/_health_report"
4+
{{#if resource_ssl}}
5+
resource.ssl:
6+
{{resource_ssl}}
7+
{{/if}}
8+
9+
{{#if username}}
10+
auth.basic.user: {{escape_string username}}
11+
{{/if}}
12+
{{#if password}}
13+
auth.basic.password: {{escape_string password}}
14+
{{/if}}
15+
{{#if condition}}
16+
condition: {{ condition }}
17+
{{/if}}
18+
19+
redact:
20+
fields: ~
21+
22+
program: |
23+
get(state.url)
24+
.as(resp, bytes(resp.Body)
25+
.decode_json().as(body,
26+
[
27+
[{
28+
"logstash.node": {
29+
"name":body.name,
30+
"address":body.http_address,
31+
"uuid":body.id,
32+
"version":body.version,
33+
"status":body.status,
34+
"symptom":body.symptom,
35+
},
36+
}],
37+
body.indicators.pipelines.indicators.as(pipelines, // pipelines = body.indicators.pipelines.indicators
38+
pipelines.map(pipeline_name, pipelines[pipeline_name].as(pipeline, { // pipeline = pipelines[pipeline_name]
39+
"logstash": {
40+
"node": {
41+
"name": body.name,
42+
"version": body.version,
43+
"address": body.http_address,
44+
"uuid": body.id
45+
},
46+
"pipeline": {
47+
"id":pipeline_name,
48+
"status":pipeline.status,
49+
"symptom":pipeline.symptom,
50+
"state":pipeline.details.status.state,
51+
"flow": pipeline.details.flow,
52+
"diagnosis": has(pipeline.diagnosis) ? pipeline.diagnosis[0] : {},
53+
"impacts": has(pipeline.impacts) ? pipeline.impacts[0] : {},
54+
}
55+
}
56+
}
57+
))
58+
)
59+
].as(entries, {
60+
"events": entries.flatten()
61+
})
62+
)
63+
)

‎packages/logstash/data_stream/health_report/fields/base-fields.yml

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
- name: data_stream.type
2+
external: ecs
3+
- name: data_stream.dataset
4+
external: ecs
5+
- name: data_stream.namespace
6+
external: ecs
7+
- name: service.hostname
8+
type: keyword
9+
description: Hostname of the service

‎packages/logstash/data_stream/health_report/fields/ecs.yml

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
- name: '@timestamp'
2+
external: ecs
3+
- name: service.id
4+
external: ecs
5+
- name: service.type
6+
external: ecs
7+
- name: service.version
8+
external: ecs
9+
- name: service.address
10+
external: ecs
11+
- name: service.name
12+
external: ecs
13+
- name: process.pid
14+
external: ecs
15+
- name: ecs.version
16+
external: ecs
17+
- name: event.dataset
18+
external: ecs
19+
- name: event.duration
20+
external: ecs
21+
- name: event.module
22+
external: ecs
23+
- name: error.message
24+
external: ecs

‎packages/logstash/data_stream/health_report/fields/fields.yml

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
- name: logstash
2+
type: group
3+
fields:
4+
- name: node
5+
type: group
6+
fields:
7+
- name: name
8+
type: keyword
9+
- name: version
10+
type: keyword
11+
- name: address
12+
type: keyword
13+
- name: symptom
14+
type: keyword
15+
multi_fields:
16+
- name: text
17+
type: match_only_text
18+
- name: uuid
19+
type: keyword
20+
- name: status
21+
type: keyword
22+
- name: pipeline
23+
type: group
24+
fields:
25+
- name: id
26+
type: keyword
27+
- name: status
28+
type: keyword
29+
- name: state
30+
type: keyword
31+
- name: symptom
32+
type: keyword
33+
multi_fields:
34+
- name: text
35+
type: match_only_text
36+
- name: diagnosis
37+
type: group
38+
fields:
39+
- name: id
40+
type: keyword
41+
- name: cause
42+
type: keyword
43+
multi_fields:
44+
- name: text
45+
type: match_only_text
46+
- name: action
47+
type: keyword
48+
multi_fields:
49+
- name: text
50+
type: match_only_text
51+
- name: help_url
52+
type: keyword
53+
format: url
54+
multi_fields:
55+
- name: text
56+
type: match_only_text
57+
- name: impacts
58+
type: group
59+
fields:
60+
- name: id
61+
type: keyword
62+
- name: severity
63+
type: short
64+
- name: description
65+
type: keyword
66+
multi_fields:
67+
- name: text
68+
type: match_only_text
69+
- name: impact_areas
70+
type: keyword
71+
- name: flow
72+
type: group
73+
fields:
74+
- name: worker_utilization
75+
type: group
76+
fields:
77+
- name: current
78+
type: float
79+
- name: last_1_hour
80+
type: float
81+
- name: last_5_minutes
82+
type: float
83+
- name: last_15_minutes
84+
type: float
85+
- name: lifetime
86+
type: float
87+
- name: last_1_minute
88+
type: float
89+
- name: last_24_hours
90+
type: float

‎packages/logstash/data_stream/health_report/manifest.yml

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
title: "Health Report"
2+
type: metrics
3+
dataset: logstash.health_report
4+
release: beta
5+
elasticsearch:
6+
index_template:
7+
mappings:
8+
dynamic: false
9+
streams:
10+
- input: cel
11+
title: "Collect Health Report"
12+
description: "Health Report for Logstash instance"
13+
vars:
14+
- name: period
15+
type: text
16+
title: Period
17+
multi: false
18+
required: true
19+
show_user: true
20+
default: 30s

‎packages/logstash/data_stream/health_report/sample_event.json

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
{
2+
"logstash": {
3+
"pipeline": {
4+
"symptom": "The pipeline is unhealthy; 1 area is impacted and 1 diagnosis is available",
5+
"diagnosis": {
6+
"help_url": "https://www.elastic.co/guide/en/logstash/8.17/health-report-pipeline-status.html#terminated",
7+
"action": "view logs to determine the cause of abnormal pipeline shutdown",
8+
"cause": "pipeline is not running, likely because it has encountered an error",
9+
"id": "logstash:health:pipeline:status:diagnosis:terminated"
10+
},
11+
"state": "TERMINATED",
12+
"impacts": {
13+
"severity": 1,
14+
"impact_areas": [
15+
"pipeline_execution"
16+
],
17+
"description": "the pipeline is not currently processing",
18+
"id": "logstash:health:pipeline:status:impact:not_processing"
19+
},
20+
"id": "bad",
21+
"flow": {
22+
"worker_utilization": {
23+
"current": 0.000001826,
24+
"last_24_hours": 0.000001826,
25+
"last_5_minutes": 0.000001826,
26+
"last_1_hour": 0.000001826,
27+
"last_15_minutes": 0.000001826,
28+
"lifetime": 0.000001826,
29+
"last_1_minute": 0.000001826
30+
}
31+
},
32+
"status": "red"
33+
},
34+
"node": {
35+
"address": "0.0.0.0:9600",
36+
"name": "9cbebe0b66d9",
37+
"version": "8.17.1",
38+
"uuid": "1a3d4358-0d0a-4b82-8cf7-21f59ea592e8"
39+
}
40+
}
41+
}

‎packages/logstash/data_stream/node/manifest.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ elasticsearch:
88
streams:
99
- input: logstash/metrics
1010
title: Logstash node metrics
11+
enabled: false
1112
description: Collect Logstash node metrics
1213
vars:
1314
- name: period

‎packages/logstash/data_stream/node_cel/manifest.yml

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@ elasticsearch:
77
dynamic: false
88
streams:
99
- input: cel
10-
enabled: false
1110
title: "Collect Node Metrics"
1211
description: "Collect Metrics related to Nodes running Logstash"
1312
template_path: cel.yml.hbs

‎packages/logstash/data_stream/node_stats/manifest.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ elasticsearch:
88
streams:
99
- input: logstash/metrics
1010
title: Logstash node stats metrics
11+
enabled: false
1112
description: Collect Logstash node stats metrics
1213
vars:
1314
- name: period

‎packages/logstash/data_stream/pipeline/manifest.yml

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,12 @@
11
type: metrics
22
title: Logstash pipeline
3-
release: experimental
43
elasticsearch:
54
index_mode: "time_series"
65
index_template:
76
mappings:
87
dynamic: false
98
streams:
109
- input: cel
11-
enabled: false
1210
title: "Collect Pipeline Metrics"
1311
description: "Collect Metrics related to Logstash Pipeline usage"
1412
template_path: cel.yml.hbs

‎packages/logstash/data_stream/plugins/manifest.yml

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,12 @@
11
type: metrics
22
title: Logstash plugins
3-
release: experimental
43
elasticsearch:
54
index_mode: "time_series"
65
index_template:
76
mappings:
87
dynamic: false
98
streams:
109
- input: cel
11-
enabled: false
1210
title: "Collect Plugin Metrics"
1311
description: "Collect metrics for Logstash plugin use. Note that large pipelines will increase the volume of plugin metrics, and a slower rate of collection may be appropriate"
1412
template_path: cel.yml.hbs

0 commit comments

Comments
 (0)