Skip to content

Commit e2bb84d

Browse files
smoorssmoors
authored
Merge pull request #18 from wpoely86/pixiu_alarm
Add pattens for pixiu alerts of space/files quota
2 parents fc64377 + 804b320 commit e2bb84d

File tree

5 files changed

+157
-11
lines changed

5 files changed

+157
-11
lines changed

‎.github/workflows/testpatterns.yml

+6-9
Original file line numberDiff line numberDiff line change
@@ -4,15 +4,12 @@ on: [push, pull_request]
44
jobs:
55
test_patterns:
66
runs-on: ubuntu-latest
7-
strategy:
8-
matrix:
9-
python-version: [2.7, 3.6]
107
steps:
118
- name: add logstash repo
129
uses: myci-actions/add-deb-repo@4
1310
with:
14-
repo: deb https://artifacts.elastic.co/packages/7.x/apt stable main
15-
repo-name: elastic-7.x
11+
repo: deb https://artifacts.elastic.co/packages/8.x/apt stable main
12+
repo-name: elastic-8.x
1613
keys: D27D666CD88E42B4
1714
- name: install logstash
1815
run: sudo apt-get install logstash
@@ -21,11 +18,11 @@ jobs:
2118
- name: fix directory permissions of logstash data directory
2219
run: sudo chmod 777 /usr/share/logstash/data
2320
- name: Checkout code
24-
uses: actions/checkout@v2
25-
- name: Set up Python ${{ matrix.python-version }}
26-
uses: actions/setup-python@v2
21+
uses: actions/checkout@v3
22+
- name: Set up Python
23+
uses: actions/setup-python@v4
2724
with:
28-
python-version: ${{ matrix.python-version }}
25+
python-version: '3.9'
2926
- name: Install dependencies
3027
run: |
3128
python -m pip install --upgrade pip

‎files/pixiu

+16
Original file line numberDiff line numberDiff line change
@@ -5,3 +5,19 @@ PIXIU_APACHE \[%{HTTPDATE:timestamp}\] %{IPORHOST:clientip} %{HTTPDUSER:auth} %{
55
PIXIU_NGINX \s*%{IPORHOST:syslog_hostname} %{SYSLOGPROG} %{IPORHOST:clientip} - %{HTTPDUSER:auth}\s+\[%{HTTPDATE:timestamp}\] \"%{WORD:verb} %{NOTSPACE:request} HTTP/%{NUMBER:httpversion}\" %{NUMBER:response:int} %{GREEDYDATA:other}
66

77
PIXIU_TOMCAT \s*%{IPORHOST:syslog_hostname} %{SYSLOGPROG} \[%{HTTPDATE:timestamp}\]\^%{IPORHOST:clientip}\^%{HTTPDUSER:auth}\^%{WORD:verb} %{NOTSPACE:request} HTTP/%{NUMBER:httpversion}\^%{GREEDYDATA:other}
8+
9+
PIXIU_ALARM_PREFIX \s*%{IPORHOST:syslog_hostname} %{SYSLOGPROG}: .+%{IPORHOST} (?<alarm_object>OceanStor-Distributed-Storage)\s+%{NUMBER} (?<alarm_id>0x[A-F0-9]+) (?<alarm_severity>Major).*:
10+
11+
PIXIU_ALARM_USER_TYPE user name, owner name %{USERNAME:username}
12+
13+
PIXIU_BYTES [0-9]+(?:K|M|G|T|P)?B
14+
15+
PIXIU_QUOTA_SIZE_TYPE (?:(?<used_files>%{NUMBER:int})|(?<used_space>%{PIXIU_BYTES}))
16+
17+
PIXIU_ALARM_COMMON The used (?<quota_used_type>%{WORD}) \(%{PIXIU_QUOTA_SIZE_TYPE}\) of (?:the )?quota \(type (?<quota_type>%{WORD}) quota(?:, %{PIXIU_ALARM_USER_TYPE})?\) of dtree \(name (?<dtree>%{WORD}), ID (?<dtree_id>%{NUMBER:int})\) in namespace \(name (?<namespace>%{WORD}), ID (?<namespace_id>%{NUMBER:int})\)
18+
19+
PIXIU_USER_INODE_ALARM %{PIXIU_ALARM_PREFIX} %{PIXIU_ALARM_COMMON} reaches (?:or approaches )?to the (:?(?<quota_limit_type>%{WORD}) (?:file|space) quantity quota of|(?:file|space) quantity (?<quota_limit_type>%{WORD}) quota of) \((?<quota_files_limit>%{NUMBER:int})\).
20+
21+
PIXIU_USER_SPACE_ALARM %{PIXIU_ALARM_PREFIX} %{PIXIU_ALARM_COMMON} reaches (?:or approaches )?to the (:?(?<quota_limit_type>%{WORD}) space(?: quantity)? quota of|space(?: quantity)? (?<quota_limit_type>%{WORD}) quota of) \((?<quota_space_limit>%{PIXIU_BYTES})\).
22+
23+
PIXIU_ALARM %{PIXIU_USER_INODE_ALARM}|%{PIXIU_USER_SPACE_ALARM}

‎setup.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@
3232
import glob
3333

3434
setup(name="logstash-patterns",
35-
version="1.2.9",
35+
version="1.2.10",
3636
description="Grok patterns for logstash",
3737
long_description="""Grok patterns for parsing log messages with logstash.
3838

‎tests/data/pixiu

+125
Original file line numberDiff line numberDiff line change
@@ -49,4 +49,129 @@ data = [
4949
"request": "/dsware/service/noAuth/managerStatus",
5050
},
5151
},
52+
{
53+
"raw": "<166>Oct 21 11:32:53 C4STO01-Node2 libstorage-iostat-enable[1965645]: /home/abuild/rpmbuild/BUILD/ublock-1.0/src/ublock_rpc.c:1420:ublock_get_sockaddr_shm: *ERROR*: [ublock] shm_open failed: Permission denied",
54+
"expected": {
55+
"@source_host": "C4STO01-Node2",
56+
"program": "libstorage-iostat-enable",
57+
"pid": "1965645",
58+
},
59+
},
60+
{
61+
"raw": "<187>Jul 19 14:22:39 C4STO01-Node1 alarm[2613490]: <186>2023-07-19 14:11:57 DST 2023-07-19 14:22:37 DST 172.19.104.10 OceanStor-Distributed-Storage 216045 0xFEA6A000F Major(2): The used files (39293) of quota (type user quota, user name, owner name vsc10800) of dtree (name 108, ID 4137) in namespace (name data, ID 69) reaches to the file quantity soft quota of (38095).",
62+
"expected": {
63+
"@source_host": "C4STO01-Node1",
64+
"program": "alarm",
65+
"alarm_severity": "Major",
66+
"alarm_object": "OceanStor-Distributed-Storage",
67+
"alarm_id": "0xFEA6A000F",
68+
"used_files": 39293,
69+
"username": "vsc10800",
70+
"quota_type": "user",
71+
"quota_used_type": "files",
72+
"dtree": "108",
73+
"dtree_id": 4137,
74+
"namespace": "data",
75+
"namespace_id": 69,
76+
"quota_limit_type": "soft",
77+
"quota_files_limit": 38095,
78+
},
79+
},
80+
{
81+
"raw": "<187>Jul 19 14:22:39 C4STO01-Node1 alarm[2613490]: <186>2023-07-19 14:06:48 DST 2023-07-19 14:22:37 DST 172.19.104.10 OceanStor-Distributed-Storage 216042 0xFEA6A0011 Major(2): The used files (40000) of the quota (type user quota, user name, owner name vsc10800) of dtree (name 108, ID 4137) in namespace (name data, ID 69) reaches or approaches to the hard file quantity quota of (40000). The user or user group will fail to write data.",
82+
"expected": {
83+
"@source_host": "C4STO01-Node1",
84+
"program": "alarm",
85+
"alarm_severity": "Major",
86+
"alarm_object": "OceanStor-Distributed-Storage",
87+
"alarm_id": "0xFEA6A0011",
88+
"used_files": 40000,
89+
"username": "vsc10800",
90+
"quota_type": "user",
91+
"quota_used_type": "files",
92+
"dtree": "108",
93+
"dtree_id": 4137,
94+
"namespace": "data",
95+
"namespace_id": 69,
96+
"quota_limit_type": "hard",
97+
"quota_files_limit": 40000,
98+
},
99+
},
100+
{
101+
"raw": "<187>Aug 20 20:18:25 C4STO01-Node1 alarm[2613490]: <186>2023-08-20 20:18:25 DST 172.19.104.10 OceanStor-Distributed-Storage 259068 0xFEA6A000E Major(1): The used files (1501) of quota (type directory quota) of dtree (name test1, ID 4101) in namespace (name admin, ID 71) reaches to the file quantity soft quota of (1000).",
102+
"expected": {
103+
"@source_host": "C4STO01-Node1",
104+
"program": "alarm",
105+
"alarm_severity": "Major",
106+
"alarm_object": "OceanStor-Distributed-Storage",
107+
"alarm_id": "0xFEA6A000E",
108+
"used_files": 1501,
109+
"quota_used_type": "files",
110+
"quota_type": "directory",
111+
"dtree": "test1",
112+
"dtree_id": 4101,
113+
"namespace": "admin",
114+
"namespace_id": 71,
115+
"quota_limit_type": "soft",
116+
"quota_files_limit": 1000,
117+
},
118+
},
119+
{
120+
"raw": "<187>Aug 20 20:16:52 C4STO01-Node1 alarm[2613490]: <186>2023-08-20 20:16:50 DST 172.19.104.10 OceanStor-Distributed-Storage 259067 0xFEA6A000B Major(1): The used files (2000) of quota (type directory quota) of dtree (name test1, ID 4101) in namespace (name admin, ID 71) reaches or approaches to the file quantity hard quota of (2000). Data will fail to be written.",
121+
"expected": {
122+
"@source_host": "C4STO01-Node1",
123+
"program": "alarm",
124+
"alarm_severity": "Major",
125+
"alarm_object": "OceanStor-Distributed-Storage",
126+
"alarm_id": "0xFEA6A000B",
127+
"used_files": 2000,
128+
"quota_used_type": "files",
129+
"quota_type": "directory",
130+
"dtree": "test1",
131+
"dtree_id": 4101,
132+
"namespace": "admin",
133+
"namespace_id": 71,
134+
"quota_limit_type": "hard",
135+
"quota_files_limit": 2000,
136+
},
137+
},
138+
{
139+
"raw": "<187>Aug 21 09:56:38 C4STO01-Node1 alarm[2613490]: <186>2023-08-21 09:56:37 DST 172.19.104.10 OceanStor-Distributed-Storage 260204 0xFEA6A000B Major(1): The used space (1024MB) of quota (type directory quota) of dtree (name test1, ID 4101) in namespace (name admin, ID 71) reaches or approaches to the space hard quota of (1024MB). Data will fail to be written.",
140+
"expected": {
141+
"@source_host": "C4STO01-Node1",
142+
"program": "alarm",
143+
"alarm_severity": "Major",
144+
"alarm_object": "OceanStor-Distributed-Storage",
145+
"alarm_id": "0xFEA6A000B",
146+
"used_space": 1024000000,
147+
"quota_used_type": "space",
148+
"quota_type": "directory",
149+
"dtree": "test1",
150+
"dtree_id": 4101,
151+
"namespace": "admin",
152+
"namespace_id": 71,
153+
"quota_limit_type": "hard",
154+
"quota_space_limit": 1024000000,
155+
},
156+
},
157+
{
158+
"raw": "<187>Aug 20 19:40:26 C4STO01-Node1 alarm[2613490]: <186>2023-08-20 19:32:24 DST 2023-08-20 19:40:26 DST 172.19.104.10 OceanStor-Distributed-Storage 259045 0xFEA6A000F Major(2): The used space (6638MB) of quota (type user quota, user name, owner name vsc10042) of dtree (name 100, ID 8193) in namespace (name user, ID 70) reaches to the space soft quota of (6144MB).",
159+
"expected": {
160+
"@source_host": "C4STO01-Node1",
161+
"program": "alarm",
162+
"alarm_severity": "Major",
163+
"alarm_object": "OceanStor-Distributed-Storage",
164+
"alarm_id": "0xFEA6A000F",
165+
"used_space": 6638000000,
166+
"quota_used_type": "space",
167+
"quota_type": "user",
168+
"username": "vsc10042",
169+
"dtree": "100",
170+
"dtree_id": 8193,
171+
"namespace": "user",
172+
"namespace_id": 70,
173+
"quota_limit_type": "soft",
174+
"quota_space_limit": 6144000000,
175+
},
176+
},
52177
]

‎tests/logstash_7.6.2.conf

+9-1
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ filter {
1616

1717
match => {
1818
# RSYSLOGCUSTOM always last (and no PREFIX)!
19-
"message" => ["%{RSYSLOGCUSTOMHEADER}%{PIXIU_BASH}", "%{RSYSLOGCUSTOMHEADER}%{PIXIU_NGINX}", "%{RSYSLOGCUSTOMHEADER}%{PIXIU_TOMCAT}", "%{RSYSLOGPREFIX}%{PIXIU_APACHE}", "%{RSYSLOGPREFIX}%{SU_MSG}", "%{RSYSLOGPREFIX}%{SUDO_MSG}", "%{RSYSLOGPREFIX}%{REFRAME_MSG}", "%{RSYSLOGPREFIX}%{BASH_MSG}", "%{RSYSLOGPREFIX}%{FAIL2BAN_MSG}", "%{RSYSLOGPREFIX}%{SINGULARITY_MSG}", "%{RSYSLOGPREFIX}%{DHCPD_MSG}", "%{RSYSLOGPREFIX}%{SSH_MSG}", "%{RSYSLOGPREFIX}%{MODULECMD_MSG}", "%{RSYSLOGPREFIX}%{LMOD_MSG}", "%{RSYSLOGPREFIX}%{NFS_MSG}", "%{RSYSLOGPREFIX}%{CEPH_MSG}", "%{RSYSLOGPREFIX}%{OPENNEBULA_MSG}", "%{RSYSLOGPREFIX}%{JUBE_MSG}", "%{RSYSLOGPREFIX}%{SHOREWALL_MSG}", "%{RSYSLOGPREFIX}%{KEYVALUE_MSG}", "%{RSYSLOGPREFIX}%{QUATTOR_MSG}", "%{RSYSLOGPREFIX}%{SNOOPY_MSG}", "%{RSYSLOGCUSTOM}"]
19+
"message" => ["%{RSYSLOGCUSTOMHEADER}%{PIXIU_ALARM}", "%{RSYSLOGCUSTOMHEADER}%{PIXIU_BASH}", "%{RSYSLOGCUSTOMHEADER}%{PIXIU_NGINX}", "%{RSYSLOGCUSTOMHEADER}%{PIXIU_TOMCAT}", "%{RSYSLOGPREFIX}%{PIXIU_APACHE}", "%{RSYSLOGPREFIX}%{SU_MSG}", "%{RSYSLOGPREFIX}%{SUDO_MSG}", "%{RSYSLOGPREFIX}%{REFRAME_MSG}", "%{RSYSLOGPREFIX}%{BASH_MSG}", "%{RSYSLOGPREFIX}%{FAIL2BAN_MSG}", "%{RSYSLOGPREFIX}%{SINGULARITY_MSG}", "%{RSYSLOGPREFIX}%{DHCPD_MSG}", "%{RSYSLOGPREFIX}%{SSH_MSG}", "%{RSYSLOGPREFIX}%{MODULECMD_MSG}", "%{RSYSLOGPREFIX}%{LMOD_MSG}", "%{RSYSLOGPREFIX}%{NFS_MSG}", "%{RSYSLOGPREFIX}%{CEPH_MSG}", "%{RSYSLOGPREFIX}%{OPENNEBULA_MSG}", "%{RSYSLOGPREFIX}%{JUBE_MSG}", "%{RSYSLOGPREFIX}%{SHOREWALL_MSG}", "%{RSYSLOGPREFIX}%{KEYVALUE_MSG}", "%{RSYSLOGPREFIX}%{QUATTOR_MSG}", "%{RSYSLOGPREFIX}%{SNOOPY_MSG}", "%{RSYSLOGCUSTOM}"]
2020
}
2121
}
2222

@@ -60,6 +60,8 @@ filter {
6060
"objrecovthr" => "bytes"
6161
"actwrite" => "bytes"
6262
"actread" => "bytes"
63+
"used_space" => "bytes"
64+
"quota_space_limit" => "bytes"
6365
}
6466
}
6567

@@ -72,6 +74,12 @@ filter {
7274
"objrecovthr" => "integer"
7375
"actwrite" => "integer"
7476
"actread" => "integer"
77+
"dtree_id" => "integer"
78+
"namespace_id" => "integer"
79+
"used_files" => "integer"
80+
"used_space" => "integer"
81+
"quota_files_limit" => "integer"
82+
"quota_space_limit" => "integer"
7583
}
7684
}
7785
}

0 commit comments

Comments
 (0)