From 6e4e79d9a06e54aad8a72052fa33e1b899888387 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Tue, 22 Apr 2025 10:58:19 +0100 Subject: [PATCH 1/2] feat: add `alertmanager-0.28.1` --- doc/source/configuration/monitoring.rst | 15 ++++++++-- etc/kayobe/kolla-image-tags.yml | 3 ++ ...metheus-alertmanager.msteamsv2.yml.example | 29 +++++++++++++++++++ .../bump-alertmanager-958f90fa2bc9b562.yaml | 5 ++++ 4 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 etc/kayobe/kolla/config/prometheus/prometheus-alertmanager.msteamsv2.yml.example create mode 100644 releasenotes/notes/bump-alertmanager-958f90fa2bc9b562.yaml diff --git a/doc/source/configuration/monitoring.rst b/doc/source/configuration/monitoring.rst index 77c5e47f7..7414a5398 100644 --- a/doc/source/configuration/monitoring.rst +++ b/doc/source/configuration/monitoring.rst @@ -81,8 +81,8 @@ on the overcloud hosts: SMART reporting should now be enabled along with a Prometheus alert for unhealthy disks and a Grafana dashboard called ``Hardware Overview``. -Alertmanager and Slack -====================== +Alertmanager, Slack and Microsoft Teams +======================================= StackHPC Kayobe configuration comes bundled with an array of alerts but does not enable any receivers for notifications by default. Various receivers can be @@ -119,6 +119,17 @@ available `here `__. They simply need to be added to one of the ``*.rules`` files in the prometheus configuration directory. +If however you are using Microsoft Teams instead of Slack, you can use Prometheus +Alertmanager's built-in support for the new message format based on Power Automate flows. +You will need an incoming webhook URL for your Teams channel. +This can be done by following `these instructions `__. +To set up a receiver, create a ``prometheus-alertmanager.yml`` file under +``etc/kayobe/kolla/config/prometheus/``. +An example config is stored in this directory known as ``prometheus-alertmanager.msteamvs2.yml.example``. +The example configuration uses two Slack channels. +One channel receives all alerts while the other only receives alerts tagged as critical. +Feel free to modify the example configuration to suit your needs. + Ceph Monitoring =============== diff --git a/etc/kayobe/kolla-image-tags.yml b/etc/kayobe/kolla-image-tags.yml index 16d316158..d849595cc 100644 --- a/etc/kayobe/kolla-image-tags.yml +++ b/etc/kayobe/kolla-image-tags.yml @@ -49,6 +49,9 @@ kolla_image_tags: ovn: rocky-9: 2024.1-rocky-9-20250219T113722 ubuntu-jammy: 2024.1-ubuntu-jammy-20250219T113722 + prometheus_alertmanager: + rocky-9: 2024.1-rocky-9-20250422T103147 + ubuntu-jammy: 2024.1-ubuntu-jammy-20250422T103147 skyline_apiserver: rocky-9: 2024.1-rocky-9-20250408T133253 ubuntu-jammy: 2024.1-ubuntu-jammy-20250408T133253 diff --git a/etc/kayobe/kolla/config/prometheus/prometheus-alertmanager.msteamsv2.yml.example b/etc/kayobe/kolla/config/prometheus/prometheus-alertmanager.msteamsv2.yml.example new file mode 100644 index 000000000..f437429df --- /dev/null +++ b/etc/kayobe/kolla/config/prometheus/prometheus-alertmanager.msteamsv2.yml.example @@ -0,0 +1,29 @@ +--- +global: + resolve_timeout: 5m + smtp_require_tls: true + +route: + receiver: 'msteamsv2-notifications' + group_by: [alertname] + group_wait: 30s + group_interval: 5m + repeat_interval: 4h + + routes: + - matchers: + - severity=~"critical|alert" + receiver: 'msteamvs2-critical-notifications' + +receivers: + - name: 'msteamsv2-notifications' + msteamsv2_configs: + - webhook_url: '{{ secrets_msteams_notification_channel_url }}' + send_resolved: true + - name: 'msteamsv2-critical-notifications' + msteamsv2_configs: + - webhook_url: '{{ secrets_msteams_notification_critical_channel_url }}' + send_resolved: true + +templates: + - '/etc/prometheus/*.tmpl' diff --git a/releasenotes/notes/bump-alertmanager-958f90fa2bc9b562.yaml b/releasenotes/notes/bump-alertmanager-958f90fa2bc9b562.yaml new file mode 100644 index 000000000..22b60e653 --- /dev/null +++ b/releasenotes/notes/bump-alertmanager-958f90fa2bc9b562.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Prometheus Alertmanager has been updated to ``0.28.1``. This release + includes support for Microsoft Teams notifications. From d31858bd4dd1f27bd68c13e2cd52e122cdfead2c Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Fri, 25 Apr 2025 15:44:49 +0100 Subject: [PATCH 2/2] fix: handle undefined variable in `msteams` example --- .../prometheus/prometheus-alertmanager.msteamsv2.yml.example | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/kayobe/kolla/config/prometheus/prometheus-alertmanager.msteamsv2.yml.example b/etc/kayobe/kolla/config/prometheus/prometheus-alertmanager.msteamsv2.yml.example index f437429df..e4575f831 100644 --- a/etc/kayobe/kolla/config/prometheus/prometheus-alertmanager.msteamsv2.yml.example +++ b/etc/kayobe/kolla/config/prometheus/prometheus-alertmanager.msteamsv2.yml.example @@ -18,11 +18,11 @@ route: receivers: - name: 'msteamsv2-notifications' msteamsv2_configs: - - webhook_url: '{{ secrets_msteams_notification_channel_url }}' + - webhook_url: '{{ secrets_msteams_notification_channel_url | default('https://prod-01.westeurope.logic.azure.com/workflows/') }}' send_resolved: true - name: 'msteamsv2-critical-notifications' msteamsv2_configs: - - webhook_url: '{{ secrets_msteams_notification_critical_channel_url }}' + - webhook_url: '{{ secrets_msteams_notification_critical_channel_url | default('https://prod-01.westeurope.logic.azure.com/workflows/') }}' send_resolved: true templates: