generated from giantswarm/template-app
-
Notifications
You must be signed in to change notification settings - Fork 3
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add alert
ClusterCrossplaneResourcesNotReady
for Crossplane resourc…
…es that are critical for clusters
- Loading branch information
Showing
3 changed files
with
57 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
30 changes: 30 additions & 0 deletions
30
helm/prometheus-rules/templates/kaas/phoenix/alerting-rules/cluster-crossplane.rules.yml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,30 @@ | ||
apiVersion: monitoring.coreos.com/v1 | ||
kind: PrometheusRule | ||
metadata: | ||
creationTimestamp: null | ||
labels: | ||
{{- include "labels.common" . | nindent 4 }} | ||
name: cluster-crossplane.rules | ||
namespace: {{ .Values.namespace }} | ||
spec: | ||
groups: | ||
- name: cluster-crossplane | ||
rules: | ||
- alert: ClusterCrossplaneResourcesNotReady | ||
annotations: | ||
# Crossplane doesn't offer object names and the objects are stored on the MC, so right | ||
# now (2025-01), we can't make this alert WC-specific. | ||
description: '{{`Not all managed Crossplane resources of type "{{ $labels.gvk }}" on {{ $labels.cluster_id }} are ready. This could affect creation or health of workload clusters.`}}' | ||
opsrecipe: cluster-crossplane-resources | ||
# Match critical resources deployed by cluster-aws via aws-nth-crossplane-resources, | ||
# cilium-crossplane-resources, ... | ||
expr: crossplane_managed_resource_exists{gvk=~".*Kind=(Queue|QueuePolicy|Role|Rule|SecurityGroup|SecurityGroupEgressRule|SecurityGroupIngressRule|Target)"} != crossplane_managed_resource_ready{gvk=~".*Kind=(Queue|QueuePolicy|Role|Rule|SecurityGroup|SecurityGroupEgressRule|SecurityGroupIngressRule|Target)"} | ||
for: 15m | ||
labels: | ||
area: kaas | ||
cancel_if_cluster_status_creating: "true" | ||
cancel_if_cluster_status_deleting: "true" | ||
cancel_if_cluster_status_updating: "true" | ||
cancel_if_outside_working_hours: {{ include "workingHoursOnly" . }} | ||
severity: page | ||
team: phoenix |
23 changes: 23 additions & 0 deletions
23
test/tests/providers/capi/capa/kaas/phoenix/alerting-rules/cluster-crossplane.rules.test.yml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,23 @@ | ||
rule_files: | ||
- cluster-crossplane.rules.yml | ||
|
||
tests: | ||
- interval: 1m | ||
input_series: | ||
- series: 'crossplane_managed_resource_exists{gvk="cloudwatchevents.aws.upbound.io/v1beta1, Kind=Rule", cluster_id="mymc"}' | ||
values: "6x20" | ||
- series: 'crossplane_managed_resource_ready{gvk="cloudwatchevents.aws.upbound.io/v1beta1, Kind=Rule", cluster_id="mymc"}' | ||
values: "5x20" | ||
|
||
alert_rule_test: | ||
- alertname: ClusterCrossplaneResourcesNotReady | ||
eval_time: 20m | ||
exp_alerts: | ||
- exp_labels: | ||
area: kaas | ||
cancel_if_outside_working_hours: "true" | ||
severity: page | ||
team: phoenix | ||
exp_annotations: | ||
description: "TODO does this test work?" | ||
opsrecipe: cluster-crossplane-resources |