Add recording rules and a entrypoint readme for k8s scripts

Signed-off-by: K mehant <[email protected]>

scripts/kubernetes/README.md

@@ -0,0 +1,3 @@
+# K8s deployment scripts for Bassa
+
+This folder container K8s scripts for Bassa deployment on a single node cluster. If you are looking out to use container based deployment infrastructure with K8s orchestration tool, then these configurations will surely help you get started for your own infra.

scripts/kubernetes/manifests/monitoring/prometheus-monitor/rules.yaml

@@ -7,3 +7,66 @@ metadata:
   namespace: bassa-monitoring
 data:
   alert.rules: |-
+    groups:
+      - name: Deployment
+        rules:
+        - alert: Deployment at 0 Replicas
+          annotations:
+            summary: Deployment {{$labels.deployment}} in {{$labels.namespace}} is currently having no pods running
+          expr: |
+            sum(kube_deployment_status_replicas{pod_template_hash=""}) by (deployment,namespace)  < 1
+          for: 1m
+          labels:
+            name: 'bassa-mail-notifications'
+      - name: Pods
+        rules:
+        - alert: Container restarted
+          annotations:
+            summary: Container named {{$labels.container}} in {{$labels.pod}} in {{$labels.namespace}} was restarted
+          expr: |
+            sum(increase(kube_pod_container_status_restarts_total{namespace!="kube-system",pod_template_hash=""}[1m])) by (pod,namespace,container) > 0
+          for: 0m
+          labels:
+            name: 'bassa-mail-notifications'
+        - alert: High Memory Usage of Container 
+          annotations: 
+            summary: Container named {{$labels.container}} in {{$labels.pod}} in {{$labels.namespace}} is using more than 75% of Memory Limit
+          expr: | 
+            ((( sum(container_memory_usage_bytes{image!="",container_name!="POD", namespace!="kube-system"}) by (namespace,container_name,pod_name)  / sum(container_spec_memory_limit_bytes{image!="",container_name!="POD",namespace!="kube-system"}) by (namespace,container_name,pod_name) ) * 100 ) < +Inf ) > 75
+          for: 5m
+          labels: 
+            name: 'bassa-mail-notifications'
+        - alert: High CPU Usage of Container 
+          annotations: 
+            summary: Container named {{$labels.container}} in {{$labels.pod}} in {{$labels.namespace}} is using more than 75% of CPU Limit
+          expr: | 
+            ((sum(irate(container_cpu_usage_seconds_total{image!="",container_name!="POD", namespace!="kube-system"}[30s])) by (namespace,container_name,pod_name) / sum(container_spec_cpu_quota{image!="",container_name!="POD", namespace!="kube-system"} / container_spec_cpu_period{image!="",container_name!="POD", namespace!="kube-system"}) by (namespace,container_name,pod_name) ) * 100)  > 75
+          for: 5m
+          labels: 
+            name: 'bassa-mail-notifications'
+      - name: Nodes
+        rules:
+        - alert: High Node Memory Usage
+          annotations:
+            summary: Node {{$labels.kubernetes_io_hostname}} has more than 80% memory used. Plan Capcity
+          expr: |
+            (sum (container_memory_working_set_bytes{id="/",container_name!="POD"}) by (kubernetes_io_hostname) / sum (machine_memory_bytes{}) by (kubernetes_io_hostname) * 100) > 80
+          for: 5m
+          labels:
+            name: 'bassa-mail-notifications'
+        - alert: High Node CPU Usage
+          annotations:
+            summary: Node {{$labels.kubernetes_io_hostname}} has more than 80% allocatable cpu used. Plan Capacity.
+          expr: |
+            (sum(rate(container_cpu_usage_seconds_total{id="/", container_name!="POD"}[1m])) by (kubernetes_io_hostname) / sum(machine_cpu_cores) by (kubernetes_io_hostname)  * 100) > 80
+          for: 5m
+          labels:
+            name: 'bassa-mail-notifications'
+        - alert: High Node Disk Usage
+          annotations:
+            summary: Node {{$labels.kubernetes_io_hostname}} has more than 85% disk used. Plan Capacity.
+          expr: |
+            (sum(container_fs_usage_bytes{device=~"^/dev/[sv]d[a-z][1-9]$",id="/",container_name!="POD"}) by (kubernetes_io_hostname) / sum(container_fs_limit_bytes{container_name!="POD",device=~"^/dev/[sv]d[a-z][1-9]$",id="/"}) by (kubernetes_io_hostname)) * 100 > 85
+          for: 5m
+          labels:
+            name: 'bassa-mail-notifications'