Add NGINX log exporter

docker/cron/periodic/daily/truncate_nginx_exporter_logs

@@ -0,0 +1,12 @@
+#!/bin/sh
+
+# truncate nginx logs that are shared with exporter so they don't fill the disk space
+
+set -e
+
+if [ ! "$CRON_DAILY_TRUNCATE_EXPORTER_LOGS" = "True" ];then
+  exit 0
+fi
+
+# logs are only used by exporter which reads then directly, no need to keep old logs around
+truncate -s0 /var/log/nginx/prometheus-nginxlog-exporter/access.log

docker/defaults.env

@@ -198,6 +198,7 @@ LOGGING_DRIVER=journald
 
 # enable daily or weekly database backups to volume
 CRON_DAILY_POSTGRESQL_BACKUP=True
+CRON_DAILY_TRUNCATE_EXPORTER_LOGS=True
 CRON_WEEKLY_POSTGRESQL_BACKUP=False
 
 INTERNETNL_BRANDING=False

docker/docker-compose.yml

@@ -55,6 +55,7 @@ services:
       # persist certbot configuration between restarts
       - certbot-config:/etc/letsencrypt
       - htpasswd-files:/etc/nginx/htpasswd/external
+      - nginx-logs-exporter:/var/log/nginx/prometheus-nginxlog-exporter/
 
     healthcheck:
       test: ["CMD", "service", "nginx", "status"]
@@ -645,6 +646,7 @@ services:
     volumes:
       - manual-hof:/app/manual-hall-of-fame/
       - postgres-backups:/var/lib/postgresql/backups
+      - nginx-logs-exporter:/var/log/nginx/prometheus-nginxlog-exporter/
 
     healthcheck:
       test: ["CMD", "pgrep", "crond"]
@@ -819,6 +821,24 @@ services:
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
 
+  nginx_logs_exporter:
+    image: ghcr.io/martin-helmich/prometheus-nginxlog-exporter/exporter:v1
+    command:
+      - -config-file=/config.hcl
+    configs:
+      - source: nginx_logs_exporter_config
+        target: /config.hcl
+
+    restart: unless-stopped
+    logging:
+      driver: $LOGGING_DRIVER
+      options:
+        tag: '{{.Name}}'
+    networks:
+      - internal
+    volumes:
+      - nginx-logs-exporter:/var/log/nginx/prometheus-nginxlog-exporter/
+
 volumes:
   postgres: {}
   postgres-backups: {}
@@ -835,6 +855,24 @@ volumes:
   grafana-data: {}
   # shares hosters HoF file between cron and app
   manual-hof: {}
+  # shares nginx log files with log exporter
+  nginx-logs-exporter: {}
+
+configs:
+  nginx_logs_exporter_config:
+    content: |
+      namespace "nginx" {
+        format = "$$remote_addr $$host $$remote_user [$$time_local] $$proxy_host \"$$request\" $$status $$body_bytes_sent \"$$http_referer\" \"$$http_user_agent\" \"$$http_x_forwarded_for\""
+
+        source = {
+          files = ["/var/log/nginx/prometheus-nginxlog-exporter/access.log"]
+        }
+
+        relabel "proxy_host" { from = "proxy_host" }
+        relabel "remote_user" { from = "remote_user" }
+      }
+
+
 
 networks:
   # disable default network

docker/monitoring/grafana/dashboards/nginx-logs.json

@@ -0,0 +1,155 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [],
+          "type": "dashboard"
+        },
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "reqpm"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 19,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 3,
+      "interval": "1m",
+      "options": {
+        "legend": {
+          "calcs": [
+            "sum",
+            "mean",
+            "last"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true,
+          "sortBy": "Total",
+          "sortDesc": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "9.3.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "oUXCLhCMk"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "sum by (status) (rate(nginx_http_response_count_total{proxy_host=\"192.168.42.103:8080\"}[$__rate_interval])) * 60",
+          "format": "time_series",
+          "instant": false,
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Frontend HTTP requests",
+      "type": "timeseries"
+    }
+  ],
+  "refresh": "30s",
+  "schemaVersion": 38,
+  "style": "dark",
+  "tags": [
+    "beta"
+  ],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-3h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "NGINX logs (webserver)",
+  "uid": "2R3DkoIIz",
+  "version": 3,
+  "weekStart": ""
+}

docker/monitoring/prometheus/prometheus.yaml

@@ -85,4 +85,16 @@ scrape_configs:
         action: drop
     static_configs:
       - targets:
-        - docker_stats_exporter:9338
+        - docker_stats_exporter:9338
+
+  - job_name: nginx_logs_exporter
+    scheme: http
+    metrics_path: /metrics
+    metric_relabel_configs:
+      # drop exporter internal metrics
+      - source_labels: [ "__name__" ]
+        regex: '(go|python|process)_.+'
+        action: drop
+    static_configs:
+      - targets:
+        - nginx_logs_exporter:4040

docker/webserver/nginx_templates/app.conf.template

@@ -5,6 +5,13 @@ log_format default '$remote_addr $host $remote_user [$time_local] "$request" '
 
 access_log /var/log/nginx/access.log default;
 
+# create seperate logfile, since the access.log file is a symlink to stdout
+# it cannot be used by prometheus-nginxlog-exporter
+log_format nginx_logs_exporter '$remote_addr $host $remote_user [$time_local] $proxy_host "$request" '
+                   '$status $body_bytes_sent "$http_referer" '
+                   '"$http_user_agent" "$http_x_forwarded_for"';
+access_log /var/log/nginx/prometheus-nginxlog-exporter/access.log nginx_logs_exporter;
+
 # disable version in server banner
 server_tokens off;
 # requires openresty