From 68b7f7fb8adad9af0a54fc6b90030137fb84795c Mon Sep 17 00:00:00 2001 From: Johan Bloemberg Date: Mon, 4 Mar 2024 17:41:43 +0100 Subject: [PATCH] Add NGINX log exporter --- .../daily/truncate_nginx_exporter_logs | 12 ++ docker/defaults.env | 1 + docker/docker-compose.yml | 38 +++++ .../grafana/dashboards/nginx-logs.json | 155 ++++++++++++++++++ docker/monitoring/prometheus/prometheus.yaml | 14 +- .../nginx_templates/app.conf.template | 7 + 6 files changed, 226 insertions(+), 1 deletion(-) create mode 100755 docker/cron/periodic/daily/truncate_nginx_exporter_logs create mode 100644 docker/monitoring/grafana/dashboards/nginx-logs.json diff --git a/docker/cron/periodic/daily/truncate_nginx_exporter_logs b/docker/cron/periodic/daily/truncate_nginx_exporter_logs new file mode 100755 index 000000000..4336f1ca5 --- /dev/null +++ b/docker/cron/periodic/daily/truncate_nginx_exporter_logs @@ -0,0 +1,12 @@ +#!/bin/sh + +# truncate nginx logs that are shared with exporter so they don't fill the disk space + +set -e + +if [ ! "$CRON_DAILY_TRUNCATE_EXPORTER_LOGS" = "True" ];then + exit 0 +fi + +# logs are only used by exporter which reads then directly, no need to keep old logs around +truncate -s0 /var/log/nginx/prometheus-nginxlog-exporter/access.log \ No newline at end of file diff --git a/docker/defaults.env b/docker/defaults.env index 9dd37d739..6ade7cf2c 100644 --- a/docker/defaults.env +++ b/docker/defaults.env @@ -198,6 +198,7 @@ LOGGING_DRIVER=journald # enable daily or weekly database backups to volume CRON_DAILY_POSTGRESQL_BACKUP=True +CRON_DAILY_TRUNCATE_EXPORTER_LOGS=True CRON_WEEKLY_POSTGRESQL_BACKUP=False INTERNETNL_BRANDING=False diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index dafc3c137..e8f31c196 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -55,6 +55,7 @@ services: # persist certbot configuration between restarts - certbot-config:/etc/letsencrypt - htpasswd-files:/etc/nginx/htpasswd/external + - nginx-logs-exporter:/var/log/nginx/prometheus-nginxlog-exporter/ healthcheck: test: ["CMD", "service", "nginx", "status"] @@ -645,6 +646,7 @@ services: volumes: - manual-hof:/app/manual-hall-of-fame/ - postgres-backups:/var/lib/postgresql/backups + - nginx-logs-exporter:/var/log/nginx/prometheus-nginxlog-exporter/ healthcheck: test: ["CMD", "pgrep", "crond"] @@ -819,6 +821,24 @@ services: volumes: - /var/run/docker.sock:/var/run/docker.sock + nginx_logs_exporter: + image: ghcr.io/martin-helmich/prometheus-nginxlog-exporter/exporter:v1 + command: + - -config-file=/config.hcl + configs: + - source: nginx_logs_exporter_config + target: /config.hcl + + restart: unless-stopped + logging: + driver: $LOGGING_DRIVER + options: + tag: '{{.Name}}' + networks: + - internal + volumes: + - nginx-logs-exporter:/var/log/nginx/prometheus-nginxlog-exporter/ + volumes: postgres: {} postgres-backups: {} @@ -835,6 +855,24 @@ volumes: grafana-data: {} # shares hosters HoF file between cron and app manual-hof: {} + # shares nginx log files with log exporter + nginx-logs-exporter: {} + +configs: + nginx_logs_exporter_config: + content: | + namespace "nginx" { + format = "$$remote_addr $$host $$remote_user [$$time_local] $$proxy_host \"$$request\" $$status $$body_bytes_sent \"$$http_referer\" \"$$http_user_agent\" \"$$http_x_forwarded_for\"" + + source = { + files = ["/var/log/nginx/prometheus-nginxlog-exporter/access.log"] + } + + relabel "proxy_host" { from = "proxy_host" } + relabel "remote_user" { from = "remote_user" } + } + + networks: # disable default network diff --git a/docker/monitoring/grafana/dashboards/nginx-logs.json b/docker/monitoring/grafana/dashboards/nginx-logs.json new file mode 100644 index 000000000..2e0101b27 --- /dev/null +++ b/docker/monitoring/grafana/dashboards/nginx-logs.json @@ -0,0 +1,155 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqpm" + }, + "overrides": [] + }, + "gridPos": { + "h": 19, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 3, + "interval": "1m", + "options": { + "legend": { + "calcs": [ + "sum", + "mean", + "last" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Total", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "oUXCLhCMk" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (status) (rate(nginx_http_response_count_total{proxy_host=\"192.168.42.103:8080\"}[$__rate_interval])) * 60", + "format": "time_series", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Frontend HTTP requests", + "type": "timeseries" + } + ], + "refresh": "30s", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "beta" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "NGINX logs (webserver)", + "uid": "2R3DkoIIz", + "version": 3, + "weekStart": "" +} \ No newline at end of file diff --git a/docker/monitoring/prometheus/prometheus.yaml b/docker/monitoring/prometheus/prometheus.yaml index 2274aad0e..d6edb80f9 100644 --- a/docker/monitoring/prometheus/prometheus.yaml +++ b/docker/monitoring/prometheus/prometheus.yaml @@ -85,4 +85,16 @@ scrape_configs: action: drop static_configs: - targets: - - docker_stats_exporter:9338 \ No newline at end of file + - docker_stats_exporter:9338 + + - job_name: nginx_logs_exporter + scheme: http + metrics_path: /metrics + metric_relabel_configs: + # drop exporter internal metrics + - source_labels: [ "__name__" ] + regex: '(go|python|process)_.+' + action: drop + static_configs: + - targets: + - nginx_logs_exporter:4040 \ No newline at end of file diff --git a/docker/webserver/nginx_templates/app.conf.template b/docker/webserver/nginx_templates/app.conf.template index ac607212e..dfa5a9571 100644 --- a/docker/webserver/nginx_templates/app.conf.template +++ b/docker/webserver/nginx_templates/app.conf.template @@ -5,6 +5,13 @@ log_format default '$remote_addr $host $remote_user [$time_local] "$request" ' access_log /var/log/nginx/access.log default; +# create seperate logfile, since the access.log file is a symlink to stdout +# it cannot be used by prometheus-nginxlog-exporter +log_format nginx_logs_exporter '$remote_addr $host $remote_user [$time_local] $proxy_host "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; +access_log /var/log/nginx/prometheus-nginxlog-exporter/access.log nginx_logs_exporter; + # disable version in server banner server_tokens off; # requires openresty