feat: monitoring with grafana & prometheus

Author: fenos

docker-compose-infra.yml renamed to .docker/docker-compose-infra.yml

@@ -95,14 +95,14 @@ services:
             \"upstream_ssl\": false,
             \"enforce_ssl\": false,
             \"default_max_clients\": 200,
-            \"default_pool_size\": 15,
+            \"default_pool_size\": 10,
             \"users\": [
               {
                 \"db_user\": \"postgres\",
                 \"db_password\": \"postgres\",
                 \"mode_type\": \"transaction\",
-                \"pool_size\": 15,
-                \"pool_checkout_timeout\": 100
+                \"pool_size\": 2,
+                \"pool_checkout_timeout\": 5000
               }
             ]
           }

.docker/docker-compose-monitoring.yml

@@ -0,0 +1,46 @@
+version: '3'
+services:
+  pg_bouncer_exporter:
+    image: spreaker/prometheus-pgbouncer-exporter
+    environment:
+      PGBOUNCER_EXPORTER_HOST: 0.0.0.0
+      PGBOUNCER_EXPORTER_PORT: 9127
+      PGBOUNCER_HOST: pg_bouncer
+      PGBOUNCER_PORT: 6432
+      PGBOUNCER_USER: postgres
+      PGBOUNCER_PASS: postgres
+    depends_on:
+      - pg_bouncer
+    ports:
+      - 9127:9127
+
+  postgres_exporter:
+    image: quay.io/prometheuscommunity/postgres-exporter
+    ports:
+      - "9187:9187"
+    environment:
+      DATA_SOURCE_NAME: postgres://postgres:postgres@tenant_db:5432/postgres?sslmode=disable
+
+  prometheus:
+    image: prom/prometheus
+    container_name: prometheus
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+    ports:
+      - 9090:9090
+    restart: unless-stopped
+    volumes:
+      - ./monitoring/prometheus:/etc/prometheus
+
+  grafana:
+    image: grafana/grafana
+    container_name: grafana
+    ports:
+      - 3000:3000
+    restart: unless-stopped
+    environment:
+      - GF_SECURITY_ADMIN_USER=admin
+      - GF_SECURITY_ADMIN_PASSWORD=grafana
+    volumes:
+      - ./monitoring/grafana/config:/etc/grafana/provisioning
+      - ./monitoring/grafana/dashboards:/var/lib/grafana/dashboards

.env.sample

@@ -24,20 +24,13 @@ RESPONSE_S_MAXAGE=0
 AUTH_JWT_SECRET=f023d3db-39dc-4ac9-87b2-b2be72e9162b
 AUTH_JWT_ALGORITHM=HS256
 
-
-#######################################
-# Single Tenant
-#######################################
-TENANT_ID=bjhaohmqunupljrqypxz
-
 #######################################
 # Multi Tenancy
 # uncomment MULTI_TENANT=true to enable multi tenancy
-# the [Single Tenant] configurations will be ignored
 #######################################
 # MULTI_TENANT=true
 DATABASE_MULTITENANT_URL=postgresql://postgres:[email protected]:5433/postgres
-REQUEST_X_FORWARDED_HOST_REGEXP=
+REQUEST_X_FORWARDED_HOST_REGEXP=^([a-z]{20}).local.(?:com|dev)$
 SERVER_ADMIN_API_KEYS=apikey
 AUTH_ENCRYPTION_KEY=encryptionkey
 
@@ -85,7 +78,7 @@ STORAGE_BACKEND=s3
 #######################################
 # S3 Backend
 #######################################
-STORAGE_S3_BUCKET=name-of-your-s3-bucket
+STORAGE_S3_BUCKET=supa-storage-bucket
 STORAGE_S3_MAX_SOCKETS=200
 STORAGE_S3_ENDPOINT=http://127.0.0.1:9000
 STORAGE_S3_FORCE_PATH_STYLE=true

Dockerfile

@@ -31,10 +31,6 @@ COPY migrations migrations
 COPY --from=production-deps /app/node_modules node_modules
 # Copy build artifacts from the build stage
 COPY --from=build /app/dist dist
-COPY ./docker-entrypoint.sh .
-
-RUN node dist/scripts/migration-hash.js
 
 EXPOSE 5000
-ENTRYPOINT ["./docker-entrypoint.sh"]
 CMD ["node", "dist/server.js"]

README.md

@@ -23,9 +23,10 @@ A scalable, light-weight object storage service.
 
 - Copy `.env.sample` to `.env` file.
 - Copy `.env.test.sample` to `.env.test`.
-- Change `GLOBAL_S3_BUCKET` and `REGION` to the name and region of a S3 bucket.
-  - If you just want to run the tests and not develop locally, you can skip this step because S3 calls are mocked in our tests.
-- [Set up your AWS credentials](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html). Your user must have permissions to `s3:PutObject, s3:GetObject, s3:DeleteObject` in the bucket you have chosen.
+
+```bash
+cp .env.sample .env && cp .env.test.sample .env.test
+````
 
 **Your root directory should now have both `.env` and `.env.test` files.**
 

docker-compose-multi-tenant.yml

@@ -51,51 +51,71 @@ services:
       IMGPROXY_URL: http://imgproxy:8080
       IMGPROXY_REQUEST_TIMEOUT: 15
 
+      PG_QUEUE_ENABLE: "true"
+      DEBUG: "knex:tx"
+
   tenant_db:
     extends:
       service: tenant_db
-      file: docker-compose-infra.yml
+      file: ./.docker/docker-compose-infra.yml
 
   multitenant_db:
     extends:
       service: multitenant_db
-      file: docker-compose-infra.yml
+      file: ./.docker/docker-compose-infra.yml
 
   supavisor:
     extends:
       service: supavisor
-      file: docker-compose-infra.yml
+      file: ./.docker/docker-compose-infra.yml
 
   supavisor_setup:
     depends_on:
       supavisor:
         condition: service_healthy
     extends:
       service: supavisor_setup
-      file: docker-compose-infra.yml
+      file: ./.docker/docker-compose-infra.yml
 
   minio:
     extends:
       service: minio
-      file: docker-compose-infra.yml
+      file: ./.docker/docker-compose-infra.yml
 
   minio_setup:
     extends:
       service: minio_setup
-      file: docker-compose-infra.yml
+      file: ./.docker/docker-compose-infra.yml
     depends_on:
       - minio
 
   imgproxy:
     extends:
       service: imgproxy
-      file: docker-compose-infra.yml
+      file: ./.docker/docker-compose-infra.yml
 
 #  Optional for rate-limiting
 #  redis:
 #    extends:
 #      service: redis
-#      file: docker-compose-infra.yml
+#      file: ./.docker/docker-compose-infra.yml
+
+# Optional for monitoring:
+#  grafana:
+#    extends:
+#      service: grafana
+#      file: ./.docker/docker-compose-monitoring.yml
+#
+#  prometheus:
+#    extends:
+#      service: prometheus
+#      file: ./.docker/docker-compose-monitoring.yml
+#
+#  postgres_exporter:
+#    extends:
+#      service: postgres_exporter
+#      file: ./.docker/docker-compose-monitoring.yml
+
 
 configs:
   init.sql:

docker-compose.yml

@@ -3,7 +3,9 @@
 version: '3'
 services:
   storage:
-    image: supabase/storage-api:latest
+#    image: supabase/storage-api:latest
+    build:
+      context: .
     ports:
       - '5000:5000'
       - '5001:5001'
@@ -47,33 +49,54 @@ services:
   tenant_db:
     extends:
       service: tenant_db
-      file: docker-compose-infra.yml
+      file: ./.docker/docker-compose-infra.yml
 
   pg_bouncer:
     extends:
       service: pg_bouncer
-      file: docker-compose-infra.yml
+      file: ./.docker/docker-compose-infra.yml
 
   minio:
     extends:
       service: minio
-      file: docker-compose-infra.yml
+      file: ./.docker/docker-compose-infra.yml
 
   minio_setup:
     extends:
       service: minio_setup
-      file: docker-compose-infra.yml
+      file: ./.docker/docker-compose-infra.yml
     depends_on:
       minio:
         condition: service_healthy
 
   imgproxy:
     extends:
       service: imgproxy
-      file: docker-compose-infra.yml
+      file: ./.docker/docker-compose-infra.yml
 
 #  Optional for rate-limiting
 #  redis:
 #    extends:
 #      service: redis
-#      file: docker-compose-infra.yml
+#      file: ./.docker/docker-compose-infra.yml
+
+#  Optional for monitoring
+#  pg_bouncer_exporter:
+#    extends:
+#      service: pg_bouncer_exporter
+#      file: ./.docker/docker-compose-monitoring.yml
+
+#  postgres_exporter:
+#    extends:
+#      service: postgres_exporter
+#      file: ./.docker/docker-compose-monitoring.yml
+#
+#  grafana:
+#    extends:
+#      service: grafana
+#      file: ./.docker/docker-compose-monitoring.yml
+#
+#  prometheus:
+#    extends:
+#      service: prometheus
+#      file: ./.docker/docker-compose-monitoring.yml

docker-entrypoint.sh

@@ -0,0 +1,16 @@
+apiVersion: 1
+
+providers:
+  # <string> an unique provider name. Required
+  - name: 'Default Dashboard'
+    # <string> name of the dashboard folder.
+    folder: 'Storage'
+    type: file
+    disableDeletion: false
+    # <int> how often Grafana will scan for changed dashboards
+    updateIntervalSeconds: 10
+    allowUiUpdates: false
+    options:
+      # <string, required> path to dashboard files on disk. Required when using the 'file' type
+      path: /var/lib/grafana/dashboards
+      foldersFromFilesStructure: true

monitoring/grafana/config/dashboards/default.yml

@@ -0,0 +1,10 @@
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    type: prometheus
+    url: http://prometheus:9090
+    isDefault: true
+    access: proxy
+    editable: true
+    uid: local_prometheus