feat: monitoring with grafana & prometheus

docker-compose-infra.yml → .docker/docker-compose-infra.yml

@@ -102,7 +102,8 @@ services:
                 \"db_password\": \"postgres\",
                 \"mode_type\": \"transaction\",
                 \"pool_size\": 15,
-                \"pool_checkout_timeout\": 100
+                \"max_clients\": 200,
+                \"pool_checkout_timeout\": 5000
               }
             ]
           }

.docker/docker-compose-monitoring.yml

@@ -0,0 +1,46 @@
+version: '3'
+services:
+  pg_bouncer_exporter:
+    image: spreaker/prometheus-pgbouncer-exporter
+    environment:
+      PGBOUNCER_EXPORTER_HOST: 0.0.0.0
+      PGBOUNCER_EXPORTER_PORT: 9127
+      PGBOUNCER_HOST: pg_bouncer
+      PGBOUNCER_PORT: 6432
+      PGBOUNCER_USER: postgres
+      PGBOUNCER_PASS: postgres
+    depends_on:
+      - pg_bouncer
+    ports:
+      - 9127:9127
+
+  postgres_exporter:
+    image: quay.io/prometheuscommunity/postgres-exporter
+    ports:
+      - "9187:9187"
+    environment:
+      DATA_SOURCE_NAME: postgres://postgres:postgres@tenant_db:5432/postgres?sslmode=disable
+
+  prometheus:
+    image: prom/prometheus
+    container_name: prometheus
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+    ports:
+      - 9090:9090
+    restart: unless-stopped
+    volumes:
+      - ../monitoring/prometheus:/etc/prometheus
+
+  grafana:
+    image: grafana/grafana
+    container_name: grafana
+    ports:
+      - 3000:3000
+    restart: unless-stopped
+    environment:
+      - GF_SECURITY_ADMIN_USER=admin
+      - GF_SECURITY_ADMIN_PASSWORD=grafana
+    volumes:
+      - ../monitoring/grafana/config:/etc/grafana/provisioning
+      - ../monitoring/grafana/dashboards:/var/lib/grafana/dashboards

.dockerignore

@@ -1,4 +1,3 @@
 node_modules
 dist
-.env
-DB_MIGRATION_HASH_FILE
+.env

.env.sample

@@ -6,7 +6,7 @@ SERVER_PORT=5000
 SERVER_ADMIN_PORT=5001
 SERVER_KEEP_ALIVE_TIMEOUT=61
 SERVER_HEADERS_TIMEOUT=65
-SERVER_REGION=region-of-where-your-service-is-running
+SERVER_REGION=local
 
 
 #######################################
@@ -24,20 +24,13 @@ RESPONSE_S_MAXAGE=0
 AUTH_JWT_SECRET=f023d3db-39dc-4ac9-87b2-b2be72e9162b
 AUTH_JWT_ALGORITHM=HS256
 
-
-#######################################
-# Single Tenant
-#######################################
-TENANT_ID=bjhaohmqunupljrqypxz
-
 #######################################
 # Multi Tenancy
 # uncomment MULTI_TENANT=true to enable multi tenancy
-# the [Single Tenant] configurations will be ignored
 #######################################
 # MULTI_TENANT=true
 DATABASE_MULTITENANT_URL=postgresql://postgres:[email protected]:5433/postgres
-REQUEST_X_FORWARDED_HOST_REGEXP=
+REQUEST_X_FORWARDED_HOST_REGEXP=^([a-z]{20}).local.(?:com|dev)$
 SERVER_ADMIN_API_KEYS=apikey
 AUTH_ENCRYPTION_KEY=encryptionkey
 
@@ -85,7 +78,7 @@ STORAGE_BACKEND=s3
 #######################################
 # S3 Backend
 #######################################
-STORAGE_S3_BUCKET=name-of-your-s3-bucket
+STORAGE_S3_BUCKET=supa-storage-bucket
 STORAGE_S3_MAX_SOCKETS=200
 STORAGE_S3_ENDPOINT=http://127.0.0.1:9000
 STORAGE_S3_FORCE_PATH_STYLE=true

.env.test.sample

@@ -1,7 +1,20 @@
 AUTHENTICATED_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJhdXRoZW50aWNhdGVkIiwic3ViIjoiMzE3ZWFkY2UtNjMxYS00NDI5LWEwYmItZjE5YTdhNTE3YjRhIiwiZW1haWwiOiJpbmlhbit0ZXN0MUBzdXBhYmFzZS5pbyIsImV4cCI6MTkzOTEwNzk4NSwiYXBwX21ldGFkYXRhIjp7InByb3ZpZGVyIjoiZW1haWwifSwidXNlcl9tZXRhZGF0YSI6e30sInJvbGUiOiJhdXRoZW50aWNhdGVkIn0.E-x3oYcHIjFCdUO1M3wKDl1Ln32mik0xdHT2PjrvN70
+ANON_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJyb2xlIjoiYW5vbiIsImlhdCI6MTYxMzUzMTk4NSwiZXhwIjoxOTI5MTA3OTg1fQ.mqfi__KnQB4v6PkIjkhzfwWrYyF94MEbSC6LnuvVniE
+SERVICE_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJyb2xlIjoic2VydmljZV9yb2xlIiwiaWF0IjoxNjEzNTMxOTg1LCJleHAiOjE5MjkxMDc5ODV9.th84OKK0Iz8QchDyXZRrojmKSEZ-OuitQm_5DvLiSIc
+
+TENANT_ID=bjhaohmqunupljrqypxz
+ENABLE_DEFAULT_METRICS=false
+PG_QUEUE_ENABLE=false
+MULTI_TENANT=false
+ADMIN_API_KEYS=apikey
+
+DB_INSTALL_ROLES=true
+IMAGE_TRANSFORMATION_ENABLED=true
+
 AWS_ACCESS_KEY_ID=supa-storage
 AWS_SECRET_ACCESS_KEY=secret1234
 AWS_DEFAULT_REGION=ap-southeast-1
-GLOBAL_S3_ENDPOINT=http://127.0.0.1:9000
-GLOBAL_S3_PROTOCOL=http
-GLOBAL_S3_FORCE_PATH_STYLE=true
+
+STORAGE_S3_ENDPOINT=http://127.0.0.1:9000
+STORAGE_S3_PROTOCOL=http
+STORAGE_S3_FORCE_PATH_STYLE=true

.github/workflows/ci.yml

@@ -76,6 +76,9 @@ jobs:
           GLOBAL_S3_PROTOCOL: http
           GLOBAL_S3_FORCE_PATH_STYLE: true
           DB_INSTALL_ROLES: true
+          ENABLE_DEFAULT_METRICS: false
+          PG_QUEUE_ENABLE: false
+          MULTI_TENANT: false
 
       - name: Upload coverage results to Coveralls
         uses: coverallsapp/github-action@master

.gitignore

@@ -8,5 +8,4 @@ static/api.json
 data/
 bin/
 coverage/
-.idea/
-DB_MIGRATION_HASH_FILE
+.idea/

Dockerfile

@@ -31,10 +31,6 @@ COPY migrations migrations
 COPY --from=production-deps /app/node_modules node_modules
 # Copy build artifacts from the build stage
 COPY --from=build /app/dist dist
-COPY ./docker-entrypoint.sh .
-
-RUN node dist/scripts/migration-hash.js
 
 EXPOSE 5000
-ENTRYPOINT ["./docker-entrypoint.sh"]
 CMD ["node", "dist/server.js"]

README.md

@@ -23,9 +23,10 @@ A scalable, light-weight object storage service.
 
 - Copy `.env.sample` to `.env` file.
 - Copy `.env.test.sample` to `.env.test`.
-- Change `GLOBAL_S3_BUCKET` and `REGION` to the name and region of a S3 bucket.
-  - If you just want to run the tests and not develop locally, you can skip this step because S3 calls are mocked in our tests.
-- [Set up your AWS credentials](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html). Your user must have permissions to `s3:PutObject, s3:GetObject, s3:DeleteObject` in the bucket you have chosen.
+
+```bash
+cp .env.sample .env && cp .env.test.sample .env.test
+````
 
 **Your root directory should now have both `.env` and `.env.test` files.**
 

docker-compose-multi-tenant.yml

@@ -51,51 +51,70 @@ services:
       IMGPROXY_URL: http://imgproxy:8080
       IMGPROXY_REQUEST_TIMEOUT: 15
 
+      PG_QUEUE_ENABLE: "true"
+
   tenant_db:
     extends:
       service: tenant_db
-      file: docker-compose-infra.yml
+      file: ./.docker/docker-compose-infra.yml
 
   multitenant_db:
     extends:
       service: multitenant_db
-      file: docker-compose-infra.yml
+      file: ./.docker/docker-compose-infra.yml
 
   supavisor:
     extends:
       service: supavisor
-      file: docker-compose-infra.yml
+      file: ./.docker/docker-compose-infra.yml
 
   supavisor_setup:
     depends_on:
       supavisor:
         condition: service_healthy
     extends:
       service: supavisor_setup
-      file: docker-compose-infra.yml
+      file: ./.docker/docker-compose-infra.yml
 
   minio:
     extends:
       service: minio
-      file: docker-compose-infra.yml
+      file: ./.docker/docker-compose-infra.yml
 
   minio_setup:
     extends:
       service: minio_setup
-      file: docker-compose-infra.yml
+      file: ./.docker/docker-compose-infra.yml
     depends_on:
       - minio
 
   imgproxy:
     extends:
       service: imgproxy
-      file: docker-compose-infra.yml
+      file: ./.docker/docker-compose-infra.yml
 
 #  Optional for rate-limiting
 #  redis:
 #    extends:
 #      service: redis
-#      file: docker-compose-infra.yml
+#      file: ./.docker/docker-compose-infra.yml
+
+# Optional for monitoring:
+#  grafana:
+#    extends:
+#      service: grafana
+#      file: ./.docker/docker-compose-monitoring.yml
+#
+#  prometheus:
+#    extends:
+#      service: prometheus
+#      file: ./.docker/docker-compose-monitoring.yml
+#
+#  postgres_exporter:
+#    extends:
+#      service: postgres_exporter
+#      file: ./.docker/docker-compose-monitoring.yml
+
 
 configs:
   init.sql:

docker-compose.yml

@@ -47,33 +47,54 @@ services:
   tenant_db:
     extends:
       service: tenant_db
-      file: docker-compose-infra.yml
+      file: ./.docker/docker-compose-infra.yml
 
   pg_bouncer:
     extends:
       service: pg_bouncer
-      file: docker-compose-infra.yml
+      file: ./.docker/docker-compose-infra.yml
 
   minio:
     extends:
       service: minio
-      file: docker-compose-infra.yml
+      file: ./.docker/docker-compose-infra.yml
 
   minio_setup:
     extends:
       service: minio_setup
-      file: docker-compose-infra.yml
+      file: ./.docker/docker-compose-infra.yml
     depends_on:
       minio:
         condition: service_healthy
 
   imgproxy:
     extends:
       service: imgproxy
-      file: docker-compose-infra.yml
+      file: ./.docker/docker-compose-infra.yml
 
 #  Optional for rate-limiting
 #  redis:
 #    extends:
 #      service: redis
-#      file: docker-compose-infra.yml
+#      file: ./.docker/docker-compose-infra.yml
+
+#  Optional for monitoring
+#  pg_bouncer_exporter:
+#    extends:
+#      service: pg_bouncer_exporter
+#      file: ./.docker/docker-compose-monitoring.yml
+
+#  postgres_exporter:
+#    extends:
+#      service: postgres_exporter
+#      file: ./.docker/docker-compose-monitoring.yml
+#
+#  grafana:
+#    extends:
+#      service: grafana
+#      file: ./.docker/docker-compose-monitoring.yml
+#
+#  prometheus:
+#    extends:
+#      service: prometheus
+#      file: ./.docker/docker-compose-monitoring.yml

jest-setup.ts

@@ -0,0 +1,5 @@
+import { setEnvPaths } from './src/config'
+
+setEnvPaths(['.env.test', '.env'])
+
+console.log('setup')

jest.config.js

@@ -4,6 +4,7 @@ module.exports = {
   transform: {
     '^.+\\.(t|j)sx?$': 'ts-jest',
   },
+  setupFiles: ['<rootDir>/jest-setup.ts'],
   testEnvironment: 'node',
   testPathIgnorePatterns: ['node_modules', 'dist'],
   coverageProvider: 'v8',

migrations/multitenant/0006-tenants-add-created-at-migrations-version.sql

@@ -1,5 +1,6 @@
 ALTER TABLE tenants ADD COLUMN IF NOT EXISTS cursor_id SERIAL;
 ALTER TABLE tenants ADD COLUMN IF NOT EXISTS created_at TIMESTAMP DEFAULT current_timestamp;
 ALTER TABLE tenants ADD COLUMN IF NOT EXISTS migrations_version text null DEFAULT null;
+ALTER TABLE tenants ADD COLUMN IF NOT EXISTS migrations_status text null DEFAULT null;
 
-create index if not exists tenants_migration_version_idx on tenants(cursor_id, migrations_version);
+create index if not exists tenants_migration_version_idx on tenants(cursor_id, migrations_version, migrations_status);

monitoring/grafana/config/dashboards/default.yml

@@ -0,0 +1,16 @@
+apiVersion: 1
+
+providers:
+  # <string> an unique provider name. Required
+  - name: 'Default Dashboard'
+    # <string> name of the dashboard folder.
+    folder: 'Storage'
+    type: file
+    disableDeletion: false
+    # <int> how often Grafana will scan for changed dashboards
+    updateIntervalSeconds: 10
+    allowUiUpdates: false
+    options:
+      # <string, required> path to dashboard files on disk. Required when using the 'file' type
+      path: /var/lib/grafana/dashboards
+      foldersFromFilesStructure: true

monitoring/grafana/config/datasources/datacourse.yml

@@ -0,0 +1,10 @@
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    type: prometheus
+    url: http://prometheus:9090
+    isDefault: true
+    access: proxy
+    editable: true
+    uid: local_prometheus