Skip to content

Commit e291a85

Browse files
fenosfenos
committed
feat: monitoring with grafana & prometheus
1 parent b8e74aa commit e291a85

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

63 files changed

+10811
-511
lines changed

‎docker-compose-infra.yml renamed to ‎.docker/docker-compose-infra.yml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,8 @@ services:
102102
\"db_password\": \"postgres\",
103103
\"mode_type\": \"transaction\",
104104
\"pool_size\": 15,
105-
\"pool_checkout_timeout\": 100
105+
\"max_clients\": 200,
106+
\"pool_checkout_timeout\": 5000
106107
}
107108
]
108109
}

‎.docker/docker-compose-monitoring.yml

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
version: '3'
2+
services:
3+
pg_bouncer_exporter:
4+
image: spreaker/prometheus-pgbouncer-exporter
5+
environment:
6+
PGBOUNCER_EXPORTER_HOST: 0.0.0.0
7+
PGBOUNCER_EXPORTER_PORT: 9127
8+
PGBOUNCER_HOST: pg_bouncer
9+
PGBOUNCER_PORT: 6432
10+
PGBOUNCER_USER: postgres
11+
PGBOUNCER_PASS: postgres
12+
depends_on:
13+
- pg_bouncer
14+
ports:
15+
- 9127:9127
16+
17+
postgres_exporter:
18+
image: quay.io/prometheuscommunity/postgres-exporter
19+
ports:
20+
- "9187:9187"
21+
environment:
22+
DATA_SOURCE_NAME: postgres://postgres:postgres@tenant_db:5432/postgres?sslmode=disable
23+
24+
prometheus:
25+
image: prom/prometheus
26+
container_name: prometheus
27+
command:
28+
- '--config.file=/etc/prometheus/prometheus.yml'
29+
ports:
30+
- 9090:9090
31+
restart: unless-stopped
32+
volumes:
33+
- ../monitoring/prometheus:/etc/prometheus
34+
35+
grafana:
36+
image: grafana/grafana
37+
container_name: grafana
38+
ports:
39+
- 3000:3000
40+
restart: unless-stopped
41+
environment:
42+
- GF_SECURITY_ADMIN_USER=admin
43+
- GF_SECURITY_ADMIN_PASSWORD=grafana
44+
volumes:
45+
- ../monitoring/grafana/config:/etc/grafana/provisioning
46+
- ../monitoring/grafana/dashboards:/var/lib/grafana/dashboards

‎.dockerignore

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
11
node_modules
22
dist
3-
.env
4-
DB_MIGRATION_HASH_FILE
3+
.env

‎.env.sample

Lines changed: 3 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ SERVER_PORT=5000
66
SERVER_ADMIN_PORT=5001
77
SERVER_KEEP_ALIVE_TIMEOUT=61
88
SERVER_HEADERS_TIMEOUT=65
9-
SERVER_REGION=region-of-where-your-service-is-running
9+
SERVER_REGION=local
1010

1111

1212
#######################################
@@ -24,20 +24,13 @@ RESPONSE_S_MAXAGE=0
2424
AUTH_JWT_SECRET=f023d3db-39dc-4ac9-87b2-b2be72e9162b
2525
AUTH_JWT_ALGORITHM=HS256
2626

27-
28-
#######################################
29-
# Single Tenant
30-
#######################################
31-
TENANT_ID=bjhaohmqunupljrqypxz
32-
3327
#######################################
3428
# Multi Tenancy
3529
# uncomment MULTI_TENANT=true to enable multi tenancy
36-
# the [Single Tenant] configurations will be ignored
3730
#######################################
3831
# MULTI_TENANT=true
3932
DATABASE_MULTITENANT_URL=postgresql://postgres:[email protected]:5433/postgres
40-
REQUEST_X_FORWARDED_HOST_REGEXP=
33+
REQUEST_X_FORWARDED_HOST_REGEXP=^([a-z]{20}).local.(?:com|dev)$
4134
SERVER_ADMIN_API_KEYS=apikey
4235
AUTH_ENCRYPTION_KEY=encryptionkey
4336

@@ -85,7 +78,7 @@ STORAGE_BACKEND=s3
8578
#######################################
8679
# S3 Backend
8780
#######################################
88-
STORAGE_S3_BUCKET=name-of-your-s3-bucket
81+
STORAGE_S3_BUCKET=supa-storage-bucket
8982
STORAGE_S3_MAX_SOCKETS=200
9083
STORAGE_S3_ENDPOINT=http://127.0.0.1:9000
9184
STORAGE_S3_FORCE_PATH_STYLE=true

‎.env.test.sample

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,20 @@
11
AUTHENTICATED_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJhdXRoZW50aWNhdGVkIiwic3ViIjoiMzE3ZWFkY2UtNjMxYS00NDI5LWEwYmItZjE5YTdhNTE3YjRhIiwiZW1haWwiOiJpbmlhbit0ZXN0MUBzdXBhYmFzZS5pbyIsImV4cCI6MTkzOTEwNzk4NSwiYXBwX21ldGFkYXRhIjp7InByb3ZpZGVyIjoiZW1haWwifSwidXNlcl9tZXRhZGF0YSI6e30sInJvbGUiOiJhdXRoZW50aWNhdGVkIn0.E-x3oYcHIjFCdUO1M3wKDl1Ln32mik0xdHT2PjrvN70
2+
ANON_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJyb2xlIjoiYW5vbiIsImlhdCI6MTYxMzUzMTk4NSwiZXhwIjoxOTI5MTA3OTg1fQ.mqfi__KnQB4v6PkIjkhzfwWrYyF94MEbSC6LnuvVniE
3+
SERVICE_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJyb2xlIjoic2VydmljZV9yb2xlIiwiaWF0IjoxNjEzNTMxOTg1LCJleHAiOjE5MjkxMDc5ODV9.th84OKK0Iz8QchDyXZRrojmKSEZ-OuitQm_5DvLiSIc
4+
5+
TENANT_ID=bjhaohmqunupljrqypxz
6+
ENABLE_DEFAULT_METRICS=false
7+
PG_QUEUE_ENABLE=false
8+
MULTI_TENANT=false
9+
ADMIN_API_KEYS=apikey
10+
11+
DB_INSTALL_ROLES=true
12+
IMAGE_TRANSFORMATION_ENABLED=true
13+
214
AWS_ACCESS_KEY_ID=supa-storage
315
AWS_SECRET_ACCESS_KEY=secret1234
416
AWS_DEFAULT_REGION=ap-southeast-1
5-
GLOBAL_S3_ENDPOINT=http://127.0.0.1:9000
6-
GLOBAL_S3_PROTOCOL=http
7-
GLOBAL_S3_FORCE_PATH_STYLE=true
17+
18+
STORAGE_S3_ENDPOINT=http://127.0.0.1:9000
19+
STORAGE_S3_PROTOCOL=http
20+
STORAGE_S3_FORCE_PATH_STYLE=true

‎.github/workflows/ci.yml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,9 @@ jobs:
7676
GLOBAL_S3_PROTOCOL: http
7777
GLOBAL_S3_FORCE_PATH_STYLE: true
7878
DB_INSTALL_ROLES: true
79+
ENABLE_DEFAULT_METRICS: false
80+
PG_QUEUE_ENABLE: false
81+
MULTI_TENANT: false
7982

8083
- name: Upload coverage results to Coveralls
8184
uses: coverallsapp/github-action@master

‎.gitignore

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,5 +8,4 @@ static/api.json
88
data/
99
bin/
1010
coverage/
11-
.idea/
12-
DB_MIGRATION_HASH_FILE
11+
.idea/

‎Dockerfile

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,10 +31,6 @@ COPY migrations migrations
3131
COPY --from=production-deps /app/node_modules node_modules
3232
# Copy build artifacts from the build stage
3333
COPY --from=build /app/dist dist
34-
COPY ./docker-entrypoint.sh .
35-
36-
RUN node dist/scripts/migration-hash.js
3734

3835
EXPOSE 5000
39-
ENTRYPOINT ["./docker-entrypoint.sh"]
4036
CMD ["node", "dist/server.js"]

‎README.md

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,10 @@ A scalable, light-weight object storage service.
2323

2424
- Copy `.env.sample` to `.env` file.
2525
- Copy `.env.test.sample` to `.env.test`.
26-
- Change `GLOBAL_S3_BUCKET` and `REGION` to the name and region of a S3 bucket.
27-
- If you just want to run the tests and not develop locally, you can skip this step because S3 calls are mocked in our tests.
28-
- [Set up your AWS credentials](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html). Your user must have permissions to `s3:PutObject, s3:GetObject, s3:DeleteObject` in the bucket you have chosen.
26+
27+
```bash
28+
cp .env.sample .env && cp .env.test.sample .env.test
29+
````
2930

3031
**Your root directory should now have both `.env` and `.env.test` files.**
3132

‎docker-compose-multi-tenant.yml

Lines changed: 27 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -51,51 +51,70 @@ services:
5151
IMGPROXY_URL: http://imgproxy:8080
5252
IMGPROXY_REQUEST_TIMEOUT: 15
5353

54+
PG_QUEUE_ENABLE: "true"
55+
5456
tenant_db:
5557
extends:
5658
service: tenant_db
57-
file: docker-compose-infra.yml
59+
file: ./.docker/docker-compose-infra.yml
5860

5961
multitenant_db:
6062
extends:
6163
service: multitenant_db
62-
file: docker-compose-infra.yml
64+
file: ./.docker/docker-compose-infra.yml
6365

6466
supavisor:
6567
extends:
6668
service: supavisor
67-
file: docker-compose-infra.yml
69+
file: ./.docker/docker-compose-infra.yml
6870

6971
supavisor_setup:
7072
depends_on:
7173
supavisor:
7274
condition: service_healthy
7375
extends:
7476
service: supavisor_setup
75-
file: docker-compose-infra.yml
77+
file: ./.docker/docker-compose-infra.yml
7678

7779
minio:
7880
extends:
7981
service: minio
80-
file: docker-compose-infra.yml
82+
file: ./.docker/docker-compose-infra.yml
8183

8284
minio_setup:
8385
extends:
8486
service: minio_setup
85-
file: docker-compose-infra.yml
87+
file: ./.docker/docker-compose-infra.yml
8688
depends_on:
8789
- minio
8890

8991
imgproxy:
9092
extends:
9193
service: imgproxy
92-
file: docker-compose-infra.yml
94+
file: ./.docker/docker-compose-infra.yml
9395

9496
# Optional for rate-limiting
9597
# redis:
9698
# extends:
9799
# service: redis
98-
# file: docker-compose-infra.yml
100+
# file: ./.docker/docker-compose-infra.yml
101+
102+
# Optional for monitoring:
103+
# grafana:
104+
# extends:
105+
# service: grafana
106+
# file: ./.docker/docker-compose-monitoring.yml
107+
#
108+
# prometheus:
109+
# extends:
110+
# service: prometheus
111+
# file: ./.docker/docker-compose-monitoring.yml
112+
#
113+
# postgres_exporter:
114+
# extends:
115+
# service: postgres_exporter
116+
# file: ./.docker/docker-compose-monitoring.yml
117+
99118

100119
configs:
101120
init.sql:

‎docker-compose.yml

Lines changed: 27 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -47,33 +47,54 @@ services:
4747
tenant_db:
4848
extends:
4949
service: tenant_db
50-
file: docker-compose-infra.yml
50+
file: ./.docker/docker-compose-infra.yml
5151

5252
pg_bouncer:
5353
extends:
5454
service: pg_bouncer
55-
file: docker-compose-infra.yml
55+
file: ./.docker/docker-compose-infra.yml
5656

5757
minio:
5858
extends:
5959
service: minio
60-
file: docker-compose-infra.yml
60+
file: ./.docker/docker-compose-infra.yml
6161

6262
minio_setup:
6363
extends:
6464
service: minio_setup
65-
file: docker-compose-infra.yml
65+
file: ./.docker/docker-compose-infra.yml
6666
depends_on:
6767
minio:
6868
condition: service_healthy
6969

7070
imgproxy:
7171
extends:
7272
service: imgproxy
73-
file: docker-compose-infra.yml
73+
file: ./.docker/docker-compose-infra.yml
7474

7575
# Optional for rate-limiting
7676
# redis:
7777
# extends:
7878
# service: redis
79-
# file: docker-compose-infra.yml
79+
# file: ./.docker/docker-compose-infra.yml
80+
81+
# Optional for monitoring
82+
# pg_bouncer_exporter:
83+
# extends:
84+
# service: pg_bouncer_exporter
85+
# file: ./.docker/docker-compose-monitoring.yml
86+
87+
# postgres_exporter:
88+
# extends:
89+
# service: postgres_exporter
90+
# file: ./.docker/docker-compose-monitoring.yml
91+
#
92+
# grafana:
93+
# extends:
94+
# service: grafana
95+
# file: ./.docker/docker-compose-monitoring.yml
96+
#
97+
# prometheus:
98+
# extends:
99+
# service: prometheus
100+
# file: ./.docker/docker-compose-monitoring.yml

‎docker-entrypoint.sh

Lines changed: 0 additions & 11 deletions
This file was deleted.

‎jest-setup.ts

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
import { getConfig, setEnvPaths } from './src/config'
2+
3+
setEnvPaths(['.env.test', '.env'])

‎jest.config.js

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ module.exports = {
44
transform: {
55
'^.+\\.(t|j)sx?$': 'ts-jest',
66
},
7+
setupFiles: ['<rootDir>/jest-setup.ts'],
78
testEnvironment: 'node',
89
testPathIgnorePatterns: ['node_modules', 'dist'],
910
coverageProvider: 'v8',

‎migrations/multitenant/0006-tenants-add-created-at-migrations-version.sql

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
ALTER TABLE tenants ADD COLUMN IF NOT EXISTS cursor_id SERIAL;
22
ALTER TABLE tenants ADD COLUMN IF NOT EXISTS created_at TIMESTAMP DEFAULT current_timestamp;
33
ALTER TABLE tenants ADD COLUMN IF NOT EXISTS migrations_version text null DEFAULT null;
4+
ALTER TABLE tenants ADD COLUMN IF NOT EXISTS migrations_status text null DEFAULT null;
45

5-
create index if not exists tenants_migration_version_idx on tenants(cursor_id, migrations_version);
6+
create index if not exists tenants_migration_version_idx on tenants(cursor_id, migrations_version, migrations_status);

‎monitoring/grafana/config/dashboards/default.yml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
apiVersion: 1
2+
3+
providers:
4+
# <string> an unique provider name. Required
5+
- name: 'Default Dashboard'
6+
# <string> name of the dashboard folder.
7+
folder: 'Storage'
8+
type: file
9+
disableDeletion: false
10+
# <int> how often Grafana will scan for changed dashboards
11+
updateIntervalSeconds: 10
12+
allowUiUpdates: false
13+
options:
14+
# <string, required> path to dashboard files on disk. Required when using the 'file' type
15+
path: /var/lib/grafana/dashboards
16+
foldersFromFilesStructure: true

‎monitoring/grafana/config/datasources/datacourse.yml

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
apiVersion: 1
2+
3+
datasources:
4+
- name: Prometheus
5+
type: prometheus
6+
url: http://prometheus:9090
7+
isDefault: true
8+
access: proxy
9+
editable: true
10+
uid: local_prometheus

0 commit comments

Comments
 (0)