From 0dc1d4f1f9e5b2dcecfb3ddb18d31de930d47110 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Thu, 4 Apr 2024 17:08:32 -0400 Subject: [PATCH 01/17] Remove KoboCAT, secondary postgres, update nginx config --- ...docker-compose.backend.override.yml.sample | 4 +- docker-compose.backend.primary.yml | 15 -- ...pose.backend.secondary.override.yml.sample | 28 --- docker-compose.backend.secondary.yml | 10 - ...template.yml => docker-compose.backend.yml | 15 +- docker-compose.frontend.override.yml.sample | 21 -- docker-compose.frontend.yml | 189 ++++++++++++------ docker-compose.maintenance.yml | 4 +- env/envfiles/aws.txt | 14 -- env/envfiles/databases.txt | 6 - env/envfiles/django.txt | 10 + env/envfiles/external_services.txt | 5 +- env/envfiles/kobocat.txt | 26 --- env/envfiles/kpi.txt | 22 -- env/envfiles/nginx.txt | 10 - .../docker-entrypoint.d/30-init-kobo-nginx.sh | 72 ++----- nginx/kobo-docker-scripts/nginx.conf | 4 +- .../templates/nginx_site_default.conf.tmpl | 20 +- .../templates/proxy_pass.conf.tmpl | 8 +- postgres/{shared => conf}/pg_hba.conf | 1 - postgres/conf/postgres.conf | 27 +++ postgres/entrypoint.sh | 9 +- postgres/primary/init_01_replication.sh | 12 -- postgres/primary/postgres.conf | 30 --- postgres/{ => scripts}/backup-to-disk.bash | 0 postgres/{ => scripts}/backup-to-s3.py | 0 .../clone_data_from_kc_to_kpi.sh | 0 .../drop_kpi_tables_in_kc.sh | 0 .../init_01_set_postgres_config.sh} | 11 +- .../init_02_firstrun.sh} | 0 .../{ => scripts}/toggle-backup-activation.sh | 59 ++---- postgres/{shared => scripts}/upsert_users.sh | 0 postgres/secondary/failover.sh | 41 ---- postgres/secondary/init_01_replication.sh | 42 ---- postgres/secondary/postgres.conf | 40 ---- postgres/secondary/recovery.conf | 3 - scripts/runtime_variables_kobocat.source.bash | 27 --- scripts/runtime_variables_kpi.source.bash | 29 --- scripts/wait_for_kpi.bash | 12 -- 39 files changed, 238 insertions(+), 588 deletions(-) rename docker-compose.backend.primary.override.yml.sample => docker-compose.backend.override.yml.sample (97%) delete mode 100644 docker-compose.backend.primary.yml delete mode 100644 docker-compose.backend.secondary.override.yml.sample delete mode 100644 docker-compose.backend.secondary.yml rename docker-compose.backend.template.yml => docker-compose.backend.yml (89%) create mode 100644 env/envfiles/django.txt delete mode 100644 env/envfiles/kobocat.txt delete mode 100644 env/envfiles/kpi.txt delete mode 100644 env/envfiles/nginx.txt rename postgres/{shared => conf}/pg_hba.conf (92%) create mode 100644 postgres/conf/postgres.conf delete mode 100644 postgres/primary/init_01_replication.sh delete mode 100644 postgres/primary/postgres.conf rename postgres/{ => scripts}/backup-to-disk.bash (100%) rename postgres/{ => scripts}/backup-to-s3.py (100%) rename postgres/{primary => scripts}/clone_data_from_kc_to_kpi.sh (100%) rename postgres/{primary => scripts}/drop_kpi_tables_in_kc.sh (100%) rename postgres/{shared/init_02_set_postgres_config.sh => scripts/init_01_set_postgres_config.sh} (59%) rename postgres/{shared/init_03_firstrun.sh => scripts/init_02_firstrun.sh} (100%) rename postgres/{ => scripts}/toggle-backup-activation.sh (57%) rename postgres/{shared => scripts}/upsert_users.sh (100%) delete mode 100644 postgres/secondary/failover.sh delete mode 100644 postgres/secondary/init_01_replication.sh delete mode 100644 postgres/secondary/postgres.conf delete mode 100644 postgres/secondary/recovery.conf delete mode 100644 scripts/runtime_variables_kobocat.source.bash delete mode 100644 scripts/runtime_variables_kpi.source.bash delete mode 100755 scripts/wait_for_kpi.bash diff --git a/docker-compose.backend.primary.override.yml.sample b/docker-compose.backend.override.yml.sample similarity index 97% rename from docker-compose.backend.primary.override.yml.sample rename to docker-compose.backend.override.yml.sample index 1e10663d..7ad4cf34 100644 --- a/docker-compose.backend.primary.override.yml.sample +++ b/docker-compose.backend.override.yml.sample @@ -1,5 +1,5 @@ -# Override for primary back-end server -version: '2.2' +# Override for back-end server +version: '3' services: diff --git a/docker-compose.backend.primary.yml b/docker-compose.backend.primary.yml deleted file mode 100644 index d73d5277..00000000 --- a/docker-compose.backend.primary.yml +++ /dev/null @@ -1,15 +0,0 @@ -# For primary back-end server -version: '2.2' - -services: - postgres: - extends: - file: docker-compose.backend.template.yml - service: postgres - environment: - - KOBO_POSTGRES_DB_SERVER_ROLE=primary - - mongo: - extends: - file: docker-compose.backend.template.yml - service: mongo diff --git a/docker-compose.backend.secondary.override.yml.sample b/docker-compose.backend.secondary.override.yml.sample deleted file mode 100644 index 77f6ac9b..00000000 --- a/docker-compose.backend.secondary.override.yml.sample +++ /dev/null @@ -1,28 +0,0 @@ -# Override for secondary back-end server - -services: - postgres: - volumes: - - ../kobo-env/postgres/secondary/postgres.conf:/kobo-docker-scripts/secondary/postgres.conf - ports: - - 5432:5432 - extra_hosts: - # Change `x.x.x.x` for the primary back-end server - - postgres.kobo.private:x.x.x.x - - primary.postgres.kobo.private:x.x.x.x - -# Uncomment the section below if you want to start redis containers from -# the secondary back-end server -# redis_main: -# extends: -# file: docker-compose.backend.template.yml -# service: redis_main -# ports: -# - 6379:6379 -# -# redis_cache: -# extends: -# file: docker-compose.backend.template.yml -# service: redis_cache -# ports: -# - 6380:6380 diff --git a/docker-compose.backend.secondary.yml b/docker-compose.backend.secondary.yml deleted file mode 100644 index 78bffb71..00000000 --- a/docker-compose.backend.secondary.yml +++ /dev/null @@ -1,10 +0,0 @@ -# For secondary back-end server -version: '2.2' - -services: - postgres: - extends: - file: docker-compose.backend.template.yml - service: postgres - environment: - - KOBO_POSTGRES_DB_SERVER_ROLE=secondary diff --git a/docker-compose.backend.template.yml b/docker-compose.backend.yml similarity index 89% rename from docker-compose.backend.template.yml rename to docker-compose.backend.yml index 49a6d625..32430b7c 100644 --- a/docker-compose.backend.template.yml +++ b/docker-compose.backend.yml @@ -1,14 +1,13 @@ -# Back-end containers template. -# The primary and secondary compose files extend this file +# Back-end containers -version: '2.2' +version: '3' services: postgres: image: postgis/postgis:14-3.2 hostname: postgres env_file: - - ../kobo-env/envfile.txt + - ../kobo-env/envfiles/domains.txt - ../kobo-env/envfiles/databases.txt - ../kobo-env/envfiles/aws.txt volumes: @@ -26,7 +25,7 @@ services: environment: - MONGO_DATA=/data/db env_file: - - ../kobo-env/envfile.txt + - ../kobo-env/envfiles/domains.txt - ../kobo-env/envfiles/databases.txt - ../kobo-env/envfiles/aws.txt volumes: @@ -39,9 +38,9 @@ services: stop_grace_period: 5m redis_main: - image: redis:6.2 + image: redis:7.2 env_file: - - ../kobo-env/envfile.txt + - ../kobo-env/envfiles/domains.txt - ../kobo-env/envfiles/databases.txt - ../kobo-env/envfiles/aws.txt environment: @@ -60,7 +59,7 @@ services: command: "bash /tmp/redis/entrypoint.sh" redis_cache: - image: redis:6.2 + image: redis:7.2 env_file: - ../kobo-env/envfiles/databases.txt environment: diff --git a/docker-compose.frontend.override.yml.sample b/docker-compose.frontend.override.yml.sample index 0723765b..72e36e4a 100644 --- a/docker-compose.frontend.override.yml.sample +++ b/docker-compose.frontend.override.yml.sample @@ -2,29 +2,8 @@ version: '3' services: - kobocat: - environment: - # change `ENKETO_PROTOCOL` to http if HTTPS is not used - - ENKETO_PROTOCOL=https - # `NGINX_PUBLIC_PORT` is the port used to access KoboToolbox (e.g. `https://kc.kobotoolbox.org:`) - - NGINX_PUBLIC_PORT=80 - # Uncomment the lines below to tweak uWSGI - #- KC_UWSGI_WORKERS_COUNT=2 - #- KC_UWSGI_CHEAPER_WORKERS_COUNT=1 - #- KC_UWSGI_MAX_REQUESTS=512 - #- KC_UWSGI_CHEAPER_RSS_LIMIT_SOFT=134217728 - #- KC_UWSGI_HARAKIRI=120 - #- KC_UWSGI_WORKER_RELOAD_MERCY=120 - networks: - kobo-be-network: - aliases: - - kobocat - - kobocat.docker.container - kpi: environment: - # `NGINX_PUBLIC_PORT` is the port used to access KoboToolbox (e.g. `https://kf.kobotoolbox.org:`) - - NGINX_PUBLIC_PORT=80 # Uncomment the lines below to tweak uWSGI #- KPI_UWSGI_WORKERS_COUNT=2 #- KPI_UWSGI_CHEAPER_WORKERS_COUNT=1 diff --git a/docker-compose.frontend.yml b/docker-compose.frontend.yml index e3d48aff..31b6a6a0 100644 --- a/docker-compose.frontend.yml +++ b/docker-compose.frontend.yml @@ -4,59 +4,19 @@ version: '3' services: - kobocat: - image: kobotoolbox/kobocat:2.024.09 - hostname: kobocat - env_file: - - ../kobo-env/envfile.txt - - ../kobo-env/envfiles/databases.txt - - ../kobo-env/envfiles/aws.txt - - ../kobo-env/envfiles/external_services.txt - - ../kobo-env/envfiles/kobocat.txt - - ../kobo-env/envfiles/nginx.txt - - ../kobo-env/envfiles/smtp.txt - sysctls: - - net.core.somaxconn=2048 - environment: - - ENKETO_PROTOCOL=https - - KPI_PORT=8000 - - KC_UWSGI_MAX_REQUESTS=512 - - KC_UWSGI_WORKERS_COUNT=2 - - KC_UWSGI_CHEAPER_RSS_LIMIT_SOFT=134217728 - - KC_UWSGI_CHEAPER_WORKERS_COUNT=1 - - KC_UWSGI_HARAKIRI=120 - - KC_UWSGI_WORKER_RELOAD_MERCY=120 - volumes: - - ./.vols/static/kobocat:/srv/static - - ./.vols/kobocat_media_uploads:/srv/src/kobocat/media - - ./backups/kobocat:/srv/backups - - ./log/kobocat:/srv/logs - - ./scripts/wait_for_mongo.bash:/srv/init/wait_for_mongo.bash:ro - - ./scripts/wait_for_postgres.bash:/srv/init/wait_for_postgres.bash:ro - - ./scripts/runtime_variables_kobocat.source.bash:/etc/profile.d/runtime_variables_kobocat.source.bash.sh:ro - restart: always - networks: - kobo-fe-network: - aliases: - - kobocat - - kobocat.docker.container - - kobocat.internal - kpi: image: kobotoolbox/kpi:2.024.09 hostname: kpi env_file: - - ../kobo-env/envfile.txt - - ../kobo-env/envfiles/databases.txt - ../kobo-env/envfiles/aws.txt + - ../kobo-env/envfiles/databases.txt + - ../kobo-env/envfiles/django.txt - ../kobo-env/envfiles/external_services.txt - - ../kobo-env/envfiles/kpi.txt - - ../kobo-env/envfiles/nginx.txt + - ../kobo-env/envfiles/domains.txt - ../kobo-env/envfiles/smtp.txt sysctls: - net.core.somaxconn=2048 environment: - - SYNC_KOBOCAT_XFORMS=False # Should be True on at least one frontend environment - KPI_UWSGI_MAX_REQUESTS=512 - KPI_UWSGI_WORKERS_COUNT=2 - KPI_UWSGI_CHEAPER_RSS_LIMIT_SOFT=134217728 @@ -68,28 +28,144 @@ services: - ./log/kpi:/srv/logs - ./scripts/wait_for_mongo.bash:/srv/init/wait_for_mongo.bash:ro - ./scripts/wait_for_postgres.bash:/srv/init/wait_for_postgres.bash:ro - - ./scripts/runtime_variables_kpi.source.bash:/etc/profile.d/runtime_variables_kpi.source.bash.sh:ro # Persistent storage for FileFields when S3 not used (e.g. exports, uploaded map layers) - ./.vols/kpi_media:/srv/src/kpi/media # Allow access to KoBoCAT media uploads within KPI - ./.vols/kobocat_media_uploads:/srv/src/kobocat/media - restart: always + restart: unless-stopped networks: kobo-fe-network: aliases: - - kpi - kpi.docker.container - kpi.internal + - kobocat.docker.container + - kobocat.internal + command: "bash -c exec /entrypoint.sh" +# worker: +# image: kobotoolbox/kpi:2.024.09 +# hostname: worker +# env_file: +# - ../kobo-env/envfiles/aws.txt +# - ../kobo-env/envfiles/databases.txt +# - ../kobo-env/envfiles/django.txt +# - ../kobo-env/envfiles/external_services.txt +# - ../kobo-env/envfiles/domains.txt +# - ../kobo-env/envfiles/smtp.txt +# volumes: +# - ./.vols/static/kpi:/srv/static +# - ./log/kpi:/srv/logs +# - ./scripts/wait_for_mongo.bash:/srv/init/wait_for_mongo.bash:ro +# - ./scripts/wait_for_postgres.bash:/srv/init/wait_for_postgres.bash:ro +# # Persistent storage for FileFields when S3 not used (e.g. exports, uploaded map layers) +# - ./.vols/kpi_media:/srv/src/kpi/media +# # Allow access to KoBoCAT media uploads within KPI +# - ./.vols/kobocat_media_uploads:/srv/src/kobocat/media +# restart: unless-stopped +# command: celery -A kobo worker --loglevel=info \ +# --hostname=kpi_main_worker@%h \ +# --logfile=/srv/logs/celery_kpi.log \ +# --pidfile=/tmp/celery_kpi.pid \ +# --queues=kpi_queue \ +# --exclude-queues=kpi_low_priority_queue,kobocat_queue \ +# --uid=$${UWSGI_USER} \ +# --gid=$${UWSGI_GROUP} \ +# --autoscale $${CELERY_AUTOSCALE_MIN},$${CELERY_AUTOSCALE_MAX} +# +# worker_low_priority: +# image: kobotoolbox/kpi:2.024.09 +# hostname: worker_low_priority +# env_file: +# - ../kobo-env/envfiles/aws.txt +# - ../kobo-env/envfiles/databases.txt +# - ../kobo-env/envfiles/django.txt +# - ../kobo-env/envfiles/external_services.txt +# - ../kobo-env/envfiles/domains.txt +# - ../kobo-env/envfiles/smtp.txt +# volumes: +# - ./.vols/static/kpi:/srv/static +# - ./log/kpi:/srv/logs +# - ./scripts/wait_for_mongo.bash:/srv/init/wait_for_mongo.bash:ro +# - ./scripts/wait_for_postgres.bash:/srv/init/wait_for_postgres.bash:ro +# # Persistent storage for FileFields when S3 not used (e.g. exports, uploaded map layers) +# - ./.vols/kpi_media:/srv/src/kpi/media +# # Allow access to KoBoCAT media uploads within KPI +# - ./.vols/kobocat_media_uploads:/srv/src/kobocat/media +# restart: unless-stopped +# command: celery -A kobo worker --loglevel=info \ +# --hostname=kpi_main_worker@%h \ +# --logfile=/srv/logs/celery_kpi.log \ +# --pidfile=/tmp/celery_kpi.pid \ +# --queues=kpi_queue \ +# --exclude-queues=kpi_low_priority_queue,kobocat_queue \ +# --uid=$${UWSGI_USER} \ +# --gid=$${UWSGI_GROUP} \ +# --autoscale $${CELERY_AUTOSCALE_MIN},$${CELERY_AUTOSCALE_MAX} +# +# worker_kobocat: +# image: kobotoolbox/kpi:2.024.09 +# hostname: worker_kobocat +# env_file: +# - ../kobo-env/envfiles/aws.txt +# - ../kobo-env/envfiles/databases.txt +# - ../kobo-env/envfiles/django.txt +# - ../kobo-env/envfiles/external_services.txt +# - ../kobo-env/envfiles/domains.txt +# - ../kobo-env/envfiles/smtp.txt +# volumes: +# - ./.vols/static/kpi:/srv/static +# - ./log/kpi:/srv/logs +# - ./scripts/wait_for_mongo.bash:/srv/init/wait_for_mongo.bash:ro +# - ./scripts/wait_for_postgres.bash:/srv/init/wait_for_postgres.bash:ro +# # Persistent storage for FileFields when S3 not used (e.g. exports, uploaded map layers) +# - ./.vols/kpi_media:/srv/src/kpi/media +# # Allow access to KoBoCAT media uploads within KPI +# - ./.vols/kobocat_media_uploads:/srv/src/kobocat/media +# restart: unless-stopped +# command: celery -A kobo worker --loglevel=info \ +# --hostname=kobocat_main_worker@%h \ +# --logfile=/srv/logs/celery_kobocat.log \ +# --pidfile=/tmp/celery_kobocat.pid \ +# --queues=kobocat_queue \ +# --exclude-queues=kpi_low_priority_queue,kpi_queue \ +# --uid=$${UWSGI_USER} \ +# --gid=$${UWSGI_GROUP} \ +# --autoscale $${CELERY_AUTOSCALE_MIN},$${CELERY_AUTOSCALE_MAX} +# +# beat: +# image: kobotoolbox/kpi:2.024.09 +# hostname: beat +# env_file: +# - ../kobo-env/envfiles/aws.txt +# - ../kobo-env/envfiles/databases.txt +# - ../kobo-env/envfiles/django.txt +# - ../kobo-env/envfiles/external_services.txt +# - ../kobo-env/envfiles/domains.txt +# - ../kobo-env/envfiles/smtp.txt +# volumes: +# - ./.vols/static/kpi:/srv/static +# - ./log/django:/srv/logs +# - ./scripts/wait_for_mongo.bash:/srv/init/wait_for_mongo.bash:ro +# - ./scripts/wait_for_postgres.bash:/srv/init/wait_for_postgres.bash:ro +# # Persistent storage for FileFields when S3 not used (e.g. exports, uploaded map layers) +# - ./.vols/kpi_media:/srv/src/kpi/media +# # Allow access to KoBoCAT media uploads within KPI +# - ./.vols/kobocat_media_uploads:/srv/src/kobocat/media +# restart: unless-stopped +# command: celery -A kobo beat --loglevel=info \ +# --logfile=/srv/logs/celery_beat.log \ +# --pidfile=/tmp/celery_beat.pid \ +# --scheduler django_celery_beat.schedulers:DatabaseScheduler \ +# --uid=$${UWSGI_USER} \ +# --gid=$${UWSGI_GROUP} +# nginx: - image: nginx:1.21 + image: nginx:1.23 hostname: nginx env_file: - - ../kobo-env/envfile.txt + - ../kobo-env/envfiles/domains.txt - ../kobo-env/envfiles/databases.txt - - ../kobo-env/envfiles/nginx.txt - - ../kobo-env/envfiles/kobocat.txt - - ../kobo-env/envfiles/kpi.txt + - ../kobo-env/envfiles/django.txt environment: - UWSGI_PASS_TIMEOUT=130 - TEMPLATED_VAR_REFS=$${PUBLIC_REQUEST_SCHEME} $${INTERNAL_DOMAIN_NAME} $${PUBLIC_DOMAIN_NAME} $${KOBOFORM_PUBLIC_SUBDOMAIN} $${KOBOCAT_PUBLIC_SUBDOMAIN} $${ENKETO_EXPRESS_PUBLIC_SUBDOMAIN} @@ -100,25 +176,26 @@ services: - ./.vols/kpi_media/__public:/srv/kpi_media/__public:ro - ./nginx/docker-entrypoint.d/30-init-kobo-nginx.sh:/docker-entrypoint.d/30-init-kobo-nginx.sh - ./nginx/kobo-docker-scripts/:/kobo-docker-scripts - restart: always + restart: unless-stopped networks: kobo-fe-network: aliases: - - nginx - nginx.internal + - nginx.docker.container enketo_express: image: kobotoolbox/enketo-express-extra-widgets:6.2.2 + hostname: enketo_express env_file: - - ../kobo-env/envfile.txt - restart: always + - ../kobo-env/envfiles/domains.txt + restart: unless-stopped volumes: - ../kobo-env/enketo_express/config.json:/srv/src/enketo_express/config/config.json networks: kobo-fe-network: aliases: - - enketo_express - enketo_express.internal + - enketo_express.docker.container networks: kobo-fe-network: diff --git a/docker-compose.maintenance.yml b/docker-compose.maintenance.yml index 8e5d8afd..bdcbcb21 100644 --- a/docker-compose.maintenance.yml +++ b/docker-compose.maintenance.yml @@ -5,10 +5,10 @@ services: maintenance: # set to match `docker-compose.frontend.yml` and avoid pulling different image - image: nginx:1.21 + image: nginx:1.23 hostname: maintenance env_file: - - ../kobo-env/envfile.txt + - ../kobo-env/envfiles/domains.txt volumes: - ./log/nginx:/var/log/nginx - ./nginx/docker-entrypoint.d/30-init-maintenance-nginx.sh:/docker-entrypoint.d/30-init-kobo-nginx.sh diff --git a/env/envfiles/aws.txt b/env/envfiles/aws.txt index 27b059c4..5b5622f3 100644 --- a/env/envfiles/aws.txt +++ b/env/envfiles/aws.txt @@ -6,20 +6,6 @@ #AWS_SECRET_ACCESS_KEY= -################### -# E-mail settings # -################### - -# NOTE: These and the settings from `envfiles/smtp.txt` are mutually exclusive; do not use both. -# In order to use SES to send e-mails, you must already have SES set up on AWS, and the owner of your `AWS_ACCESS_KEY_ID` must have the appropriate SES permissions. -#EMAIL_BACKEND=django_ses.SESBackend -# E.g. `us-east-1` -#AWS_SES_REGION_NAME= -# E.g. `email.us-east-1.amazonaws.com`. -#AWS_SES_REGION_ENDPOINT= -#DEFAULT_FROM_EMAIL= - - #################### # Storage settings # #################### diff --git a/env/envfiles/databases.txt b/env/envfiles/databases.txt index f85c5c77..0319f9ea 100644 --- a/env/envfiles/databases.txt +++ b/env/envfiles/databases.txt @@ -36,12 +36,6 @@ KPI_POSTGRES_DB=koboform KC_DATABASE_URL=postgis://kobo:kobo@postgres.domain.name:5432/kobotoolbox KPI_DATABASE_URL=postgis://kobo:kobo@postgres.domain.name:5432/kobotoolbox -# Replication. Password is mandatory -KOBO_POSTGRES_REPLICATION_USER=kobo_replication -KOBO_POSTGRES_REPLICATION_PASSWORD= - -KOBO_POSTGRES_PRIMARY_ENDPOINT=primary.postgres.domain.name - # Default Postgres backup schedule is weekly at 02:00 AM UTC on Sunday. #POSTGRES_BACKUP_SCHEDULE=0 2 * * 0 diff --git a/env/envfiles/django.txt b/env/envfiles/django.txt new file mode 100644 index 00000000..bfe895e1 --- /dev/null +++ b/env/envfiles/django.txt @@ -0,0 +1,10 @@ +DJANGO_DEBUG=False +TEMPLATE_DEBUG=False +USE_X_FORWARDED_HOST=False + +CELERY_BROKER_URL=redis://:kobo@redis-main.domain.name:6379/1 + +CELERY_AUTOSCALE_MIN=2 +CELERY_AUTOSCALE_MAX=6 + +${USE_SERVICE_ACCOUNT_WHITELISTED_HOSTS}SERVICE_ACCOUNT_WHITELISTED_HOSTS=${KOBOFORM_SUBDOMAIN}.${INTERNAL_DOMAIN_NAME} diff --git a/env/envfiles/external_services.txt b/env/envfiles/external_services.txt index fec6615b..f0192dee 100644 --- a/env/envfiles/external_services.txt +++ b/env/envfiles/external_services.txt @@ -3,6 +3,5 @@ ############################################################################ GOOGLE_ANALYTICS_TOKEN= -KOBOCAT_RAVEN_DSN= -KPI_RAVEN_DSN= -KPI_RAVEN_JS_DSN= +SENTRY_DSN= +SENTRY_JS_DSN= diff --git a/env/envfiles/kobocat.txt b/env/envfiles/kobocat.txt deleted file mode 100644 index 69298641..00000000 --- a/env/envfiles/kobocat.txt +++ /dev/null @@ -1,26 +0,0 @@ -KOBOCAT_DJANGO_DEBUG=False -TEMPLATE_DEBUG=False -USE_X_FORWARDED_HOST=False - -DJANGO_SETTINGS_MODULE=onadata.settings.kc_environ -ENKETO_VERSION=Express - -KOBOCAT_BROKER_URL=redis://:kobo@redis-main.domain.name:6379/2 -KOBOCAT_CELERY_LOG_FILE=/srv/logs/celery.log - -#ENKETO_OFFLINE_SURVEYS=True - -# Mongo values come from mongo.txt - -#KOBOCAT_MEDIA_URL=media -#KOBOCAT_ROOT_URI_PREFIX= -#KOBOCAT_TEMPLATES_PATH=../kobocat-template -#KOBOFORM_SERVER=localhost -#KOBOFORM_SERVER_PORT=8000 -#KOBOFORM_SERVER_PROTOCOL=http -#KOBOCAT_DEFAULT_FILE_STORAGE= -#EMAIL_FILE_PATH=./emails - -# Dev: One or more mappings from PyDev remote debugging machine file paths to `kobocat` container -# file paths (see https://github.com/kobotoolbox/kobocat/blob/master/docker/setup_pydev.bash). -#KOBOCAT_PATH_FROM_ECLIPSE_TO_PYTHON_PAIRS=~/devel/kobocat -> /srv/src/kobocat | ~/.virtualenvs/kobocat/lib/python2.7/site-packages -> /usr/local/lib/python2.7/dist-packages diff --git a/env/envfiles/kpi.txt b/env/envfiles/kpi.txt deleted file mode 100644 index 9053015f..00000000 --- a/env/envfiles/kpi.txt +++ /dev/null @@ -1,22 +0,0 @@ -KPI_DJANGO_DEBUG=False -TEMPLATE_DEBUG=False -USE_X_FORWARDED_HOST=False - - -ENKETO_VERSION=Express -KPI_PREFIX=/ -KPI_BROKER_URL=redis://:kobo@redis-main.domain.name:6379/1 - -DJANGO_LANGUAGE_CODES=en fr es ar zh-hans hi ku -#DKOBO_PREFIX=False -#KOBO_SURVEY_PREVIEW_EXPIRATION=24 -#SKIP_CELERY=False -#EMAIL_FILE_PATH=./emails -#SYNC_KOBOCAT_XFORMS_PERIOD_MINUTES=30 -#KPI_UWSGI_PROCESS_COUNT=2 -#KOBO_SUPPORT_URL=http://support.kobotoolbox.org/ - - -# Dev: One or more mappings from PyDev remote debugging machine file paths to `kpi` container file -# paths (see https://github.com/kobotoolbox/kpi/blob/master/docker/setup_pydev.bash). -#KPI_PATH_FROM_ECLIPSE_TO_PYTHON_PAIRS=~/devel/kpi -> /srv/src/kpi | ~/devel/formpack/src/formpack -> /usr/local/lib/python2.7/dist-packages/formpack diff --git a/env/envfiles/nginx.txt b/env/envfiles/nginx.txt deleted file mode 100644 index 5f51200d..00000000 --- a/env/envfiles/nginx.txt +++ /dev/null @@ -1,10 +0,0 @@ -# Options for the following are "uWSGI" or "runserver_plus" (for debugging). -KPI_WEB_SERVER=uWSGI -KOBOCAT_WEB_SERVER=uWSGI - -# Options for the following are "Nginx" or "Django". -# NOTE: In order to serve static files from Django, the corresponding -# `..._DJANGO_DEBUG` environment variable must be set to "True" -# in `envfiles/kobocat.txt` and/or `envfiles/kpi.txt`. -KOBOCAT_STATIC_FILES_SERVER=Nginx -KPI_STATIC_FILES_SERVER=Nginx diff --git a/nginx/docker-entrypoint.d/30-init-kobo-nginx.sh b/nginx/docker-entrypoint.d/30-init-kobo-nginx.sh index 1487cf25..fff8b123 100755 --- a/nginx/docker-entrypoint.d/30-init-kobo-nginx.sh +++ b/nginx/docker-entrypoint.d/30-init-kobo-nginx.sh @@ -13,68 +13,28 @@ cp ${KOBO_DOCKER_SCRIPTS_DIR}/nginx.conf /etc/nginx/nginx.conf echo "Clearing out any default configurations" rm -rf /etc/nginx/conf.d/* -templated_var_refs="${TEMPLATED_VAR_REFS}" -declare -A container_ports -container_ports=( ['kpi']='8000' ['kobocat']='8001' ) +TEMPLATED_VAR_REFS="${TEMPLATED_VAR_REFS} \${NGINX_PUBLIC_PORT}" -if [ "${NGINX_PUBLIC_PORT:-80}" != "80" ]; then - export container_public_port=":${NGINX_PUBLIC_PORT}" -else - export container_public_port="" -fi -templated_var_refs+=" \${container_public_port}" - -for container_name in "${!container_ports[@]}"; do - export container_name - export container_port="${container_ports[${container_name}]}" - - # Set up proxying to apps. - web_server_varname="${container_name^^}_WEB_SERVER" # E.g. `kpi_WEB_SERVER`. - web_server="${!web_server_varname}" - if [[ "${web_server^^}" != 'UWSGI' ]] ; then - echo "Proxying directly (debug) to \`${container_name}\` without uWSGI." - - if [ "${container_name}" == "kpi" ]; then - export container_x_forwarded_host="${KOBOFORM_PUBLIC_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}" - else - export container_x_forwarded_host="${KOBOCAT_PUBLIC_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}" - fi +if [[ "${WSGI}" != 'UWSGI' ]] ; then + echo "Proxying directly (debug) to Django without uWSGI." - # Create a `proxy_pass` configuration for this container. - cat ${KOBO_DOCKER_SCRIPTS_DIR}/templates/proxy_pass.conf.tmpl \ - | envsubst '${container_name} ${container_port} ${container_public_port} ${container_x_forwarded_host}' \ - > ${INCLUDES_DIR}/${container_name}_proxy_pass.conf - - # Prepare to include the generated `proxy_pass` config. and no `uwsgi_pass` config. - include_proxy_pass="include ${INCLUDES_DIR}/${container_name}_proxy_pass.conf;" - include_uwsgi_pass='' - else - echo "Proxying to \`${container_name}\` through uWSGI." - - # Create a `uwsgi_pass` configuration for this container. - cat ${KOBO_DOCKER_SCRIPTS_DIR}/templates/uwsgi_pass.conf.tmpl \ - | envsubst '${container_name} ${container_port} ${UWSGI_PASS_TIMEOUT}' \ - > ${INCLUDES_DIR}/${container_name}_uwsgi_pass.conf - - # Prepare to include the generated `uwsgi_pass` config. and no `proxy_pass` config. - include_proxy_pass='' - include_uwsgi_pass="include ${INCLUDES_DIR}/${container_name}_uwsgi_pass.conf;" - fi - include_proxy_pass_varname="${container_name}_include_proxy_pass" - export ${include_proxy_pass_varname}="${include_proxy_pass}" - include_uwsgi_pass_varname="${container_name}_include_uwsgi_pass" - export ${include_uwsgi_pass_varname}="${include_uwsgi_pass}" - - # Register the include directive variables (e.g. `kpi_include_proxy_pass` and `kpi_include_uwsgi_pass`) - # for template substitution. - templated_var_refs+=" \${${include_proxy_pass_varname}} \${${include_uwsgi_pass_varname}}" + # Create a `proxy_pass` configuration for this container. + cat ${KOBO_DOCKER_SCRIPTS_DIR}/templates/proxy_pass.conf.tmpl \ + | envsubst '${KOBOFORM_PUBLIC_SUBDOMAIN} ${PUBLIC_DOMAIN_NAME} ${NGINX_PUBLIC_PORT}' \ + > ${INCLUDES_DIR}/proxy_pass.conf +else + echo "Proxying to Django through uWSGI." -done + # Create a `uwsgi_pass` configuration for this container. + cat ${KOBO_DOCKER_SCRIPTS_DIR}/templates/uwsgi_pass.conf.tmpl \ + | envsubst '${UWSGI_PASS_TIMEOUT}' \ + > ${INCLUDES_DIR}/uwsgi_pass.conf +fi # Do environment variable substitutions and activate the resulting config. file. -cat ${KOBO_DOCKER_SCRIPTS_DIR}/templates/nginx_site_default.conf.tmpl | envsubst "${templated_var_refs}" > /etc/nginx/conf.d/default.conf +cat ${KOBO_DOCKER_SCRIPTS_DIR}/templates/nginx_site_default.conf.tmpl | envsubst "${TEMPLATED_VAR_REFS}" > /etc/nginx/conf.d/default.conf # Copy includes files -cat ${KOBO_DOCKER_SCRIPTS_DIR}/templates/include.https_redirection.conf.tmpl | envsubst "${templated_var_refs}" > /etc/nginx/includes/https_redirection.conf +cat ${KOBO_DOCKER_SCRIPTS_DIR}/templates/include.https_redirection.conf.tmpl | envsubst "${TEMPLATED_VAR_REFS}" > /etc/nginx/includes/https_redirection.conf cp ${KOBO_DOCKER_SCRIPTS_DIR}/include.server_directive_common.conf /etc/nginx/includes/server_directive_common.conf cp ${KOBO_DOCKER_SCRIPTS_DIR}/include.protected_directive.conf /etc/nginx/includes/protected_directive.conf diff --git a/nginx/kobo-docker-scripts/nginx.conf b/nginx/kobo-docker-scripts/nginx.conf index 89d2bd06..cf3e3d7f 100644 --- a/nginx/kobo-docker-scripts/nginx.conf +++ b/nginx/kobo-docker-scripts/nginx.conf @@ -22,7 +22,7 @@ http { types_hash_max_size 2048; # server_tokens off; - server_names_hash_bucket_size 64; + server_names_hash_bucket_size 128; # server_name_in_redirect off; include /etc/nginx/mime.types; @@ -77,5 +77,3 @@ http { include /etc/nginx/conf.d/*.conf; } - - diff --git a/nginx/kobo-docker-scripts/templates/nginx_site_default.conf.tmpl b/nginx/kobo-docker-scripts/templates/nginx_site_default.conf.tmpl index e927de90..b0db9ac8 100644 --- a/nginx/kobo-docker-scripts/templates/nginx_site_default.conf.tmpl +++ b/nginx/kobo-docker-scripts/templates/nginx_site_default.conf.tmpl @@ -29,7 +29,7 @@ server { } } -# KoBoCAT HTTP. +# KoboCAT HTTP. server { listen 80; server_name ${KOBOCAT_PUBLIC_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME} ${KOBOCAT_PUBLIC_SUBDOMAIN}.${INTERNAL_DOMAIN_NAME}; @@ -54,7 +54,7 @@ server { include /etc/nginx/includes/protected_directive.conf; } -# KoBoForm public HTTP +# KoboForm public HTTP server { listen 80; server_name ${KOBOFORM_PUBLIC_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME} ${KOBOFORM_PUBLIC_SUBDOMAIN}.${INTERNAL_DOMAIN_NAME}; @@ -77,14 +77,9 @@ server { alias /srv/kpi_media/__public; } - error_page 418 = /static/html/Offline.html; - location / { - # Proxy through uWSGI. - ${kpi_include_uwsgi_pass} - - # Debug proxying directly to container. - ${kpi_include_proxy_pass} + # Proxy through uWSGI or Django webserver + include /etc/nginx/includes/proxy_pass.conf; } include /etc/nginx/includes/protected_directive.conf; @@ -94,8 +89,7 @@ server { listen 80; server_name ${ENKETO_EXPRESS_PUBLIC_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME} ${ENKETO_EXPRESS_PUBLIC_SUBDOMAIN}.${INTERNAL_DOMAIN_NAME}; - resolver 8.8.4.4 8.8.8.8 valid=300s; - resolver_timeout 10s; + resolver 127.0.0.11 ipv6=off valid=1s; include /etc/nginx/includes/server_directive_common.conf; @@ -103,7 +97,7 @@ server { # add_header X-Frame-Options DENY; add_header X-Content-Type-Options nosniff; - proxy_set_header Host $host${container_public_port}; + proxy_set_header Host $host${NGINX_PUBLIC_PORT}; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto; # Needs ELB protocol @@ -112,7 +106,7 @@ server { error_log /var/log/nginx/enketo_express.error.log; location / { - proxy_pass http://enketo_express:8005/; + proxy_pass http://enketo_express:8005; proxy_redirect off; } } diff --git a/nginx/kobo-docker-scripts/templates/proxy_pass.conf.tmpl b/nginx/kobo-docker-scripts/templates/proxy_pass.conf.tmpl index 3213c911..d9b8c1a1 100644 --- a/nginx/kobo-docker-scripts/templates/proxy_pass.conf.tmpl +++ b/nginx/kobo-docker-scripts/templates/proxy_pass.conf.tmpl @@ -1,12 +1,16 @@ # `envsubst` template. # Context: location -proxy_pass http://${container_name}:${container_port}; +# Nginx DNS resolution issue fix +# See https://www.jethrocarr.com/2013/11/02/nginx-reverse-proxies-and-dns-resolution/ +# 127.0.0.11 is internal docker DNS server (see `cat /etc/resolv.conf`) +resolver 127.0.0.11 ipv6=off valid=1s; +proxy_pass http://kpi:8000; proxy_set_header Host $host:$proxy_port; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $remote_addr; proxy_set_header X-Forwarded-Proto $scheme; -proxy_set_header X-Forwarded-Host ${container_x_forwarded_host}${container_public_port}; +proxy_set_header X-Forwarded-Host ${KOBOFORM_PUBLIC_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}${NGINX_PUBLIC_PORT}; proxy_redirect off; # Add ranges support. diff --git a/postgres/shared/pg_hba.conf b/postgres/conf/pg_hba.conf similarity index 92% rename from postgres/shared/pg_hba.conf rename to postgres/conf/pg_hba.conf index af2cea2f..0f00db16 100644 --- a/postgres/shared/pg_hba.conf +++ b/postgres/conf/pg_hba.conf @@ -13,4 +13,3 @@ host all all ::1/128 trust #host replication postgres ::1/128 trust host all all all md5 -host replication KOBO_POSTGRES_REPLICATION_USER all md5 diff --git a/postgres/conf/postgres.conf b/postgres/conf/postgres.conf new file mode 100644 index 00000000..22dc1a98 --- /dev/null +++ b/postgres/conf/postgres.conf @@ -0,0 +1,27 @@ +# Generated by PGConfig 3.1.4 (1fe6d98dedcaad1d0a114617cfd08b4fed1d8a01) +# https://api.pgconfig.org/v1/tuning/get-config?format=conf&&log_format=csvlog&max_connections=100&pg_version=14&environment_name=WEB&total_ram=4GB&cpus=2&drive_type=SSD&arch=x86-64&os_type=linux + +# Memory Configuration +shared_buffers = 1GB +effective_cache_size = 3GB +work_mem = 10MB +maintenance_work_mem = 205MB + +# Checkpoint Related Configuration +min_wal_size = 2GB +max_wal_size = 3GB +checkpoint_completion_target = 0.9 +wal_buffers = -1 + +# Network Related Configuration +listen_addresses = '*' +max_connections = 100 + +# Storage Configuration +random_page_cost = 1.1 +effective_io_concurrency = 200 + +# Worker Processes Configuration +max_worker_processes = 8 +max_parallel_workers_per_gather = 2 +max_parallel_workers = 2 diff --git a/postgres/entrypoint.sh b/postgres/entrypoint.sh index 6e80520f..0d9928dd 100644 --- a/postgres/entrypoint.sh +++ b/postgres/entrypoint.sh @@ -12,8 +12,7 @@ export POSTGRES_LOGS_DIR=/srv/logs export KOBO_DOCKER_SCRIPTS_DIR=/kobo-docker-scripts echo "Copying init scripts ..." -cp $KOBO_DOCKER_SCRIPTS_DIR/shared/init_* /docker-entrypoint-initdb.d/ -cp $KOBO_DOCKER_SCRIPTS_DIR/$KOBO_POSTGRES_DB_SERVER_ROLE/init_* /docker-entrypoint-initdb.d/ +cp $KOBO_DOCKER_SCRIPTS_DIR/scripts/init_* /docker-entrypoint-initdb.d/ if [ ! -d $POSTGRES_LOGS_DIR ]; then mkdir -p $POSTGRES_LOGS_DIR @@ -30,7 +29,7 @@ chown -R postgres:postgres $POSTGRES_BACKUPS_DIR # if file exists. Container has already boot once if [ -f "$POSTGRES_DATA_DIR/kobo_first_run" ]; then # Recreate config first - bash $KOBO_DOCKER_SCRIPTS_DIR/shared/init_02_set_postgres_config.sh + bash $KOBO_DOCKER_SCRIPTS_DIR/scripts/init_01_set_postgres_config.sh if [ "$KOBO_POSTGRES_DB_SERVER_ROLE" == "primary" ]; then # Start server locally. @@ -39,7 +38,7 @@ if [ -f "$POSTGRES_DATA_DIR/kobo_first_run" ]; then sleep 1 done # Update users if needed - bash $KOBO_DOCKER_SCRIPTS_DIR/shared/upsert_users.sh + bash $KOBO_DOCKER_SCRIPTS_DIR/scripts/upsert_users.sh # Update PostGIS extension update-postgis.sh # Stop server @@ -54,7 +53,7 @@ fi # Send backup installation process in background to avoid blocking PostgreSQL startup -bash $KOBO_DOCKER_SCRIPTS_DIR/toggle-backup-activation.sh & +bash $KOBO_DOCKER_SCRIPTS_DIR/scripts/toggle-backup-activation.sh & echo "Launching official entrypoint..." # `exec` here is important to pass signals to the database server process; diff --git a/postgres/primary/init_01_replication.sh b/postgres/primary/init_01_replication.sh deleted file mode 100644 index 0c713613..00000000 --- a/postgres/primary/init_01_replication.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - - -# Perform all actions as $POSTGRES_USER -export PGUSER="$POSTGRES_USER" - -echo "Creating replication user..." -psql -d "${POSTGRES_DB}" -c " - DROP ROLE IF EXISTS ${KOBO_POSTGRES_REPLICATION_USER}; - CREATE USER ${KOBO_POSTGRES_REPLICATION_USER} REPLICATION LOGIN CONNECTION LIMIT 2 ENCRYPTED PASSWORD '${KOBO_POSTGRES_REPLICATION_PASSWORD}'; -" -echo "Replication user created!" diff --git a/postgres/primary/postgres.conf b/postgres/primary/postgres.conf deleted file mode 100644 index 551be4e7..00000000 --- a/postgres/primary/postgres.conf +++ /dev/null @@ -1,30 +0,0 @@ -##################################################################################### -# PRIMARY SPECIFIC -# If file must be appended to shared/postgres.conf -##################################################################################### -#------------------------------------------------------------------------------------ -# TUNING -#------------------------------------------------------------------------------------ -# These settings are based on server configuration -# https://www.pgconfig.org/#/tuning -# DB Version: 9.5 -# OS Type: linux -# DB Type: Mixed -# Hard-drive: HDD -# Total Memory (RAM): 2GB - -# Memory Configuration -shared_buffers = 512MB -effective_cache_size = 2GB -work_mem = 10MB -maintenance_work_mem = 128MB - -# Checkpoint Related Configuration -min_wal_size = 512MB -max_wal_size = 2GB -checkpoint_completion_target = 0.9 -wal_buffers = 15MB - -# Network Related Configuration -listen_addresses = '*' -max_connections = 100 diff --git a/postgres/backup-to-disk.bash b/postgres/scripts/backup-to-disk.bash similarity index 100% rename from postgres/backup-to-disk.bash rename to postgres/scripts/backup-to-disk.bash diff --git a/postgres/backup-to-s3.py b/postgres/scripts/backup-to-s3.py similarity index 100% rename from postgres/backup-to-s3.py rename to postgres/scripts/backup-to-s3.py diff --git a/postgres/primary/clone_data_from_kc_to_kpi.sh b/postgres/scripts/clone_data_from_kc_to_kpi.sh similarity index 100% rename from postgres/primary/clone_data_from_kc_to_kpi.sh rename to postgres/scripts/clone_data_from_kc_to_kpi.sh diff --git a/postgres/primary/drop_kpi_tables_in_kc.sh b/postgres/scripts/drop_kpi_tables_in_kc.sh similarity index 100% rename from postgres/primary/drop_kpi_tables_in_kc.sh rename to postgres/scripts/drop_kpi_tables_in_kc.sh diff --git a/postgres/shared/init_02_set_postgres_config.sh b/postgres/scripts/init_01_set_postgres_config.sh similarity index 59% rename from postgres/shared/init_02_set_postgres_config.sh rename to postgres/scripts/init_01_set_postgres_config.sh index b511c2e2..8e78c8d7 100644 --- a/postgres/shared/init_02_set_postgres_config.sh +++ b/postgres/scripts/init_01_set_postgres_config.sh @@ -6,11 +6,11 @@ if [ ! -f "$POSTGRES_CONFIG_FILE.orig" ]; then fi echo "Applying new configuration..." -cp $KOBO_DOCKER_SCRIPTS_DIR/shared/postgres.conf $POSTGRES_CONFIG_FILE +cp $KOBO_DOCKER_SCRIPTS_DIR/conf/postgres.conf $POSTGRES_CONFIG_FILE -if [ -f "$KOBO_DOCKER_SCRIPTS_DIR/$KOBO_POSTGRES_DB_SERVER_ROLE/postgres.conf" ]; then +if [ -f "$KOBO_DOCKER_SCRIPTS_DIR/conf/postgres.conf" ]; then echo "Appending role specific configuration..." - cat $KOBO_DOCKER_SCRIPTS_DIR/$KOBO_POSTGRES_DB_SERVER_ROLE/postgres.conf >> $POSTGRES_CONFIG_FILE + cat $KOBO_DOCKER_SCRIPTS_DIR/conf/postgres.conf >> $POSTGRES_CONFIG_FILE if grep -q "\$PGDATA" "$POSTGRES_CONFIG_FILE"; then sed -i "s#\$PGDATA#"$PGDATA"#g" $POSTGRES_CONFIG_FILE @@ -23,7 +23,4 @@ if [ ! -f "$POSTGRES_CLIENT_AUTH_FILE.orig" ]; then fi echo "Applying new client authentication configuration file..." -cp $KOBO_DOCKER_SCRIPTS_DIR/shared/pg_hba.conf "$POSTGRES_CLIENT_AUTH_FILE" - -echo "Creating hg_hba config file..." -sed -i "s/KOBO_POSTGRES_REPLICATION_USER/${KOBO_POSTGRES_REPLICATION_USER//\"/}/g" "$POSTGRES_CLIENT_AUTH_FILE" +cp $KOBO_DOCKER_SCRIPTS_DIR/config/pg_hba.conf "$POSTGRES_CLIENT_AUTH_FILE" diff --git a/postgres/shared/init_03_firstrun.sh b/postgres/scripts/init_02_firstrun.sh similarity index 100% rename from postgres/shared/init_03_firstrun.sh rename to postgres/scripts/init_02_firstrun.sh diff --git a/postgres/toggle-backup-activation.sh b/postgres/scripts/toggle-backup-activation.sh similarity index 57% rename from postgres/toggle-backup-activation.sh rename to postgres/scripts/toggle-backup-activation.sh index c8a6e976..ca0606c6 100644 --- a/postgres/toggle-backup-activation.sh +++ b/postgres/scripts/toggle-backup-activation.sh @@ -64,50 +64,25 @@ else if [[ ${USE_S3} -eq "$TRUE" ]]; then apt-get install -y curl python3-pip libffi-dev --quiet=2 > /dev/null - # Update pip to latest version compatible with Python 3.5 - curl https://bootstrap.pypa.io/pip/3.5/get-pip.py -o /tmp/get-pip.py - python3 /tmp/get-pip.py - if [[ ${USE_WAL_E} -eq "$TRUE" ]]; then - echo "Installing envdir and WAL-E for PostgreSQL backup on S3..." - apt-get install -y libevent-dev daemontools lzop pv --quiet=2 > /dev/null - python3 -m pip install --quiet wal-e aws + echo "Installing virtualenv for PostgreSQL backup on S3..." + python3 -m pip install --upgrade --quiet virtualenv + counter=1 + max_retries=3 + # Under certain circumstances a race condition occurs. Virtualenv creation + # fails because python cannot find `wheel` package folder + # e.g. `FileNotFoundError: [Errno 2] No such file or directory: '/root/.local/share/virtualenv/wheel/3.5/embed/1/wheel.json'` + until $(virtualenv --quiet -p /usr/bin/python3 /tmp/backup-virtualenv > /dev/null) + do + [[ "$counter" -eq "$max_retries" ]] && echo "Virtual environment creation failed!" && exit 1 + ((counter++)) + done + . /tmp/backup-virtualenv/bin/activate + pip install --quiet humanize smart-open==1.7.1 + pip install --quiet boto + deactivate - # Find EC2 region - EC2_AVAIL_ZONE=$(/usr/bin/curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone) #AWS zone - EC2_REGION=$(echo $EC2_AVAIL_ZONE | sed 's/[a-z]$//') #AWS region - - # Add envdir for Wal-e backup - mkdir -p ${POSTGRES_DATA_DIR}/wal-e.d/env - echo "${AWS_SECRET_ACCESS_KEY}" > ${POSTGRES_DATA_DIR}/wal-e.d/env/AWS_SECRET_ACCESS_KEY - echo "${AWS_ACCESS_KEY_ID}" > ${POSTGRES_DATA_DIR}/wal-e.d/env/AWS_ACCESS_KEY_ID - echo "s3://${BACKUP_AWS_STORAGE_BUCKET_NAME}/postgres/" > ${POSTGRES_DATA_DIR}/wal-e.d/env/WALE_S3_PREFIX - echo "${EC2_REGION}" > ${POSTGRES_DATA_DIR}/wal-e.d/env/AWS_REGION - chown -R postgres:postgres ${POSTGRES_DATA_DIR}/wal-e.d - - # Add crontab Wal-e backup to S3 - CRON_CMD="${POSTGRES_BACKUP_SCHEDULE} postgres envdir ${POSTGRES_DATA_DIR}/wal-e.d/env/ /usr/local/bin/wal-e backup-push ${POSTGRES_DATA_DIR} > /srv/logs/backup.log 2>&1" - - else - echo "Installing virtualenv for PostgreSQL backup on S3..." - python3 -m pip install --upgrade --quiet virtualenv - counter=1 - max_retries=3 - # Under certain circumstances a race condition occurs. Virtualenv creation - # fails because python cannot find `wheel` package folder - # e.g. `FileNotFoundError: [Errno 2] No such file or directory: '/root/.local/share/virtualenv/wheel/3.5/embed/1/wheel.json'` - until $(virtualenv --quiet -p /usr/bin/python3 /tmp/backup-virtualenv > /dev/null) - do - [[ "$counter" -eq "$max_retries" ]] && echo "Virtual environment creation failed!" && exit 1 - ((counter++)) - done - . /tmp/backup-virtualenv/bin/activate - pip install --quiet humanize smart-open==1.7.1 - pip install --quiet boto - deactivate - - CRON_CMD="${POSTGRES_BACKUP_SCHEDULE} root /usr/bin/nice -n 19 /usr/bin/ionice -c2 -n7 /tmp/backup-virtualenv/bin/python /kobo-docker-scripts/backup-to-s3.py > /srv/logs/backup.log 2>&1" - fi + CRON_CMD="${POSTGRES_BACKUP_SCHEDULE} root /usr/bin/nice -n 19 /usr/bin/ionice -c2 -n7 /tmp/backup-virtualenv/bin/python /kobo-docker-scripts/backup-to-s3.py > /srv/logs/backup.log 2>&1" else CRON_CMD="${POSTGRES_BACKUP_SCHEDULE} root /usr/bin/nice -n 19 /usr/bin/ionice -c2 -n7 /bin/bash /kobo-docker-scripts/backup-to-disk.bash > /srv/logs/backup.log 2>&1" fi diff --git a/postgres/shared/upsert_users.sh b/postgres/scripts/upsert_users.sh similarity index 100% rename from postgres/shared/upsert_users.sh rename to postgres/scripts/upsert_users.sh diff --git a/postgres/secondary/failover.sh b/postgres/secondary/failover.sh deleted file mode 100644 index 4a1c88c7..00000000 --- a/postgres/secondary/failover.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash - -POSTGRES_BIN_DIRECTORY=/usr/lib/postgresql/9.5/bin/ -POSTGRES_REPO=/var/lib/postgresql -POSTGRES_BIN=${POSTGRES_BIN_DIRECTORY}/postgres -POSTGRES_DATA_DIR=${POSTGRES_REPO}/data -POSTGRES_CONFIG_FILE=${POSTGRES_DATA_DIR}/postgresql.conf -POSTGRES_PERMISSION_FILE=${POSTGRES_DATA_DIR}/pg_hba.conf -POSTGRES_BACKUPS_DIR=/srv/backups -POSTGRES_FAILOVER_TRIGGER_FILE=${POSTGRES_DATA_DIR}/failover.trigger - - -IS_PRIMARY_ALIVE=$((echo > /dev/tcp/${KOBO_POSTGRES_PRIMARY_ENDPOINT//\"/}/${POSTGRES_PORT}) >/dev/null 2>&1 && echo "1" || echo "0") -UNHEALTHY_FILE="/tmp/${KOBO_POSTGRES_PRIMARY_ENDPOINT//\"/}-unhealthy-count.txt" - -if [ "$IS_PRIMARY_ALIVE" == "1" ]; then - echo "Primary is alive, nothing to do" - rm -rf $UNHEALTHY_FILE -else - UNHEALTHY_COUNT=0 - - if [ -f "$UNHEALTHY_FILE" ]; then - UNHEALTHY_COUNT=$(cat $UNHEALTHY_FILE) - fi - - UNHEALTHY_COUNT=$(($UNHEALTHY_COUNT + 1)) - - echo "UNHEALTHY_COUNT: $UNHEALTHY_COUNT" - - if [ "$UNHEALTHY_COUNT" -ge "$KOBO_THRESHOLD_FAILOVER" ]; then - - if [ ! -f "$POSTGRES_FAILOVER_TRIGGER_FILE" ]; then - echo "Starting failover process..." - su postgres -c "touch $POSTGRES_FAILOVER_TRIGGER_FILE" - else - echo "Failover process has already been launched..." - fi - else - echo $UNHEALTHY_COUNT > $UNHEALTHY_FILE - fi -fi diff --git a/postgres/secondary/init_01_replication.sh b/postgres/secondary/init_01_replication.sh deleted file mode 100644 index 2c18b542..00000000 --- a/postgres/secondary/init_01_replication.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash - -echo "Creating pgpass file..." -echo "${KOBO_POSTGRES_PRIMARY_ENDPOINT}:${POSTGRES_PORT}:*:${KOBO_POSTGRES_REPLICATION_USER}:${KOBO_POSTGRES_REPLICATION_PASSWORD}" | tr -d '"' > "$POSTGRES_REPO/.pgpass" -chown postgres:postgres "$POSTGRES_REPO/.pgpass" -chmod 600 "$POSTGRES_REPO/.pgpass" - - -echo "Let's the primary start, wait for 30s" -echo "Primary should be: $KOBO_POSTGRES_PRIMARY_ENDPOINT" -sleep 30 - -IS_OPENED=$((echo > /dev/tcp/${KOBO_POSTGRES_PRIMARY_ENDPOINT//\"/}/${POSTGRES_PORT}) >/dev/null 2>&1 && echo "1" || echo "0") - -if [ "$IS_OPENED" == "1" ]; then - - # Shutdown postgres before importing primary data - PGUSER="${PGUSER:-postgres}" pg_ctl -D "$PGDATA" -m fast -w stop - - echo "Deleting database directory..." - rm -rf $POSTGRES_DATA_DIR/* - echo "Cloning primary data..." - echo ${KOBO_POSTGRES_REPLICATION_PASSWORD} | PGUSER="${PGUSER:-postgres}" PGPASSFILE=${POSTGRES_REPO//\"/}/.pgpass pg_basebackup -h ${KOBO_POSTGRES_PRIMARY_ENDPOINT//\"/} --port=${POSTGRES_PORT//\"/} -D ${POSTGRES_DATA_DIR//\"/} -U ${KOBO_POSTGRES_REPLICATION_USER//\"/} -v -P --xlog -c fast -W - echo "Primary data cloned!" - - POSTGRES_RECOVERY_FILE=${POSTGRES_DATA_DIR}/recovery.conf - - echo "Creation recovery configuration file..." - cp $KOBO_DOCKER_SCRIPTS_DIR/secondary/recovery.conf ${POSTGRES_RECOVERY_FILE} - sed -i "s/POSTGRES_PORT/${POSTGRES_PORT//\"/}/g" "$POSTGRES_RECOVERY_FILE" - sed -i "s/KOBO_POSTGRES_PRIMARY_ENDPOINT/${KOBO_POSTGRES_PRIMARY_ENDPOINT//\"/}/g" "$POSTGRES_RECOVERY_FILE" - sed -i "s/KOBO_POSTGRES_REPLICATION_USER/${KOBO_POSTGRES_REPLICATION_USER//\"/}/g" "$POSTGRES_RECOVERY_FILE" - sed -i "s/KOBO_POSTGRES_REPLICATION_PASSWORD/${KOBO_POSTGRES_REPLICATION_PASSWORD//\"/}/g" "$POSTGRES_RECOVERY_FILE" - sed -i "s~POSTGRES_DATA_DIR~${POSTGRES_DATA_DIR//\"/}~g" "$POSTGRES_RECOVERY_FILE" - - # Restart postgres the same way it was by docker-entrypoint.sh - PGUSER="${PGUSER:-postgres}" pg_ctl -D "$PGDATA" -o "-c listen_addresses='localhost'" -w start - - echo "Done!" -else - echo "ERROR: NO PRIMARY FOUND" -fi diff --git a/postgres/secondary/postgres.conf b/postgres/secondary/postgres.conf deleted file mode 100644 index 47fbd926..00000000 --- a/postgres/secondary/postgres.conf +++ /dev/null @@ -1,40 +0,0 @@ -##################################################################################### -# SECONDARY SPECIFIC -# If file must be appended to shared/postgres.conf -##################################################################################### -#------------------------------------------------------------------------------------ -# TUNING -#------------------------------------------------------------------------------------ -# These settings are based on server configuration -# https://www.pgconfig.org/#/tuning -# DB Version: 9.5 -# OS Type: linux -# DB Type: Mixed -# Hard-drive: HDD -# Total Memory (RAM): 2GB - -# Memory Configuration -shared_buffers = 512MB -effective_cache_size = 2GB -work_mem = 10MB -maintenance_work_mem = 128MB - -# Checkpoint Related Configuration. Must be identical to primary server settings. -min_wal_size = 512MB -max_wal_size = 2GB -checkpoint_completion_target = 0.9 -wal_buffers = 15MB - -# Network Related Configuration. -listen_addresses = '*' -max_connections = 100 # Must be identical to primary server settings. - -#------------------------------------------------------------------------------------ -# REPLICATION -#------------------------------------------------------------------------------------ -hot_standby_feedback = on - -# https://stackoverflow.com/a/33282856 -# https://stackoverflow.com/a/34404303 -max_standby_streaming_delay = -1 -max_standby_archive_delay = -1 diff --git a/postgres/secondary/recovery.conf b/postgres/secondary/recovery.conf deleted file mode 100644 index dde5d2d3..00000000 --- a/postgres/secondary/recovery.conf +++ /dev/null @@ -1,3 +0,0 @@ -standby_mode = on -primary_conninfo = 'host=KOBO_POSTGRES_PRIMARY_ENDPOINT port=POSTGRES_PORT user=KOBO_POSTGRES_REPLICATION_USER password=KOBO_POSTGRES_REPLICATION_PASSWORD' -trigger_file = 'POSTGRES_DATA_DIR/failover.trigger' diff --git a/scripts/runtime_variables_kobocat.source.bash b/scripts/runtime_variables_kobocat.source.bash deleted file mode 100644 index 80c01cea..00000000 --- a/scripts/runtime_variables_kobocat.source.bash +++ /dev/null @@ -1,27 +0,0 @@ -if [[ ! -z "${PUBLIC_DOMAIN_NAME}" ]]; then - if [[ ${NGINX_PUBLIC_PORT} != "" && ${NGINX_PUBLIC_PORT} != "80" ]]; then - PUBLIC_PORT=":${NGINX_PUBLIC_PORT}" - else - PUBLIC_PORT="" - fi - # SERVER CONFIGURATION - export KOBOFORM_URL="${PUBLIC_REQUEST_SCHEME}://${KOBOFORM_PUBLIC_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}${PUBLIC_PORT}" - export KOBOFORM_INTERNAL_URL="http://${KOBOFORM_PUBLIC_SUBDOMAIN}.${INTERNAL_DOMAIN_NAME}" # Always use HTTP internally. - export KOBOCAT_URL="${PUBLIC_REQUEST_SCHEME}://${KOBOCAT_PUBLIC_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}${PUBLIC_PORT}" - export ENKETO_URL="${PUBLIC_REQUEST_SCHEME}://${ENKETO_EXPRESS_PUBLIC_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}${PUBLIC_PORT}" - export SESSION_COOKIE_DOMAIN=".${PUBLIC_DOMAIN_NAME}" - export DJANGO_ALLOWED_HOSTS=".${PUBLIC_DOMAIN_NAME} .${INTERNAL_DOMAIN_NAME}" - - # DATABASE - export DATABASE_URL="${KC_DATABASE_URL}" - export POSTGRES_DB="${KC_POSTGRES_DB}" - - # OTHER - export KOBOCAT_AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID}" - export KOBOCAT_AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY}" - export DJANGO_DEBUG="${KOBOCAT_DJANGO_DEBUG}" - export RAVEN_DSN="${KOBOCAT_RAVEN_DSN}" -else - echo 'Please fill out your `envfile`!' - exit 1 -fi diff --git a/scripts/runtime_variables_kpi.source.bash b/scripts/runtime_variables_kpi.source.bash deleted file mode 100644 index f2541681..00000000 --- a/scripts/runtime_variables_kpi.source.bash +++ /dev/null @@ -1,29 +0,0 @@ -if [[ ! -z "${PUBLIC_DOMAIN_NAME}" ]]; then - if [[ ${NGINX_PUBLIC_PORT} != "" && ${NGINX_PUBLIC_PORT} != "80" ]]; then - PUBLIC_PORT=":${NGINX_PUBLIC_PORT}" - else - PUBLIC_PORT="" - fi - # SERVER CONFIGURATION - export KOBOFORM_URL="${PUBLIC_REQUEST_SCHEME}://${KOBOFORM_PUBLIC_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}${PUBLIC_PORT}" - export KOBOFORM_INTERNAL_URL="http://${KOBOFORM_PUBLIC_SUBDOMAIN}.${INTERNAL_DOMAIN_NAME}" # Always use HTTP internally. - export ENKETO_URL="${PUBLIC_REQUEST_SCHEME}://${ENKETO_EXPRESS_PUBLIC_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}${PUBLIC_PORT}" - export ENKETO_INTERNAL_URL="http://${ENKETO_EXPRESS_PUBLIC_SUBDOMAIN}.${INTERNAL_DOMAIN_NAME}" # Always use HTTP internally. - export KOBOCAT_URL="${PUBLIC_REQUEST_SCHEME}://${KOBOCAT_PUBLIC_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}${PUBLIC_PORT}" - export KOBOCAT_INTERNAL_URL="http://${KOBOCAT_PUBLIC_SUBDOMAIN}.${INTERNAL_DOMAIN_NAME}" # Always use HTTP internally. - export SESSION_COOKIE_DOMAIN=".${PUBLIC_DOMAIN_NAME}" - export DJANGO_ALLOWED_HOSTS=".${PUBLIC_DOMAIN_NAME} .${INTERNAL_DOMAIN_NAME}" - - # DATABASE - export DATABASE_URL="${KPI_DATABASE_URL}" - export POSTGRES_DB="${KPI_POSTGRES_DB}" - - # OTHER - export DJANGO_DEBUG="${KPI_DJANGO_DEBUG}" - export RAVEN_DSN="${KPI_RAVEN_DSN}" - export RAVEN_JS_DSN="${KPI_RAVEN_JS_DSN}" - -else - echo 'Please fill out your `envfile`!' - exit 1 -fi diff --git a/scripts/wait_for_kpi.bash b/scripts/wait_for_kpi.bash deleted file mode 100755 index 99dd495a..00000000 --- a/scripts/wait_for_kpi.bash +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -set -e - -echo 'Waiting for container `kpi`.' -wait-for-it -t 300 -h kpi -p $KPI_PORT -echo 'Container `kpi` up.' - -echo 'Waiting for `kpi` web service.' -until curl kpi:${KPI_PORT} &> /dev/null || [[ "$?" == '52' ]]; do - sleep 1 -done -echo '`kpi` web service ready.' From 4dbabf6eab25fa8856f8199cfa678d9aa7e69f14 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Fri, 5 Apr 2024 12:27:05 -0400 Subject: [PATCH 02/17] Remove unused env files samples --- env/envfile.txt | 38 -------------------------------------- env/envfiles/enketo.txt | 5 ----- 2 files changed, 43 deletions(-) delete mode 100644 env/envfile.txt delete mode 100644 env/envfiles/enketo.txt diff --git a/env/envfile.txt b/env/envfile.txt deleted file mode 100644 index c275f9c7..00000000 --- a/env/envfile.txt +++ /dev/null @@ -1,38 +0,0 @@ -####################### -# Mandatory variables # -####################### - -# Choose between http or https -PUBLIC_REQUEST_SCHEME=https -# The publicly-accessible domain where your KoBo Toolbox instance will be reached (e.g. example.com). -PUBLIC_DOMAIN_NAME=domain.name -# The private domain used in docker network. Useful for communication between containers without passing through -# a load balancer. No need to be resolved by a public DNS. -INTERNAL_DOMAIN_NAME=docker.internal -# The publicly-accessible subdomain for the KoBoForm form building and management interface (e.g. koboform). -KOBOFORM_PUBLIC_SUBDOMAIN=kobo -# The publicly-accessible subdomain for the KoBoCAT data collection and project management interface (e.g.kobocat). -KOBOCAT_PUBLIC_SUBDOMAIN=kc -# The publicly-accessible subdomain for the Enketo Express web forms (e.g. enketo). -ENKETO_EXPRESS_PUBLIC_SUBDOMAIN=ee - -# For now, you must set ENKETO_API_TOKEN, used by KPI and KoBoCAT, to the same -# value as ENKETO_API_KEY. Eventually, KPI and KoBoCAT will also read -# ENKETO_API_KEY and the duplication will no longer be necessary. -# For a description of this setting, see "api key" here: -# https://github.com/kobotoolbox/enketo-express/tree/master/config#linked-form-and-data-server. -ENKETO_API_KEY= -ENKETO_API_TOKEN= - -# Canonically a 50-character random string. For Django 1.8.13, see https://docs.djangoproject.com/en/1.8/ref/settings/#secret-key and https://github.com/django/django/blob/4022b2c306e88a4ab7f80507e736ce7ac7d01186/django/core/management/commands/startproject.py#L29-L31. -# To generate a secret key in the same way as `django-admin startproject` you can run: -# docker-compose run --rm kpi python -c 'from django.utils.crypto import get_random_string; print(get_random_string(50, "abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)"))' -DJANGO_SECRET_KEY= - -# The initial superuser's username. -KOBO_SUPERUSER_USERNAME= -# The initial superuser's password. -KOBO_SUPERUSER_PASSWORD= - -# The e-mail address where your users can contact you. -KOBO_SUPPORT_EMAIL=support@domain.name diff --git a/env/envfiles/enketo.txt b/env/envfiles/enketo.txt deleted file mode 100644 index a38d1eb4..00000000 --- a/env/envfiles/enketo.txt +++ /dev/null @@ -1,5 +0,0 @@ -ENKETO_REDIS_MAIN_HOST=redis-main.domain.name -ENKETO_REDIS_CACHE_HOST=redis-cache.domain.name -ENKETO_LINKED_FORM_AND_DATA_SERVER_SERVER_URL=kobo.domain.name -ENKETO_LINKED_FORM_AND_DATA_SERVER_API_KEY=CHANGE ME!! -ENKETO_SUPPORT_EMAIL=support@domain.name From e712f31e99dcb944bc256c5a1191eda6f179f510 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Fri, 5 Apr 2024 12:27:55 -0400 Subject: [PATCH 03/17] Update docker compose files for workers --- docker-compose.backend.yml | 1 - docker-compose.frontend.yml | 139 +++++------------------------------- 2 files changed, 19 insertions(+), 121 deletions(-) diff --git a/docker-compose.backend.yml b/docker-compose.backend.yml index 32430b7c..bc5b734c 100644 --- a/docker-compose.backend.yml +++ b/docker-compose.backend.yml @@ -1,5 +1,4 @@ # Back-end containers - version: '3' services: diff --git a/docker-compose.frontend.yml b/docker-compose.frontend.yml index 31b6a6a0..34e428fc 100644 --- a/docker-compose.frontend.yml +++ b/docker-compose.frontend.yml @@ -4,8 +4,9 @@ version: '3' services: - kpi: + kpi: &django image: kobotoolbox/kpi:2.024.09 + init: true hostname: kpi env_file: - ../kobo-env/envfiles/aws.txt @@ -30,7 +31,7 @@ services: - ./scripts/wait_for_postgres.bash:/srv/init/wait_for_postgres.bash:ro # Persistent storage for FileFields when S3 not used (e.g. exports, uploaded map layers) - ./.vols/kpi_media:/srv/src/kpi/media - # Allow access to KoBoCAT media uploads within KPI + # Allow access to KoboCAT media uploads within KPI - ./.vols/kobocat_media_uploads:/srv/src/kobocat/media restart: unless-stopped networks: @@ -40,125 +41,23 @@ services: - kpi.internal - kobocat.docker.container - kobocat.internal - command: "bash -c exec /entrypoint.sh" -# worker: -# image: kobotoolbox/kpi:2.024.09 -# hostname: worker -# env_file: -# - ../kobo-env/envfiles/aws.txt -# - ../kobo-env/envfiles/databases.txt -# - ../kobo-env/envfiles/django.txt -# - ../kobo-env/envfiles/external_services.txt -# - ../kobo-env/envfiles/domains.txt -# - ../kobo-env/envfiles/smtp.txt -# volumes: -# - ./.vols/static/kpi:/srv/static -# - ./log/kpi:/srv/logs -# - ./scripts/wait_for_mongo.bash:/srv/init/wait_for_mongo.bash:ro -# - ./scripts/wait_for_postgres.bash:/srv/init/wait_for_postgres.bash:ro -# # Persistent storage for FileFields when S3 not used (e.g. exports, uploaded map layers) -# - ./.vols/kpi_media:/srv/src/kpi/media -# # Allow access to KoBoCAT media uploads within KPI -# - ./.vols/kobocat_media_uploads:/srv/src/kobocat/media -# restart: unless-stopped -# command: celery -A kobo worker --loglevel=info \ -# --hostname=kpi_main_worker@%h \ -# --logfile=/srv/logs/celery_kpi.log \ -# --pidfile=/tmp/celery_kpi.pid \ -# --queues=kpi_queue \ -# --exclude-queues=kpi_low_priority_queue,kobocat_queue \ -# --uid=$${UWSGI_USER} \ -# --gid=$${UWSGI_GROUP} \ -# --autoscale $${CELERY_AUTOSCALE_MIN},$${CELERY_AUTOSCALE_MAX} -# -# worker_low_priority: -# image: kobotoolbox/kpi:2.024.09 -# hostname: worker_low_priority -# env_file: -# - ../kobo-env/envfiles/aws.txt -# - ../kobo-env/envfiles/databases.txt -# - ../kobo-env/envfiles/django.txt -# - ../kobo-env/envfiles/external_services.txt -# - ../kobo-env/envfiles/domains.txt -# - ../kobo-env/envfiles/smtp.txt -# volumes: -# - ./.vols/static/kpi:/srv/static -# - ./log/kpi:/srv/logs -# - ./scripts/wait_for_mongo.bash:/srv/init/wait_for_mongo.bash:ro -# - ./scripts/wait_for_postgres.bash:/srv/init/wait_for_postgres.bash:ro -# # Persistent storage for FileFields when S3 not used (e.g. exports, uploaded map layers) -# - ./.vols/kpi_media:/srv/src/kpi/media -# # Allow access to KoBoCAT media uploads within KPI -# - ./.vols/kobocat_media_uploads:/srv/src/kobocat/media -# restart: unless-stopped -# command: celery -A kobo worker --loglevel=info \ -# --hostname=kpi_main_worker@%h \ -# --logfile=/srv/logs/celery_kpi.log \ -# --pidfile=/tmp/celery_kpi.pid \ -# --queues=kpi_queue \ -# --exclude-queues=kpi_low_priority_queue,kobocat_queue \ -# --uid=$${UWSGI_USER} \ -# --gid=$${UWSGI_GROUP} \ -# --autoscale $${CELERY_AUTOSCALE_MIN},$${CELERY_AUTOSCALE_MAX} -# -# worker_kobocat: -# image: kobotoolbox/kpi:2.024.09 -# hostname: worker_kobocat -# env_file: -# - ../kobo-env/envfiles/aws.txt -# - ../kobo-env/envfiles/databases.txt -# - ../kobo-env/envfiles/django.txt -# - ../kobo-env/envfiles/external_services.txt -# - ../kobo-env/envfiles/domains.txt -# - ../kobo-env/envfiles/smtp.txt -# volumes: -# - ./.vols/static/kpi:/srv/static -# - ./log/kpi:/srv/logs -# - ./scripts/wait_for_mongo.bash:/srv/init/wait_for_mongo.bash:ro -# - ./scripts/wait_for_postgres.bash:/srv/init/wait_for_postgres.bash:ro -# # Persistent storage for FileFields when S3 not used (e.g. exports, uploaded map layers) -# - ./.vols/kpi_media:/srv/src/kpi/media -# # Allow access to KoBoCAT media uploads within KPI -# - ./.vols/kobocat_media_uploads:/srv/src/kobocat/media -# restart: unless-stopped -# command: celery -A kobo worker --loglevel=info \ -# --hostname=kobocat_main_worker@%h \ -# --logfile=/srv/logs/celery_kobocat.log \ -# --pidfile=/tmp/celery_kobocat.pid \ -# --queues=kobocat_queue \ -# --exclude-queues=kpi_low_priority_queue,kpi_queue \ -# --uid=$${UWSGI_USER} \ -# --gid=$${UWSGI_GROUP} \ -# --autoscale $${CELERY_AUTOSCALE_MIN},$${CELERY_AUTOSCALE_MAX} -# -# beat: -# image: kobotoolbox/kpi:2.024.09 -# hostname: beat -# env_file: -# - ../kobo-env/envfiles/aws.txt -# - ../kobo-env/envfiles/databases.txt -# - ../kobo-env/envfiles/django.txt -# - ../kobo-env/envfiles/external_services.txt -# - ../kobo-env/envfiles/domains.txt -# - ../kobo-env/envfiles/smtp.txt -# volumes: -# - ./.vols/static/kpi:/srv/static -# - ./log/django:/srv/logs -# - ./scripts/wait_for_mongo.bash:/srv/init/wait_for_mongo.bash:ro -# - ./scripts/wait_for_postgres.bash:/srv/init/wait_for_postgres.bash:ro -# # Persistent storage for FileFields when S3 not used (e.g. exports, uploaded map layers) -# - ./.vols/kpi_media:/srv/src/kpi/media -# # Allow access to KoBoCAT media uploads within KPI -# - ./.vols/kobocat_media_uploads:/srv/src/kobocat/media -# restart: unless-stopped -# command: celery -A kobo beat --loglevel=info \ -# --logfile=/srv/logs/celery_beat.log \ -# --pidfile=/tmp/celery_beat.pid \ -# --scheduler django_celery_beat.schedulers:DatabaseScheduler \ -# --uid=$${UWSGI_USER} \ -# --gid=$${UWSGI_GROUP} -# + worker: + <<: *django + command: ['bash', '/srv/src/kpi/docker/entrypoint_celery_kpi_worker.bash'] + + worker_low_priority: + <<: *django + command: ['bash', '/srv/src/kpi/docker/entrypoint_celery_kpi_low_priority_worker.bash'] + + worker_kobocat: + <<: *django + command: ['bash', '/srv/src/kpi/docker/entrypoint_celery_kobocat_worker.bash'] + + beat: + <<: *django + command: ['bash', '/srv/src/kpi/docker/entrypoint_celery_kobocat_worker.bash'] + nginx: image: nginx:1.23 hostname: nginx From c70308924cea8f2146984c369c883efaf6045ee4 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Fri, 5 Apr 2024 12:47:25 -0400 Subject: [PATCH 04/17] Fix NGINX template --- .../docker-entrypoint.d/30-init-kobo-nginx.sh | 10 ++++++--- .../templates/nginx_site_default.conf.tmpl | 21 +++++++++---------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/nginx/docker-entrypoint.d/30-init-kobo-nginx.sh b/nginx/docker-entrypoint.d/30-init-kobo-nginx.sh index fff8b123..63f6ed2a 100755 --- a/nginx/docker-entrypoint.d/30-init-kobo-nginx.sh +++ b/nginx/docker-entrypoint.d/30-init-kobo-nginx.sh @@ -13,7 +13,8 @@ cp ${KOBO_DOCKER_SCRIPTS_DIR}/nginx.conf /etc/nginx/nginx.conf echo "Clearing out any default configurations" rm -rf /etc/nginx/conf.d/* -TEMPLATED_VAR_REFS="${TEMPLATED_VAR_REFS} \${NGINX_PUBLIC_PORT}" +export TEMPLATED_VAR_REFS="${TEMPLATED_VAR_REFS} \${NGINX_PUBLIC_PORT}" +export KOBOCAT_HEADER_FOR_DEV='' if [[ "${WSGI}" != 'UWSGI' ]] ; then echo "Proxying directly (debug) to Django without uWSGI." @@ -22,17 +23,20 @@ if [[ "${WSGI}" != 'UWSGI' ]] ; then cat ${KOBO_DOCKER_SCRIPTS_DIR}/templates/proxy_pass.conf.tmpl \ | envsubst '${KOBOFORM_PUBLIC_SUBDOMAIN} ${PUBLIC_DOMAIN_NAME} ${NGINX_PUBLIC_PORT}' \ > ${INCLUDES_DIR}/proxy_pass.conf + + # Allow to open Kobocat in an iframe + export KOBOCAT_HEADER_FOR_DEV='add_header X-Frame-Options ALLOWALL always;' else echo "Proxying to Django through uWSGI." # Create a `uwsgi_pass` configuration for this container. cat ${KOBO_DOCKER_SCRIPTS_DIR}/templates/uwsgi_pass.conf.tmpl \ | envsubst '${UWSGI_PASS_TIMEOUT}' \ - > ${INCLUDES_DIR}/uwsgi_pass.conf + > ${INCLUDES_DIR}/proxy_pass.conf fi # Do environment variable substitutions and activate the resulting config. file. -cat ${KOBO_DOCKER_SCRIPTS_DIR}/templates/nginx_site_default.conf.tmpl | envsubst "${TEMPLATED_VAR_REFS}" > /etc/nginx/conf.d/default.conf +cat ${KOBO_DOCKER_SCRIPTS_DIR}/templates/nginx_site_default.conf.tmpl | envsubst "${TEMPLATED_VAR_REFS} \${KOBOCAT_HEADER_FOR_DEV}" > /etc/nginx/conf.d/default.conf # Copy includes files cat ${KOBO_DOCKER_SCRIPTS_DIR}/templates/include.https_redirection.conf.tmpl | envsubst "${TEMPLATED_VAR_REFS}" > /etc/nginx/includes/https_redirection.conf diff --git a/nginx/kobo-docker-scripts/templates/nginx_site_default.conf.tmpl b/nginx/kobo-docker-scripts/templates/nginx_site_default.conf.tmpl index b0db9ac8..056898e1 100644 --- a/nginx/kobo-docker-scripts/templates/nginx_site_default.conf.tmpl +++ b/nginx/kobo-docker-scripts/templates/nginx_site_default.conf.tmpl @@ -40,18 +40,17 @@ server { include /etc/nginx/includes/server_directive_common.conf; location / { - # Proxy through uWSGI. - ${kobocat_include_uwsgi_pass} - - # Debug proxying directly to container. - ${kobocat_include_proxy_pass} - } - - location /static { - alias /srv/www/kobocat; + proxy_pass http://kpi:8000/; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection keep-alive; + proxy_set_header Host $host; + proxy_cache_bypass $http_upgrade; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_hide_header X-Frame-Options; + ${KOBOCAT_HEADER_FOR_DEV} } - - include /etc/nginx/includes/protected_directive.conf; } # KoboForm public HTTP From 1252c184c295f407291a9016c78a64ce6c7f7ba4 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Fri, 5 Apr 2024 14:31:44 -0400 Subject: [PATCH 05/17] Pass "init: true" to Enketo Express container --- docker-compose.frontend.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.frontend.yml b/docker-compose.frontend.yml index 34e428fc..70d14eb5 100644 --- a/docker-compose.frontend.yml +++ b/docker-compose.frontend.yml @@ -84,6 +84,7 @@ services: enketo_express: image: kobotoolbox/enketo-express-extra-widgets:6.2.2 + init: true hostname: enketo_express env_file: - ../kobo-env/envfiles/domains.txt From 28c1197f88444cad7fb2d2ccbc405fba34d0ecf4 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Fri, 5 Apr 2024 14:32:49 -0400 Subject: [PATCH 06/17] Update overrides and env variables --- docker-compose.backend.override.yml.sample | 8 ------ docker-compose.frontend.override.yml.sample | 26 ++++++++++++++++++-- env/envfiles/databases.txt | 10 ++++++-- env/envfiles/django.txt | 27 +++++++++++++++++++-- 4 files changed, 57 insertions(+), 14 deletions(-) diff --git a/docker-compose.backend.override.yml.sample b/docker-compose.backend.override.yml.sample index 7ad4cf34..b8e28134 100644 --- a/docker-compose.backend.override.yml.sample +++ b/docker-compose.backend.override.yml.sample @@ -4,8 +4,6 @@ version: '3' services: postgres: - #environment: - # - POSTGRES_BACKUP_FROM_SECONDARY=True # Uncomment `ports` section if you want to expose ports (e.g. use as separated servers) # Before uncommenting, be sure you have a firewall configured to prevent these ports # from being publicly accessible @@ -32,9 +30,6 @@ services: # Comment out section (`redis_main` and `redis_cache`) below if you want to # start redis containers from the secondary back-end server redis_main: - extends: - file: docker-compose.backend.template.yml - service: redis_main # Uncomment `ports` section if you want to expose ports (e.g. use as separated servers) # Before uncommenting, be sure you have a firewall configured to prevent these ports # from being publicly accessible @@ -47,9 +42,6 @@ services: - redis-main.kobo.private redis_cache: - extends: - file: docker-compose.backend.template.yml - service: redis_cache # Uncomment `ports` section if you want to expose ports (e.g. use as separated servers) # Before uncommenting, be sure you have a firewall configured to prevent these ports # from being publicly accessible diff --git a/docker-compose.frontend.override.yml.sample b/docker-compose.frontend.override.yml.sample index 72e36e4a..de43d121 100644 --- a/docker-compose.frontend.override.yml.sample +++ b/docker-compose.frontend.override.yml.sample @@ -16,13 +16,34 @@ services: networks: kobo-be-network: aliases: - - kpi - kpi.docker.container + worker_kobocat: + networks: + kobo-be-network: + aliases: + - worker_kobocat + - worker_kobocat.docker.container + + worker_low_priority: + networks: + kobo-be-network: + aliases: + - worker_kobocat + - worker_kobocat.docker.container + + beat: + networks: + kobo-be-network: + aliases: + - beat + - beat.docker.container + nginx: environment: # `NGINX_PUBLIC_PORT` is the port used to access KoboToolbox (e.g. `https://kf.kobotoolbox.org:`) - - NGINX_PUBLIC_PORT=80 + # Include the colons with the port + - NGINX_PUBLIC_PORT=:8080 #- UWSGI_PASS_TIMEOUT=130 ports: # :80 . If no proxies, `proxy_port` should be the same as `NGINX_PUBLIC_PORT` @@ -49,3 +70,4 @@ networks: # name: _kobo-be-network`, where `prefix` is usually the parent # folder name name: kobodocker_kobo-be-network + external: true diff --git a/env/envfiles/databases.txt b/env/envfiles/databases.txt index 0319f9ea..4e895604 100644 --- a/env/envfiles/databases.txt +++ b/env/envfiles/databases.txt @@ -45,8 +45,14 @@ KPI_DATABASE_URL=postgis://kobo:kobo@postgres.domain.name:5432/kobotoolbox #REDIS_BACKUP_SCHEDULE=0 3 * * 0 -REDIS_SESSION_URL=redis://:kobo@redis-cache.kobo.private:6390/2 +REDIS_SESSION_URL=redis://:kobo@redis-cache.kobo.private:6380/2 REDIS_PASSWORD=kobo -CACHE_URL=redis://:kobo@redis-cache.kobo.private:6390/5 +CACHE_URL=redis://:kobo@redis-cache.kobo.private:6380/5 # In bytes, uncomment to cap Redis Cache memory # REDIS_CACHE_MAX_MEMORY= + +REDIS_PASSWORD=${REDIS_PASSWORD} +CACHE_URL=redis://:kobo@redis-cache.kobo.private:6380/5 +REDIS_CACHE_MAX_MEMORY= +SERVICE_ACCOUNT_BACKEND_URL=redis://:kobo@redis-cache.kobo.private:6380/6 +ENKETO_REDIS_MAIN_URL=redis://:kobo@redis-main.kobo.private:6379/0 diff --git a/env/envfiles/django.txt b/env/envfiles/django.txt index bfe895e1..aa471604 100644 --- a/env/envfiles/django.txt +++ b/env/envfiles/django.txt @@ -3,8 +3,31 @@ TEMPLATE_DEBUG=False USE_X_FORWARDED_HOST=False CELERY_BROKER_URL=redis://:kobo@redis-main.domain.name:6379/1 - CELERY_AUTOSCALE_MIN=2 CELERY_AUTOSCALE_MAX=6 -${USE_SERVICE_ACCOUNT_WHITELISTED_HOSTS}SERVICE_ACCOUNT_WHITELISTED_HOSTS=${KOBOFORM_SUBDOMAIN}.${INTERNAL_DOMAIN_NAME} +DJANGO_SECRET_KEY=changeme!! +DJANGO_SESSION_COOKIE_AGE=604800 +DJANGO_ALLOWED_HOSTS=.domain.name .domain.internal + +SESSION_COOKIE_DOMAIN=".domain.name" + +# Comma separated domains +SERVICE_ACCOUNT_WHITELISTED_HOSTS=kf.kobo.internal + +# See "api key" here: https://github.com/kobotoolbox/enketo-express/tree/master/config#linked-form-and-data-server. +ENKETO_API_KEY=changeme!! + +# The initial superuser's username. +KOBO_SUPERUSER_USERNAME=super_admin +# The initial superuser's password. +KOBO_SUPERUSER_PASSWORD=changeme!! +# The e-mail address where your users can contact you. +KOBO_SUPPORT_EMAIL=support@domain.name + +KOBOFORM_URL=https://kf.domain.name +KOBOFORM_INTERNAL_URL=http://kf.kobo.internal # Always use HTTP internally. +ENKETO_URL=https://ee.domain.name +ENKETO_INTERNAL_URL=http://ee.kobo.internal # Always use HTTP internally. +KOBOCAT_URL=https://kc.domain.name +KOBOCAT_INTERNAL_URL=http://kc.kobo.internal # Always use HTTP internally. From 2a58ebacdb09e6cf2ab554cf4ef2927d4a4ee4d1 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Fri, 5 Apr 2024 18:21:26 -0400 Subject: [PATCH 07/17] Fix WSGI detection --- nginx/docker-entrypoint.d/30-init-kobo-nginx.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nginx/docker-entrypoint.d/30-init-kobo-nginx.sh b/nginx/docker-entrypoint.d/30-init-kobo-nginx.sh index 63f6ed2a..3c464c13 100755 --- a/nginx/docker-entrypoint.d/30-init-kobo-nginx.sh +++ b/nginx/docker-entrypoint.d/30-init-kobo-nginx.sh @@ -16,7 +16,7 @@ rm -rf /etc/nginx/conf.d/* export TEMPLATED_VAR_REFS="${TEMPLATED_VAR_REFS} \${NGINX_PUBLIC_PORT}" export KOBOCAT_HEADER_FOR_DEV='' -if [[ "${WSGI}" != 'UWSGI' ]] ; then +if [[ "${WSGI}" != 'uWSGI' ]] ; then echo "Proxying directly (debug) to Django without uWSGI." # Create a `proxy_pass` configuration for this container. From b015d16be14032cc8013ee8685ae7aa43bd7e566 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Thu, 23 May 2024 10:19:37 -0400 Subject: [PATCH 08/17] uWSGI gtemplate --- nginx/kobo-docker-scripts/templates/uwsgi_pass.conf.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nginx/kobo-docker-scripts/templates/uwsgi_pass.conf.tmpl b/nginx/kobo-docker-scripts/templates/uwsgi_pass.conf.tmpl index cc0a691e..21d5f61a 100644 --- a/nginx/kobo-docker-scripts/templates/uwsgi_pass.conf.tmpl +++ b/nginx/kobo-docker-scripts/templates/uwsgi_pass.conf.tmpl @@ -8,7 +8,7 @@ uwsgi_send_timeout ${UWSGI_PASS_TIMEOUT}; # See https://www.jethrocarr.com/2013/11/02/nginx-reverse-proxies-and-dns-resolution/ # 127.0.0.11 is internal docker DNS server (see `cat /etc/resolv.conf`) resolver 127.0.0.11 ipv6=off valid=1s; -set $upstream "${container_name}:${container_port}"; +set $upstream "kpi:8000"; uwsgi_pass $upstream; # For setting HTTP headers, see http://stackoverflow.com/a/14133533/1877326. uwsgi_param HTTP_X_REAL_IP $remote_addr; From 480197e5d273fa7885731f1c153816214702e946 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Thu, 23 May 2024 11:09:39 -0400 Subject: [PATCH 09/17] Fix uWSGI proxy with nginx --- .../templates/nginx_site_default.conf.tmpl | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/nginx/kobo-docker-scripts/templates/nginx_site_default.conf.tmpl b/nginx/kobo-docker-scripts/templates/nginx_site_default.conf.tmpl index 056898e1..888d29f5 100644 --- a/nginx/kobo-docker-scripts/templates/nginx_site_default.conf.tmpl +++ b/nginx/kobo-docker-scripts/templates/nginx_site_default.conf.tmpl @@ -29,7 +29,7 @@ server { } } -# KoboCAT HTTP. +# KoboCAT server { listen 80; server_name ${KOBOCAT_PUBLIC_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME} ${KOBOCAT_PUBLIC_SUBDOMAIN}.${INTERNAL_DOMAIN_NAME}; @@ -39,21 +39,18 @@ server { include /etc/nginx/includes/server_directive_common.conf; + location /static { + alias /srv/www/kpi; + } + location / { - proxy_pass http://kpi:8000/; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection keep-alive; - proxy_set_header Host $host; - proxy_cache_bypass $http_upgrade; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_hide_header X-Frame-Options; + # Proxy through uWSGI or Django webserver + include /etc/nginx/includes/proxy_pass.conf; ${KOBOCAT_HEADER_FOR_DEV} } } -# KoboForm public HTTP +# KPI server { listen 80; server_name ${KOBOFORM_PUBLIC_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME} ${KOBOFORM_PUBLIC_SUBDOMAIN}.${INTERNAL_DOMAIN_NAME}; @@ -84,6 +81,7 @@ server { include /etc/nginx/includes/protected_directive.conf; } +# Enketo Express server { listen 80; server_name ${ENKETO_EXPRESS_PUBLIC_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME} ${ENKETO_EXPRESS_PUBLIC_SUBDOMAIN}.${INTERNAL_DOMAIN_NAME}; From 1bea3e30549388e20ea684ff6d43ccbf4d4d90a2 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Tue, 28 May 2024 17:39:34 -0400 Subject: [PATCH 10/17] Fix $host value for dev environments --- nginx/kobo-docker-scripts/templates/proxy_pass.conf.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nginx/kobo-docker-scripts/templates/proxy_pass.conf.tmpl b/nginx/kobo-docker-scripts/templates/proxy_pass.conf.tmpl index d9b8c1a1..2b805864 100644 --- a/nginx/kobo-docker-scripts/templates/proxy_pass.conf.tmpl +++ b/nginx/kobo-docker-scripts/templates/proxy_pass.conf.tmpl @@ -10,7 +10,7 @@ proxy_set_header Host $host:$proxy_port; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $remote_addr; proxy_set_header X-Forwarded-Proto $scheme; -proxy_set_header X-Forwarded-Host ${KOBOFORM_PUBLIC_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}${NGINX_PUBLIC_PORT}; +proxy_set_header X-Forwarded-Host $host${NGINX_PUBLIC_PORT}; proxy_redirect off; # Add ranges support. From cea9364693730ca671e0112e8946424490be71f5 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Tue, 28 May 2024 18:26:24 -0400 Subject: [PATCH 11/17] Remove XFRAME_OPTION header dev environment --- nginx/docker-entrypoint.d/30-init-kobo-nginx.sh | 6 +----- .../templates/nginx_site_default.conf.tmpl | 1 - 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/nginx/docker-entrypoint.d/30-init-kobo-nginx.sh b/nginx/docker-entrypoint.d/30-init-kobo-nginx.sh index 3c464c13..ecebaf99 100755 --- a/nginx/docker-entrypoint.d/30-init-kobo-nginx.sh +++ b/nginx/docker-entrypoint.d/30-init-kobo-nginx.sh @@ -14,7 +14,6 @@ echo "Clearing out any default configurations" rm -rf /etc/nginx/conf.d/* export TEMPLATED_VAR_REFS="${TEMPLATED_VAR_REFS} \${NGINX_PUBLIC_PORT}" -export KOBOCAT_HEADER_FOR_DEV='' if [[ "${WSGI}" != 'uWSGI' ]] ; then echo "Proxying directly (debug) to Django without uWSGI." @@ -23,9 +22,6 @@ if [[ "${WSGI}" != 'uWSGI' ]] ; then cat ${KOBO_DOCKER_SCRIPTS_DIR}/templates/proxy_pass.conf.tmpl \ | envsubst '${KOBOFORM_PUBLIC_SUBDOMAIN} ${PUBLIC_DOMAIN_NAME} ${NGINX_PUBLIC_PORT}' \ > ${INCLUDES_DIR}/proxy_pass.conf - - # Allow to open Kobocat in an iframe - export KOBOCAT_HEADER_FOR_DEV='add_header X-Frame-Options ALLOWALL always;' else echo "Proxying to Django through uWSGI." @@ -36,7 +32,7 @@ else fi # Do environment variable substitutions and activate the resulting config. file. -cat ${KOBO_DOCKER_SCRIPTS_DIR}/templates/nginx_site_default.conf.tmpl | envsubst "${TEMPLATED_VAR_REFS} \${KOBOCAT_HEADER_FOR_DEV}" > /etc/nginx/conf.d/default.conf +cat ${KOBO_DOCKER_SCRIPTS_DIR}/templates/nginx_site_default.conf.tmpl | envsubst "${TEMPLATED_VAR_REFS}" > /etc/nginx/conf.d/default.conf # Copy includes files cat ${KOBO_DOCKER_SCRIPTS_DIR}/templates/include.https_redirection.conf.tmpl | envsubst "${TEMPLATED_VAR_REFS}" > /etc/nginx/includes/https_redirection.conf diff --git a/nginx/kobo-docker-scripts/templates/nginx_site_default.conf.tmpl b/nginx/kobo-docker-scripts/templates/nginx_site_default.conf.tmpl index 888d29f5..46f75f0f 100644 --- a/nginx/kobo-docker-scripts/templates/nginx_site_default.conf.tmpl +++ b/nginx/kobo-docker-scripts/templates/nginx_site_default.conf.tmpl @@ -46,7 +46,6 @@ server { location / { # Proxy through uWSGI or Django webserver include /etc/nginx/includes/proxy_pass.conf; - ${KOBOCAT_HEADER_FOR_DEV} } } From f193eb2f29eaf35d47c1ce0f77c11d78721b70fd Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Thu, 30 May 2024 16:35:50 -0400 Subject: [PATCH 12/17] Fix databases creation script --- scripts/wait_for_postgres.bash | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/scripts/wait_for_postgres.bash b/scripts/wait_for_postgres.bash index cb48c3b2..cb3a6937 100755 --- a/scripts/wait_for_postgres.bash +++ b/scripts/wait_for_postgres.bash @@ -7,7 +7,6 @@ echo 'Container `postgres` up.' echo 'Waiting for Postgres service.' # TODO: There must be a way to confirm Postgres is serving without the resulting "incomplete startup packet" warning in the logs. -POSTGRES_DB="${POSTGRES_DB:-kobotoolbox}" POSTGRES_USER="${POSTGRES_USER:-kobo}" export PGPASSWORD="${POSTGRES_PASSWORD:-kobo}" until pg_isready -h "${POSTGRES_HOST}" -p "${POSTGRES_PORT}" -U "${POSTGRES_USER}"; do @@ -16,8 +15,12 @@ done source /etc/profile -echo "Postgres service running; ensuring ${POSTGRES_DB} database exists and has PostGIS extensions..." -psql -d postgres -h "${POSTGRES_HOST}" -p "${POSTGRES_PORT}" -U "${POSTGRES_USER}" < Date: Fri, 21 Jun 2024 15:30:21 -0400 Subject: [PATCH 13/17] Bump NGINX version to 1.26 --- docker-compose.frontend.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.frontend.yml b/docker-compose.frontend.yml index 63a688dc..ca34fdf4 100644 --- a/docker-compose.frontend.yml +++ b/docker-compose.frontend.yml @@ -59,7 +59,7 @@ services: command: ['bash', '/srv/src/kpi/docker/entrypoint_celery_kobocat_worker.bash'] nginx: - image: nginx:1.23 + image: nginx:1.26 hostname: nginx env_file: - ../kobo-env/envfiles/domains.txt From 71a06c6ef841dea60c5928160c1e06f6ac85ff52 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Wed, 26 Jun 2024 15:42:09 -0400 Subject: [PATCH 14/17] Use KPI kobocat-as-django-app image --- docker-compose.frontend.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.frontend.yml b/docker-compose.frontend.yml index ca34fdf4..bc2b981e 100644 --- a/docker-compose.frontend.yml +++ b/docker-compose.frontend.yml @@ -5,7 +5,7 @@ version: '3' services: kpi: &django - image: kobotoolbox/kpi:2.024.12b + image: kobotoolbox/kpi:kobocat-as-django-app init: true hostname: kpi env_file: From f7ef09e7b1e0c82d6427097650824ab4a6abe9bf Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Mon, 22 Jul 2024 17:25:26 -0400 Subject: [PATCH 15/17] Restrict KoboCAT endpoints to KoboCAT domain --- .../templates/nginx_site_default.conf.tmpl | 85 ++++++++++++++++++- 1 file changed, 84 insertions(+), 1 deletion(-) diff --git a/nginx/kobo-docker-scripts/templates/nginx_site_default.conf.tmpl b/nginx/kobo-docker-scripts/templates/nginx_site_default.conf.tmpl index 46f75f0f..e3455347 100644 --- a/nginx/kobo-docker-scripts/templates/nginx_site_default.conf.tmpl +++ b/nginx/kobo-docker-scripts/templates/nginx_site_default.conf.tmpl @@ -43,10 +43,88 @@ server { alias /srv/www/kpi; } - location / { + # //bulk-submission + # //bulk-submission-form + location ~ ^/.+/bulk-submission(-form)?$ { + # Proxy through uWSGI or Django webserver + include /etc/nginx/includes/proxy_pass.conf; + } + + # //exports/// + # //exports/// + # //exports///delete + # //exports///new + # //exports///progress + # //forms//form\.json + # //forms//form\.xls + # //forms//form\.xml + location ~ ^/.+/(exports|forms)/.+/ { + # Proxy through uWSGI or Django webserver + include /etc/nginx/includes/proxy_pass.conf; + } + + # //formList + # //submission + # //xformsManifest/ + # //xformsMedia// + # //xformsMedia//\. + location ~ ^/.+/(formList|submission|xformsManifest|xformsMedia) { + # Proxy through uWSGI or Django webserver + include /etc/nginx/includes/proxy_pass.conf; + } + + # /attachment/ + # /attachment/ + # /media/ + # /media/ + location ~ ^/(attachment|media)/(.*)$ { + # Proxy through uWSGI or Django webserver + include /etc/nginx/includes/proxy_pass.conf; + } + + # /formList + # /formUpload + # /submission + # /upload + location ~ ^/(formList|formUpload|submission|upload)$ { + # Proxy through uWSGI or Django webserver + include /etc/nginx/includes/proxy_pass.conf; + } + + # /view/downloadSubmission + # /view/submissionList + location ~ ^/view/(downloadSubmission|submissionList)$ { + # Proxy through uWSGI or Django webserver + include /etc/nginx/includes/proxy_pass.conf; + } + + # /forms//form\.xml + # /xformsManifest/ + # /xformsMedia// + # /xformsMedia//\. + location ~ ^/(forms|xformsManifest|xformsMedia|xformsMedia)/([0-9]+) { + # Proxy through uWSGI or Django webserver + include /etc/nginx/includes/proxy_pass.conf; + } + + # Anything that starts with `/api/v1` + location /api/v1 { + # Proxy through uWSGI or Django webserver + include /etc/nginx/includes/proxy_pass.conf; + } + + # /legacy/service_health/ + # /legacy/service_health/minimal/ + location /legacy/ { # Proxy through uWSGI or Django webserver include /etc/nginx/includes/proxy_pass.conf; } + + include /etc/nginx/includes/protected_directive.conf; + + location / { + return 404; + } } # KPI @@ -60,6 +138,11 @@ server { return 301 /$1; } + # Block KoboCAT api if the domain is wrong + location /api/v1 { + return 404; + } + access_log /var/log/nginx/kpi.access.log with_host; error_log /var/log/nginx/kpi.error.log; From e0c1d1e3d2d7068881b1287244e5b03d6a343671 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Mon, 12 Aug 2024 14:08:11 -0400 Subject: [PATCH 16/17] Merge from master --- nginx/kobo-docker-scripts/nginx.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nginx/kobo-docker-scripts/nginx.conf b/nginx/kobo-docker-scripts/nginx.conf index cf3e3d7f..165a8565 100644 --- a/nginx/kobo-docker-scripts/nginx.conf +++ b/nginx/kobo-docker-scripts/nginx.conf @@ -20,7 +20,7 @@ http { tcp_nodelay on; keepalive_timeout 65; types_hash_max_size 2048; - # server_tokens off; + server_tokens off; server_names_hash_bucket_size 128; # server_name_in_redirect off; From ee8369f87e0688dc7809536ff908cec0302722b2 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Thu, 15 Aug 2024 10:28:43 -0400 Subject: [PATCH 17/17] Remove KPI_ prefix for UWSGI --- docker-compose.frontend.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docker-compose.frontend.yml b/docker-compose.frontend.yml index bc2b981e..15443273 100644 --- a/docker-compose.frontend.yml +++ b/docker-compose.frontend.yml @@ -18,12 +18,12 @@ services: sysctls: - net.core.somaxconn=2048 environment: - - KPI_UWSGI_MAX_REQUESTS=512 - - KPI_UWSGI_WORKERS_COUNT=2 - - KPI_UWSGI_CHEAPER_RSS_LIMIT_SOFT=134217728 - - KPI_UWSGI_CHEAPER_WORKERS_COUNT=1 - - KPI_UWSGI_HARAKIRI=120 - - KPI_UWSGI_WORKER_RELOAD_MERCY=120 + - UWSGI_MAX_REQUESTS=512 + - UWSGI_WORKERS_COUNT=2 + - UWSGI_CHEAPER_RSS_LIMIT_SOFT=134217728 + - UWSGI_CHEAPER_WORKERS_COUNT=1 + - UWSGI_HARAKIRI=120 + - UWSGI_WORKER_RELOAD_MERCY=120 volumes: - ./.vols/static/kpi:/srv/static - ./log/kpi:/srv/logs