wip

Author: bailletced

.env

@@ -7,12 +7,5 @@ MESSENGER_TRANSPORT_DSN=doctrine://default?auto_setup=0
 LOCK_DSN=flock
 CORS_ALLOW_ORIGIN='^https?://(localhost|127\.0\.0\.1)(:[0-9]+)?$'
 
-###> synchro scripts ###
-SQLITE_DATABASE=local.sqlite
-OPENCHURCH_HOST='https://api.openchurch.local/api'
-OPENCHURCH_API_TOKEN='giergij'
-PYTHONWARNINGS="ignore:Unverified HTTPS request"
-###< synchro scripts ###
-
 ELASTIC_PASSWORD=admin
 ELASTICSEARCH_IRI=https://elastic:admin@elasticsearch:9200

.env.test

@@ -1,7 +1,3 @@
 # define your env variables for the test env here
-KERNEL_CLASS='App\Kernel'
-APP_SECRET='$ecretf0rt3st'
-SYMFONY_DEPRECATIONS_HELPER=999999
-PANTHER_APP_ENV=panther
-PANTHER_ERROR_SCREENSHOT_DIR=./var/error-screenshots
 APP_ENV=test
+APP_SECRET='$ecretf0rt3st'

README.md

@@ -1,3 +1,14 @@
 1. Attention à la création des communautés / places !
    Les fields et leurs entités doivent être persistés ensemble avant de flush
 2. Comme nous n'avons pas encore de moyens de supprimer des entités (depuis endpoint ou admin), la synchro elastic pour la suppression n'a pas encore été implémentée
+
+### Script
+
+Load data from Wikidata: python3 synchro.py
+Push data to OpenChurch v2: python3 synchro.py push
+Print statistics: python3 synchro.py stats
+
+The local.sqlite serves as a local buffer to store the OpenChurch v2 UUIDs and remember what as already been pushed, what has not and determine what has been updated on Wikidata since the last push.
+
+1st run is very long (several hours) because it loads places (cities, countries) from Wikidata. It is recommended to put the attached SQLite database in the scripts folder to fasten the process. Every data from this file (except OpenChurch UUIDs) is from Wikidata with license is CC0.
+local.sqlite.gz

compose.override.yaml

@@ -1,10 +1,4 @@
 services:
-  ###> doctrine/doctrine-bundle ###
   db:
     ports:
       - "3306:3306"
-###< doctrine/doctrine-bundle ###
-
-###> symfony/mailer ###
-
-###< symfony/mailer ###

compose.yaml

@@ -67,19 +67,15 @@ services:
     build:
       context: .
       dockerfile: ./docker/python/Dockerfile
-    volumes:
-      - ./scripts:/app
-      - ./.env:/app/.env
-    entrypoint: ["sh", "/usr/local/bin/docker-python-entrypoint"]
-    command: ["/bin/sh"]
-    tty: true
-    stdin_open: true
     networks:
       - app-network
     extra_hosts:
       - "api.openchurch.local:host-gateway"
-    depends_on:
-      - backend
+    environment:
+      SQLITE_DATABASE: local.sqlite
+      OPENCHURCH_HOST: "https://api.openchurch.local/api"
+      OPENCHURCH_API_TOKEN: "fake_token"
+      PYTHONWARNINGS: "ignore:Unverified HTTPS request"
 
 volumes:
   openchurch_db_data: {}

docker/php/Dockerfile

@@ -2,34 +2,19 @@ FROM dunglas/frankenphp
 LABEL org.opencontainers.image.authors="[email protected]"
 
 ARG USER=www-data
-ARG SERVER_NAME=rosario.local
-
 WORKDIR /var/www/html
 
 RUN apt update && apt install -y \
     acl \
     autoconf \
     cron \
-    ffmpeg \
-    gawk \
-    g++ \ 
-    make \
-    libnss3-tools \
-    libpng-dev \
-    libexif-dev \
-    libxpm-dev \
-    libxml2-dev \
-    libxslt-dev \
-    libwebp-dev \
-    libzip-dev \
-    libgmp-dev \
-    libjpeg-dev \
     logrotate \
-    nodejs \
-    npm \
     sudo \
     bash \
     procps \
+    libzip-dev \
+    libxslt-dev \
+    libgmp-dev \
     && mkdir -p /var/www/html/var/ \
     && apt clean
 
@@ -38,14 +23,8 @@ RUN curl -sS https://getcomposer.org/installer -o /tmp/composer-setup.php \
     && php /tmp/composer-setup.php --install-dir=/usr/local/bin --filename=composer \
     && rm /tmp/composer-setup.php
 
-RUN docker-php-ext-configure gd --with-jpeg --with-webp \
-    && docker-php-ext-configure gmp \
-    && docker-php-ext-configure intl
-
 RUN docker-php-ext-install \
     pdo_mysql \
-    gd \
-    intl \
     zip \
     exif \
     xsl \
@@ -54,10 +33,8 @@ RUN docker-php-ext-install \
     gmp
 
 # add PECL extensions
-RUN pecl install redis && docker-php-ext-enable redis && \
-    pecl install apcu && docker-php-ext-enable apcu && \
+RUN pecl install apcu && docker-php-ext-enable apcu && \
     pecl install xdebug && docker-php-ext-enable xdebug
-    #pecl install amqp && docker-php-ext-enable amqp && \
 
 COPY src/ /var/www/html/src/
 COPY public/ /var/www/html/public/
@@ -72,6 +49,7 @@ COPY symfony.lock /var/www/html/
 COPY usr/local/bin/docker-php-entrypoint /usr/local/bin/
 COPY .env /var/www/html/
 COPY .env.test /var/www/html/
+COPY etc/cron.d/backend /etc/cron.d/backend
 
 # The following line is needed only for load tests
 COPY tests/ /var/www/html/tests/

docker/python/Dockerfile

@@ -1,11 +1,21 @@
-FROM alpine:3.20.3
+FROM python:3.9-slim as compiler
+ENV PYTHONUNBUFFERED 1
 
-WORKDIR /app
+WORKDIR /app/
 
-RUN apk update && apk add --update --no-cache \
-    python3 \
-    py3-pip \
-    && ln -sf python3 /usr/bin/python
+RUN python -m venv /opt/venv
+# Enable venv
+ENV PATH="/opt/venv/bin:$PATH"
 
-COPY ./usr/local/bin/docker-python-entrypoint /usr/local/bin/
-COPY .env /app/
+COPY ./scripts/requirements.txt /app/requirements.txt
+RUN pip install -Ur requirements.txt
+
+FROM python:3.9-slim as runner
+WORKDIR /app/
+COPY --from=compiler /opt/venv /opt/venv
+
+# Enable venv
+ENV PATH="/opt/venv/bin:$PATH"
+COPY ./scripts /app/
+
+ENTRYPOINT ["tail", "-f", "/dev/null"]

etc/cron.d/backend

@@ -0,0 +1,39 @@
+HOME=/var/www/html
+LANG=nb_NO.UTF-8
+LC_ALL=nb_NO.UTF-8
+
+# Each task to run has to be defined through a single line
+# indicating with different fields when the task will be run
+# and what command to run for the task
+#
+# To define the time you can provide concrete values for
+# minute (m), hour (h), day of month (dom), month (mon),
+# and day of week (dow) or use '*' in these fields (for 'any').
+#
+# Notice that tasks will be started based on the cron's system
+# daemon's notion of time and timezones.
+#
+# Output of the crontab jobs (including errors) is sent through
+# email to the user the crontab file belongs to (unless redirected).
+#
+# For example, you can run a backup of all your user accounts
+# at 5 a.m every week with:
+# 0 5 * * 1 tar -zcf /var/backups/home.tgz /home/
+#
+# For more information see the manual pages of crontab(5) and cron(8)
+#
+# m h  dom mon dow   command
+
+# 0 0 * * * root test ! -f $HOME/deploying.lock && /usr/local/bin/php $HOME/bin/console app:cron:delete-expired-refresh-tokens > /proc/1/fd/1 2>/proc/1/fd/2
+# 0 3 * * * root test ! -f $HOME/deploying.lock && /usr/local/bin/php $HOME/bin/console app:cron:clean-volatile-streams > /proc/1/fd/1 2>/proc/1/fd/2
+# 0 0 * * * root test ! -f $HOME/deploying.lock && /usr/local/bin/php $HOME/bin/console app:cron:delete-accounts > /proc/1/fd/1 2>/proc/1/fd/2
+# 0 * * * * root test ! -f $HOME/deploying.lock && /usr/local/bin/php $HOME/bin/console app:cron:set-chapelets-as-over > /proc/1/fd/1 2>/proc/1/fd/2
+# 0 3 * * * root test ! -f $HOME/deploying.lock && /usr/local/bin/php $HOME/bin/console app:cron:update-chapelets-activity-score > /proc/1/fd/1 2>/proc/1/fd/2
+# 0 6 * * * root test ! -f $HOME/deploying.lock && /usr/local/bin/php $HOME/bin/console app:cron:send-onboarding-push-notifications > /proc/1/fd/1 2>/proc/1/fd/2
+# 0 8 * * * root test ! -f $HOME/deploying.lock && /usr/local/bin/php $HOME/bin/console app:cron:inactivity > /proc/1/fd/1 2>/proc/1/fd/2
+# * * * * * root test ! -f $HOME/deploying.lock && /usr/local/bin/php $HOME/bin/console app:cron:send-announcements > /proc/1/fd/1 2>/proc/1/fd/2
+# * * * * * root test ! -f $HOME/deploying.lock && /usr/local/bin/php $HOME/bin/console app:cron:send-scheduled-push-notifications > /proc/1/fd/1 2>/proc/1/fd/2
+# 0 5 * * * root test ! -f $HOME/deploying.lock && /usr/local/bin/php $HOME/bin/console app:cron:chapelet:defragmenter > /proc/1/fd/1 2>/proc/1/fd/2
+# 0 18 * * * root test ! -f $HOME/deploying.lock && /usr/local/bin/php $HOME/bin/console app:cron:message:send-digest > /proc/1/fd/1 2>/proc/1/fd/2
+# 0 * * * * root test ! -f $HOME/deploying.lock && /usr/bin/php $HOME/bin/console app:cron:import-adjust-events > /proc/1/fd/1 2>/proc/1/fd/2
+# 0 0 * * * root /usr/sbin/logrotate -v -d > /proc/1/fd/1 2>/proc/1/fd/2

usr/local/bin/docker-php-entrypoint

@@ -11,7 +11,7 @@ echo "PHP          --> "$(php -v | grep -Eo "PHP [0-9\.]+" | sed 's/PHP //')
 echo "Composer     --> "$(composer --version | grep -o '\d.\d.\d ')
 
 if [ "$APP_ENV" != 'prod' ]; then
-	# Install dev convinience tools
+	# Install dev convenience tools
 	if [ -z "$(which mycli)" ]; then
 		apt update && apt install -y mycli inetutils-telnet zip
 	fi