WIP: shared filesystem for database storage (louking/webmodules#10)

Author: louking

.gitignore

@@ -158,3 +158,10 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
+
+# local configuration files
+*.json
+*.cfg
+*-config.js
+*_config.py
+config/

README.md

@@ -1,2 +1,4 @@
 # mysql-docker
 mysql, phpmyadmin, crond backups in docker compose
+
+provide docker container with services to support database

app/Dockerfile

@@ -0,0 +1,44 @@
+ARG PYTHON_VER
+
+# For more information, please refer to https://aka.ms/vscode-docker-python
+FROM python:${PYTHON_VER}-alpine
+
+# EXPOSE 5000
+
+# Keeps Python from generating .pyc files in the container
+ENV PYTHONDONTWRITEBYTECODE=1
+
+# Turns off buffering for easier container logging
+ENV PYTHONUNBUFFERED=1
+
+# set the working directory in the container
+WORKDIR /app
+
+# Install pip requirements
+COPY requirements.txt .
+
+# needed for mysqlclient package to compile
+# adapted from https://github.com/gliderlabs/docker-alpine/issues/181#issuecomment-444857401
+#   and https://github.com/gliderlabs/docker-alpine/issues/181#issuecomment-348608168
+RUN apk add --no-cache mariadb-connector-c-dev \
+    && apk add --no-cache --virtual .build-deps \
+        build-base \
+        mariadb-dev \
+    && pip install -r requirements.txt \
+    && rm -rf .cache/pip \
+    && apk del .build-deps
+
+# we'll be doing mysql, mysqldump commands
+RUN apk add --no-cache mysql-client
+
+# we're going to want to use bash
+RUN apk add bash
+
+# copy the content of the local src directory to the working directory
+# this isn't needed when developing as there's a bind under volumes: in the docker-compose.dev.yml file
+COPY src .
+
+# Creates a non-root user with an explicit UID and adds permission to access the /app folder
+# For more info, please refer to https://aka.ms/vscode-docker-python-configure-containers
+RUN chmod +x /app/dbupgrade_and_run.sh && adduser -u 5678 --disabled-password --gecos "" appuser && chown -R appuser /app
+USER appuser

app/requirements.txt

@@ -0,0 +1,41 @@
+alembic==1.11.1
+bcrypt==4.0.1
+blinker==1.6.2
+cffi==1.15.1
+click==8.1.3
+colorama==0.4.6
+cryptography==41.0.1
+decorator==5.1.1
+dnspython==2.3.0
+email-validator==2.0.0.post2
+fabric==3.1.0
+Flask==2.3.2
+Flask-Login==0.6.2
+Flask-Migrate==4.0.4
+Flask-Principal==0.4.0
+Flask-Security-Too==5.2.0
+Flask-SQLAlchemy==3.0.5
+Flask-WTF==1.1.1
+greenlet==2.0.2
+idna==3.4
+importlib-metadata==6.7.0
+importlib-resources==5.12.0
+invoke==2.1.3
+itsdangerous==2.1.2
+Jinja2==3.1.2
+loutilities==3.8.3
+Mako==1.2.4
+MarkupSafe==2.1.3
+mysqlclient==2.2.0
+paramiko==3.2.0
+passlib==1.7.4
+pycparser==2.21
+PyNaCl==1.5.0
+pytz==2023.3
+SQLAlchemy==2.0.17
+typing_extensions==4.6.3
+tzdata==2023.3
+tzlocal==5.0.1
+Werkzeug==2.3.6
+WTForms==3.0.1
+zipp==3.15.0

app/src/app-initdb.d/create-database.sh

@@ -0,0 +1,86 @@
+#!/bin/bash
+
+# NOTE: file_env, docker_process_sql, _mysql_passfile, logging functions COPIED from 
+# https://github.com/docker-library/mysql/blob/master/8.0/docker-entrypoint.sh
+
+# logging functions
+mysql_log() {
+	local type="$1"; shift
+	# accept argument string or stdin
+	local text="$*"; if [ "$#" -eq 0 ]; then text="$(cat)"; fi
+	local dt; dt="$(date -Iseconds)"
+	printf '%s [%s] [Entrypoint]: %s\n' "$dt" "$type" "$text"
+}
+mysql_note() {
+	mysql_log Note "$@"
+}
+mysql_warn() {
+	mysql_log Warn "$@" >&2
+}
+mysql_error() {
+	mysql_log ERROR "$@" >&2
+	exit 1
+}
+
+# usage: file_env VAR [DEFAULT]
+#    ie: file_env 'XYZ_DB_PASSWORD' 'example'
+# (will allow for "$XYZ_DB_PASSWORD_FILE" to fill in the value of
+#  "$XYZ_DB_PASSWORD" from a file, especially for Docker's secrets feature)
+file_env() {
+	local var="$1"
+	local fileVar="${var}_FILE"
+	local def="${2:-}"
+	if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then
+		mysql_error "Both $var and $fileVar are set (but are exclusive)"
+	fi
+	local val="$def"
+	if [ "${!var:-}" ]; then
+		val="${!var}"
+	elif [ "${!fileVar:-}" ]; then
+		val="$(< "${!fileVar}")"
+	fi
+	export "$var"="$val"
+	unset "$fileVar"
+}
+
+_mysql_passfile() {
+	# echo the password to the "file" the client uses
+	# the client command will use process substitution to create a file on the fly
+	# ie: --defaults-extra-file=<( _mysql_passfile )
+    cat <<-EOF
+        [client]
+        password="`cat /run/secrets/db-password`"
+	EOF
+    # note use of tab character above
+}
+
+# Execute sql script, passed via stdin
+#    ie: docker_process_sql <<<'INSERT ...'
+#    ie: docker_process_sql <my-file.sql
+docker_process_sql() {
+    # default mysql but caller can override
+	mysql --defaults-extra-file=<( _mysql_passfile) -uroot -hdb --comments --database=mysql "$@"
+}
+
+# follow pattern for each application database (https://stackoverflow.com/a/68714439/799921)
+file_env APP_PASSWORD
+echo "Creating database \`${APP_DATABASE}\`"
+mysql_note "Creating database \`${APP_DATABASE}\`"
+docker_process_sql <<<"CREATE DATABASE IF NOT EXISTS \`$APP_DATABASE\`;" || exit 1
+mysql_note "Creating user ${APP_USER}"
+docker_process_sql <<<"CREATE USER IF NOT EXISTS '$APP_USER'@'%' IDENTIFIED BY '$APP_PASSWORD' ;" || exit 1
+mysql_note "Giving user ${APP_USER} access to schema ${APP_DATABASE}"
+# grant access for user if not done before. see https://dba.stackexchange.com/a/105376
+docker_process_sql <<<"GRANT ALL ON \`${APP_DATABASE//_/\\_}\`.* TO '$APP_USER'@'%' ;" || exit 1
+# docker_process_sql <<<cat <<-EOF
+#     SET @sql_found='SELECT 1 INTO @x';
+#     SET @sql_fresh="GRANT ALL ON \`${APP_DATABASE//_/\\_}\`.* TO '$APP_USER'@'%' ;";
+#     SELECT COUNT(1) INTO @found_count FROM mysql.user WHERE user='$APP_USER' AND host='%';
+#     SET @sql=IF(@found_count=1,@sql_found,@sql_fresh);
+#     PREPARE s FROM @sql;
+#     EXECUTE s;
+#     DEALLOCATE PREPARE s;
+# EOF
+if [ $? -ne 0 ]; then
+  exit 1
+fi

app/src/app.py

@@ -0,0 +1,40 @@
+'''
+app.py is only used to support flask commands
+
+server execution from run.py
+'''
+
+# standard
+import os.path
+
+# pypi
+from flask_migrate import Migrate
+from mysql_docker.model import db
+
+# homegrown
+from mysql_docker import create_app, appname
+from mysql_docker.settings import Production
+from scripts import InitCli
+
+abspath = os.path.abspath('/config')
+configpath = os.path.join(abspath, f'{appname}.cfg')
+configfiles = [configpath]
+userconfigpath = os.path.join(abspath, 'users.cfg')
+# userconfigpath first so configpath can override
+configfiles.insert(0, userconfigpath)
+
+# init_for_operation=False because when we create app this would use database and cause
+# sqlalchemy.exc.OperationalError if one of the updating tables needs migration
+app = create_app(Production(configfiles), configfiles, init_for_operation=False)
+
+# set up flask command processing
+migrate_cli = Migrate(app, db, compare_type=True)
+init_cli = InitCli(app, db)
+
+## Needed only if serving web pages
+# # implement proxy fix (https://github.com/sjmf/reverse-proxy-minimal-example)
+# from werkzeug.middleware.proxy_fix import ProxyFix
+# app.wsgi_app = ProxyFix(app.wsgi_app, x_for=1, x_host=1, x_port=1, x_proto=1, x_prefix=1)
+
+# if __name__ == "__main__":
+#     app.run(host ='0.0.0.0', port=5000)

app/src/dbupgrade_and_run.sh

@@ -0,0 +1,16 @@
+#!/bin/sh
+
+# NOTE: file end of line characters must be LF, not CRLF (see https://stackoverflow.com/a/58220487/799921)
+
+# create database if necessary
+while ! ./app-initdb.d/create-database.sh
+do
+    sleep 5
+done
+
+# initial volume create may cause flask db upgrade to fail
+while ! flask db upgrade
+do
+    sleep 5
+done
+exec "$@"

app/src/migrations/README

@@ -0,0 +1 @@
+Multi-database configuration for Flask.

app/src/migrations/alembic.ini

@@ -0,0 +1,50 @@
+# A generic, single database configuration.
+
+[alembic]
+# template used to generate migration files
+# file_template = %%(rev)s_%%(slug)s
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic,flask_migrate
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[logger_flask_migrate]
+level = INFO
+handlers =
+qualname = flask_migrate
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S