diff --git a/.env b/.env index a2d402c..5b5d3a2 100644 --- a/.env +++ b/.env @@ -1,9 +1,26 @@ -# Add Environment Variables +# Environment variables for django/web service. +SECRET_KEY=5(15ds+i2+%ik6z&!yer+ga9m=e%jcqiz_5wszg)r-z!2--b2d +DB_NAME=django_db +DB_USER=todo +DB_PASS=pass +DB_HOST=database +# Options are: 5432 for postgres, 3306 for mysql, or available remote port for remote database. +DB_PORT=5432 +# Options are: postgres, mysql or remote for remote database. +DB_TYPE=postgres +# You must specify this if you set DB_TYPE = remote. Available options are; +# For postgres: django.db.backends.postgresql_psycopg2 +# For mysql: mysql.connector.django +# DB_ENGINE=mysql.connector.django +# For mysql +DB_ROOT_PASSWORD=password -SECRET_KEY=5(15ds+i2+%ik6z&!yer+ga9m=e%jcqiz_5wszg)r-z!2--b2d -DB_NAME=postgres -DB_USER=postgres -DB_PASS=postgres -DB_SERVICE=postgres -DB_PORT=5432 \ No newline at end of file + +# Expose ports on host, e.g. Set HOST_NGINX_PORT=80. +# You will need to change the port if already used on host. +# Note that the production deployment only exposes the database & nginx port. +HOST_WEB_PORT=0 +HOST_NGINX_PORT=0 +HOST_REDIS_PORT=0 +HOST_DATABASE_PORT=0 diff --git a/.gitignore b/.gitignore index 510d1cb..2c188ee 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ env __pycache__ .venv .cache +.sql \ No newline at end of file diff --git a/README.md b/README.md index a02831d..15b448f 100644 --- a/README.md +++ b/README.md @@ -6,14 +6,66 @@ Featuring: - Docker Compose v1.23.2 - Docker Machine v0.16.1 - Python 3.7.3 +- Postgresql +- MySQL 5.7 -Blog post -> https://realpython.com/blog/python/django-development-with-docker-compose-and-machine/ +Original Blog post -> https://realpython.com/blog/python/django-development-with-docker-compose-and-machine/ -### OS X Instructions +Building on the original repo, this repo adds the following capabilities; -1. Start new machine - `docker-machine create -d virtualbox dev;` +1. Is compatible with the original repo and simplifies running the compose stack +1. Adds flexibility by using environment variables to make the service highly configurable +1. Allows switching between postgres or mysql database. +1. Allows the use of an existing database configured by the DB_TYPE=remote & DB_* environment variables +1. Runs migrations before starting django +1. Adds live reload capability for the django code + +### General Instructions + +1. Start new machine - `docker-machine create -d virtualbox dev` 1. Configure your shell to use the new machine environment - `eval $(docker-machine env dev)` -1. Build images - `docker-compose build` -1. Start services - `docker-compose up -d` -1. Create migrations - `docker-compose run web /usr/local/bin/python manage.py migrate` +1. (Optional) Update .env to match your requirements e.g. Set HOST_NGINX_PORT=80 +1. Build and Start services - `docker-compose up -d` 1. Grab IP - `docker-machine ip dev` - and view in your browser + Example: + ```sh + $ docker-machine ip dev + 192.168.99.101 + ``` + The app will be available at http://: +1. (Optional) If you did not set HOST_NGINX_PORT i.e. HOST_NGINX_PORT=0, show Ports - `docker-compose ps` + Example: + ```sh + $ docker-compose ps + dockerizingdjango_nginx_1 /usr/sbin/nginx Up 0.0.0.0:33126->80/tcp + ``` + In this example, the app is available at http://192.168.99.101:33126. + +### Advanced Usage +To use existing remote database, change the following in .env; +1. DJANGO_DB_* variables to match your remote database +1. DB_TYPE=remote +1. Run `docker-compose up -d` + +To switch databases, change the following in .env; +1. DB_TYPE to mysql, postgres or sqlite +1. If database was already started previously, run `docker-compose stop database && docker-compose rm -f database` +1. Run `docker-compose up -d` + +To clean up, reset or start over +1. Run `docker-compose down -v`. Note that that this will delete all your containers and volumes defined in the compose file. + +### Troubleshooting +If you encounter this error + +```sh +Cannot start service web: driver failed programming external connectivity on endpoint + failed: port is already allocated +Encountered errors while bringing up the project. +``` + +* Modify your env file and change the exposed ports on host +* Alternatively, shut down services on the host that are bound to the ports +* Once done run `docker-compose up -d` + + diff --git a/docker-compose.yml b/docker-compose.yml index 85bfdbc..fb4f2da 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,47 +4,56 @@ services: web: restart: always build: ./web - expose: - - "8000" - links: - - postgres:postgres - - redis:redis + ports: + - "${HOST_WEB_PORT}:8000" volumes: - - web-django:/usr/src/app - - web-static:/usr/src/app/static + - ./web:/usr/src/app + entrypoint: ["./wait-for-it.sh", "${DB_HOST}:${DB_PORT}", "--strict", "--timeout=300", "--"] env_file: .env environment: DEBUG: 'true' - command: /usr/local/bin/gunicorn docker_django.wsgi:application -w 2 -b :8000 + DB_TYPE: ${DB_TYPE} + command: bash -c "python manage.py makemigrations && python manage.py migrate && gunicorn docker_django.wsgi:application --reload --log-level=debug -w 2 -b :8000" nginx: restart: always - build: ./nginx/ + build: ./nginx ports: - - "80:80" + - "${HOST_NGINX_PORT}:80" volumes: - - web-static:/www/static + - ./web/static:/www/static + - ./nginx/sites-enabled:/etc/nginx/sites-enabled links: - web:web - postgres: - restart: always - image: postgres:latest - ports: - - "5432:5432" - volumes: - - pgdata:/var/lib/postgresql/data/ - redis: restart: always image: redis:latest ports: - - "6379:6379" + - "${HOST_REDIS_PORT}:6379" volumes: - redisdata:/data + database: + build: ./${DB_TYPE} + image: ${DB_TYPE}_db + environment: + # POSTGRES SPECIFIC + POSTGRES_DB: ${DB_NAME} + POSTGRES_USER: ${DB_USER} + POSTGRES_PASSWORD: ${DB_PASS} + # MYSQL SPECIFIC + MYSQL_DATABASE: ${DB_NAME} + MYSQL_USER: ${DB_USER} + MYSQL_PASSWORD: ${DB_PASS} + MYSQL_ROOT_PASSWORD: ${DB_ROOT_PASSWORD} + ports: + - "${HOST_DATABASE_PORT}:${DB_PORT}" + volumes: + - mysql-vol:/var/lib/mysql + - pgdata:/var/lib/postgresql/data/ + volumes: - web-django: - web-static: + mysql-vol: pgdata: redisdata: diff --git a/mysql/Dockerfile b/mysql/Dockerfile new file mode 100644 index 0000000..30d16c5 --- /dev/null +++ b/mysql/Dockerfile @@ -0,0 +1 @@ +FROM mysql:5.7 \ No newline at end of file diff --git a/nginx/sites-enabled/django_project b/nginx/sites-enabled/django_project index 1285c6c..eabffec 100644 --- a/nginx/sites-enabled/django_project +++ b/nginx/sites-enabled/django_project @@ -5,7 +5,7 @@ server { charset utf-8; location /static { - alias /usr/src/app/static; + alias /www/static; } location / { diff --git a/postgres/Dockerfile b/postgres/Dockerfile new file mode 100644 index 0000000..0cda63a --- /dev/null +++ b/postgres/Dockerfile @@ -0,0 +1 @@ +FROM postgres:latest \ No newline at end of file diff --git a/production.yml b/production.yml index c6f5755..65fede3 100644 --- a/production.yml +++ b/production.yml @@ -4,34 +4,26 @@ services: web: restart: always build: ./web - expose: + ports: - "8000" - links: - - postgres:postgres - - redis:redis + entrypoint: ["./wait-for-it.sh", "${DB_HOST}:${DB_PORT}", "--strict", "--timeout=300", "--"] volumes: - web-static:/usr/src/app/static env_file: .env - command: /usr/local/bin/gunicorn docker_django.wsgi:application -w 2 -b :8000 + environment: + DB_TYPE: ${DB_TYPE} + command: bash -c "python manage.py makemigrations && python manage.py migrate && gunicorn docker_django.wsgi:application -w 2 -b :8000" nginx: restart: always - build: ./nginx/ + build: ./nginx ports: - - "80:80" + - "${HOST_NGINX_PORT}:80" volumes: - web-static:/www/static links: - web:web - postgres: - restart: always - image: postgres:latest - ports: - - "5432" - volumes: - - pgdata:/var/lib/postgresql/data/ - redis: restart: always image: redis:latest @@ -40,7 +32,26 @@ services: volumes: - redisdata:/data + database: + build: ./${DB_TYPE} + image: ${DB_TYPE}_db + environment: + # POSTGRES SPECIFIC + POSTGRES_DB: ${DB_NAME} + POSTGRES_USER: ${DB_USER} + POSTGRES_PASSWORD: ${DB_PASS} + # MYSQL SPECIFIC + MYSQL_DATABASE: ${DB_NAME} + MYSQL_USER: ${DB_USER} + MYSQL_PASSWORD: ${DB_PASS} + MYSQL_ROOT_PASSWORD: ${DB_ROOT_PASSWORD} + ports: + - "${HOST_DATABASE_PORT}:${DB_PORT}" + volumes: + - mysql-vol:/var/lib/mysql + - pgdata:/var/lib/postgresql/data/ + volumes: - web-static: - pgdata: + mysql-vol: redisdata: + web-static: diff --git a/remote/Dockerfile b/remote/Dockerfile new file mode 100644 index 0000000..a7a0aa8 --- /dev/null +++ b/remote/Dockerfile @@ -0,0 +1,4 @@ +# This is an idle image to dynamically replace any component if disabled. + +FROM alpine:3.7 +CMD sleep 1000000d \ No newline at end of file diff --git a/web/Dockerfile b/web/Dockerfile index dffd83b..b1c56f7 100644 --- a/web/Dockerfile +++ b/web/Dockerfile @@ -2,7 +2,9 @@ FROM python:3.7-slim RUN python -m pip install --upgrade pip +WORKDIR /usr/src/app COPY requirements.txt requirements.txt + RUN python -m pip install -r requirements.txt COPY . . diff --git a/web/docker_django/apps/todo/migrations/__init__.py b/web/docker_django/apps/todo/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/web/docker_django/settings.py b/web/docker_django/settings.py index 19ec07f..8e9f046 100644 --- a/web/docker_django/settings.py +++ b/web/docker_django/settings.py @@ -82,14 +82,26 @@ # 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), # } # } +def get_engine(): + ''' + Set database engine based on DB_TYPE environment variable. + Default Engine is 'django.db.backends.postgresql_psycopg2' + This will be overridden by specifying DB_ENGINE environment variable + ''' + db_type = os.getenv('DB_TYPE', 'postgres').lower() + if db_type in ('mysql', 'mariadb'): + return 'mysql.connector.django' + if 'sqlite' in db_type: + return 'django.db.backends.sqlite3' + return 'django.db.backends.postgresql_psycopg2' DATABASES = { 'default': { - 'ENGINE': 'django.db.backends.postgresql_psycopg2', + 'ENGINE': os.getenv('DB_ENGINE', get_engine()), 'NAME': os.environ['DB_NAME'], 'USER': os.environ['DB_USER'], 'PASSWORD': os.environ['DB_PASS'], - 'HOST': os.environ['DB_SERVICE'], + 'HOST': os.environ['DB_HOST'], 'PORT': os.environ['DB_PORT'] } } @@ -116,4 +128,4 @@ # os.path.join(BASE_DIR, 'static'), # ) # print(STATICFILES_DIRS) -STATIC_ROOT = os.path.join(BASE_DIR, 'static') +STATIC_ROOT = os.path.join(BASE_DIR, 'static') \ No newline at end of file diff --git a/web/requirements.txt b/web/requirements.txt index 004b0d3..f58ccf2 100644 --- a/web/requirements.txt +++ b/web/requirements.txt @@ -1,4 +1,5 @@ Django==2.1.7 gunicorn==19.9.0 -psycopg2==2.7.7 redis==3.2.1 +psycopg2-binary +mysql-connector-python==8.0.16 diff --git a/web/wait-for-it.sh b/web/wait-for-it.sh new file mode 100644 index 0000000..071c2be --- /dev/null +++ b/web/wait-for-it.sh @@ -0,0 +1,178 @@ +#!/usr/bin/env bash +# Use this script to test if a given TCP host/port are available + +WAITFORIT_cmdname=${0##*/} + +echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } + +usage() +{ + cat << USAGE >&2 +Usage: + $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args] + -h HOST | --host=HOST Host or IP under test + -p PORT | --port=PORT TCP port under test + Alternatively, you specify the host and port as host:port + -s | --strict Only execute subcommand if the test succeeds + -q | --quiet Don't output any status messages + -t TIMEOUT | --timeout=TIMEOUT + Timeout in seconds, zero for no timeout + -- COMMAND ARGS Execute command with args after the test finishes +USAGE + exit 1 +} + +wait_for() +{ + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + else + echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout" + fi + WAITFORIT_start_ts=$(date +%s) + while : + do + if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then + nc -z $WAITFORIT_HOST $WAITFORIT_PORT + WAITFORIT_result=$? + else + (echo > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1 + WAITFORIT_result=$? + fi + if [[ $WAITFORIT_result -eq 0 ]]; then + WAITFORIT_end_ts=$(date +%s) + echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds" + break + fi + sleep 1 + done + return $WAITFORIT_result +} + +wait_for_wrapper() +{ + # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 + if [[ $WAITFORIT_QUIET -eq 1 ]]; then + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + else + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + fi + WAITFORIT_PID=$! + trap "kill -INT -$WAITFORIT_PID" INT + wait $WAITFORIT_PID + WAITFORIT_RESULT=$? + if [[ $WAITFORIT_RESULT -ne 0 ]]; then + echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + fi + return $WAITFORIT_RESULT +} + +# process arguments +while [[ $# -gt 0 ]] +do + case "$1" in + *:* ) + WAITFORIT_hostport=(${1//:/ }) + WAITFORIT_HOST=${WAITFORIT_hostport[0]} + WAITFORIT_PORT=${WAITFORIT_hostport[1]} + shift 1 + ;; + --child) + WAITFORIT_CHILD=1 + shift 1 + ;; + -q | --quiet) + WAITFORIT_QUIET=1 + shift 1 + ;; + -s | --strict) + WAITFORIT_STRICT=1 + shift 1 + ;; + -h) + WAITFORIT_HOST="$2" + if [[ $WAITFORIT_HOST == "" ]]; then break; fi + shift 2 + ;; + --host=*) + WAITFORIT_HOST="${1#*=}" + shift 1 + ;; + -p) + WAITFORIT_PORT="$2" + if [[ $WAITFORIT_PORT == "" ]]; then break; fi + shift 2 + ;; + --port=*) + WAITFORIT_PORT="${1#*=}" + shift 1 + ;; + -t) + WAITFORIT_TIMEOUT="$2" + if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi + shift 2 + ;; + --timeout=*) + WAITFORIT_TIMEOUT="${1#*=}" + shift 1 + ;; + --) + shift + WAITFORIT_CLI=("$@") + break + ;; + --help) + usage + ;; + *) + echoerr "Unknown argument: $1" + usage + ;; + esac +done + +if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then + echoerr "Error: you need to provide a host and port to test." + usage +fi + +WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15} +WAITFORIT_STRICT=${WAITFORIT_STRICT:-0} +WAITFORIT_CHILD=${WAITFORIT_CHILD:-0} +WAITFORIT_QUIET=${WAITFORIT_QUIET:-0} + +# check to see if timeout is from busybox? +WAITFORIT_TIMEOUT_PATH=$(type -p timeout) +WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH) +if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then + WAITFORIT_ISBUSY=1 + WAITFORIT_BUSYTIMEFLAG="-t" + +else + WAITFORIT_ISBUSY=0 + WAITFORIT_BUSYTIMEFLAG="" +fi + +if [[ $WAITFORIT_CHILD -gt 0 ]]; then + wait_for + WAITFORIT_RESULT=$? + exit $WAITFORIT_RESULT +else + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + wait_for_wrapper + WAITFORIT_RESULT=$? + else + wait_for + WAITFORIT_RESULT=$? + fi +fi + +if [[ $WAITFORIT_CLI != "" ]]; then + if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then + echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess" + exit $WAITFORIT_RESULT + fi + exec "${WAITFORIT_CLI[@]}" +else + exit $WAITFORIT_RESULT +fi