dumping DB + bug fix (#695)

robkooper · web-flow · commit 4f4a7b452caf · 2020-04-10T14:26:47.000-05:00
* script to dump database * docker push + cron * only push to dockerhub if username/password set * only run on PecanProject/bety * script to easily add users to BETY * make sure to use environment variable for secret The environment variable SECRET_KEY_BASE was ignored. This fixes #696. If you had SECRET_KEY_BASE set, please unset it now, otherwise you can no longer login. load-bety.sh script now uses script/betyuser.sh to add users with correct password.
diff --git a/.github/workflows/dbdump.yml b/.github/workflows/dbdump.yml
@@ -0,0 +1,68 @@
+name: DB Dump
+
+on:
+  schedule:
+    - cron: '0 0 * * *'
+
+jobs:
+  sync:
+    if: github.repository == 'PecanProject/bety'
+
+    runs-on: ubuntu-latest
+
+    services:
+      postgres:
+        image: mdillon/postgis:9.5
+        ports: 
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Setup Database
+        run: |
+          psql -h localhost -U postgres -c "CREATE ROLE bety WITH LOGIN CREATEDB NOSUPERUSER NOCREATEROLE PASSWORD 'bety'"
+          psql -h localhost -U postgres -c "CREATE DATABASE bety WITH OWNER bety"
+          psql -h localhost -U postgres -d bety -c "CREATE EXTENSION postgis;"
+
+      - name: Sync with EBI
+        run: script/load.bety.sh -a "postgres" -p "-h localhost" -d "bety" -o bety -m 99 -r 0 -c -w https://ebi-forecast.igb.illinois.edu/pecan/dump/bety.tar.gz
+
+      - name: Sync with BU
+        run: script/load.bety.sh -a "postgres" -p "-h localhost" -d "bety" -o bety -m 99 -r 1
+
+      - name: Sync with BNL
+        run: script/load.bety.sh -a "postgres" -p "-h localhost" -d "bety" -o bety -m 99 -r 2 -w ftp://anon:anon@ftp.test.bnl.gov/outgoing/betydb/bety.tar.gz
+
+      - name: Sync with Wisconsin
+        run: script/load.bety.sh -a "postgres" -p "-h localhost" -d "bety" -o bety -m 99 -r 5 -w http://tree.aos.wisc.edu:6480/sync/dump/bety.tar.gz
+
+      - name: Dump Database
+        run: pg_dump -h localhost -U postgres -F c bety > initdb/db.dump
+
+      - name: Build Docker with Database dump
+        run: |
+          cd initdb
+          docker build --tag image --file Dockerfile .
+
+      - name: Login into registry
+        run: |
+          echo "${{ secrets.GITHUB_TOKEN }}" | docker login docker.pkg.github.com -u ${{ github.actor }} --password-stdin
+          if [ -n "${{ secrets.DOCKERHUB_USERNAME }}" -a -n "${{ secrets.DOCKERHUB_PASSWORD }}"]; then
+            echo "${{ secrets.DOCKERHUB_PASSWORD }}" | docker login -u ${{ secrets.DOCKERHUB_USERNAME }} --password-stdin
+          fi
+
+      - name: Push docker image
+        run: |
+          for T in $(date +'%Y-%V') 'latest'; do
+            docker tag image docker.pkg.github.com/${{ github.repository }}/db:$T
+            docker push docker.pkg.github.com/${{ github.repository }}/db:$T
+            if [ -n "${{ secrets.DOCKERHUB_USERNAME }}" -a -n "${{ secrets.DOCKERHUB_PASSWORD }}"]; then
+              docker tag image pecan/db:$T
+              docker push pecan/db:$T
+            fi
+          done
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -24,7 +24,9 @@ jobs:
       - name: Login into registry
         run: |
           echo "${{ secrets.GITHUB_TOKEN }}" | docker login docker.pkg.github.com -u ${{ github.actor }} --password-stdin
-          echo "${{ secrets.DOCKERHUB }}" | docker login -u kooper --password-stdin
+          if [ -n "${{ secrets.DOCKERHUB_USERNAME }}" -a -n "${{ secrets.DOCKERHUB_PASSWORD }}" ]; then
+            echo "${{ secrets.DOCKERHUB_PASSWORD }}" | docker login -u ${{ secrets.DOCKERHUB_USERNAME }} --password-stdin
+          fi
       - name: Push image
         run: |
           # image all lowercase
@@ -62,6 +64,8 @@ jobs:
           for T in $TAGS; do
             docker tag image $IMAGE_ID:$T
             docker push $IMAGE_ID:$T
-            docker tag image pecan/bety:$T
-            docker push pecan/bety:$T
+            if [ -n "${{ secrets.DOCKERHUB_USERNAME }}" -a -n "${{ secrets.DOCKERHUB_PASSWORD }}" ]; then
+              docker tag image pecan/bety:$T
+              docker push pecan/bety:$T
+            fi
           done
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -20,6 +20,7 @@ In this case you can simply pull down the version you want, and run `docker-comp
 ## [Unreleased]
 
 ### Fixes
+- #696 : actually now uses environment variable SECRET_KEY_BASE. If this was set, unset otherwise you can not login to BETY.
 - #551 : remove GUnload message from console on page change.
 - #672 : Added activemodel-serializers-xml Gem to restore functionality of "original" API XML endpoints.
 - #674 : Upgraded comma Gem to restore CSV file downloads.
@@ -30,8 +31,10 @@ In this case you can simply pull down the version you want, and run `docker-comp
 - upgrade nokogiri to 1.10.8 (dependbot fix)
 
 ### Added
+- script to add new users to bety
 - dockerfile to dump database in docker image for faster restore of database.
 - switched to github actions instead of Travis
+- cronjob on github actions to create pecan/db docker image nightly, will safe also with year-weak (i.e 2020-15)
 
 ## [5.2.2] - 2019-12-06
 
diff --git a/Dockerfile b/Dockerfile
@@ -45,7 +45,7 @@ ENV LOCAL_SERVER=99 \
     INITIALIZE_URL="-w https://ebi-forecast.igb.illinois.edu/pecan/dump/all/bety.tar.gz" \
     RAILS_ENV="production" \
     RAILS_RELATIVE_URL_ROOT="" \
-    SECRET_KEY_BASE="ThisIsNotReallySuchAGreatSecret" \
+    SECRET_KEY_BASE="thisisnotasecret" \
     UNICORN_WORKER_PROCESSES="3" \
     UNICORN_PORT="8000" \
     BETY_GIT_TAGS=${BETY_GIT_TAGS} \
diff --git a/config/application.rb b/config/application.rb
@@ -16,9 +16,9 @@ class Application < Rails::Application
     using SymbolizeHelper
 
     # Define top-level Hash constant CONFIG by merging settings in defaults.yml and application.yml.
-    ::CONFIG = YAML.load(File.read(File.expand_path('../defaults.yml', __FILE__))).deep_symbolize_keys
+    ::CONFIG = YAML.load(ERB.new(File.read(File.expand_path('../defaults.yml', __FILE__))).result).deep_symbolize_keys
     if File.exists?(File.expand_path('../application.yml', __FILE__))
-      customizations = YAML.load(File.read(File.expand_path('../application.yml', __FILE__))).deep_symbolize_keys
+      customizations = YAML.load(ERB.new(File.read(File.expand_path('../application.yml', __FILE__))).result).deep_symbolize_keys
       ::CONFIG.update customizations
       ::CONFIG.merge! CONFIG.fetch(Rails.env, {})
     end
diff --git a/config/defaults.yml b/config/defaults.yml
@@ -95,4 +95,4 @@ homepage_body:
 show_crop_map_links: true
 
 # Override this with a secret key to run a secure site:
-rest_auth_site_key: 'thisisnotasecret'
+rest_auth_site_key: <%= ENV["SECRET_KEY_BASE"] || 'thisisnotasecret' %>
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -10,7 +10,7 @@ services:
       - 8000
     environment:
       - UNICORN_WORKER_PROCESSES=1
-      - SECRET_KEY_BASE=thisissomereallllllylongsecretkeyandshouldbelongerthanthis
+      - SECRET_KEY_BASE=thisisnotasecret
     depends_on:
       - postgres
     restart: unless-stopped
diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh
@@ -15,6 +15,10 @@ case $1 in
             ./script/load.bety.sh -a "postgres" -d "bety" -p "-h postgres -p 5432" -o bety -r ${r}
         done
         ;;
+    "fix" )
+        echo "Fixing database ID"
+        ./script/load.bety.sh -a "postgres" -d "bety" -p "-h postgres -p 5432" -o bety -f -m ${LOCAL_SERVER}  -r -1
+        ;;
     "dump" )
         echo "Dump data from server ${LOCAL_SERVER}"
         ./script/dump.bety.sh -d "bety" -p "-h postgres -p 5432 -U postgres" -m ${LOCAL_SERVER} -o dump
@@ -65,6 +69,10 @@ case $1 in
         echo "Start running BETY (unicorn)"
         exec bundle exec unicorn -c config/unicorn.rb
         ;;
+    "user" )
+        shift
+        ./script/betyuser.sh "$@"
+        ;;
     "help" )
         echo "initialize : create a new database and initialize with all data from server 0"
         echo "sync       : synchronize database with remote servers ${REMOTE_SERVERS}"
@@ -78,6 +86,7 @@ case $1 in
         echo "vacuum-all : maintenance: vaccum the entire database (not VACUUM FULL)"
         echo "vacuum-full: maintenance: full vaccum of the database. Specify rarely, if ever"
         echo "autoserver : runs the server (using unicorn) after running a migrate"
+        echo "user       : add a new user to BETY database"
         echo "help       : this text"
         echo ""
         echo "Default is to run bety using unicorn. no automatic migrations."
diff --git a/initdb/Dockerfile b/initdb/Dockerfile
@@ -1,8 +1,8 @@
 # syntax = docker/dockerfile:experimental
 FROM alpine
 
-RUN apk --no-cache add postgresql \
-    && pg_dump -h postgres -U postgres -F c bety > /db.dump
-COPY initdb.sh /
+RUN apk --no-cache add postgresql
+ADD initdb.sh db.dump* /
+RUN if [ ! -e /db.dump ]; then pg_dump -F c ${PGDATABASE} > /db.dump; fi
 
 CMD /initdb.sh
diff --git a/initdb/initdb.sh b/initdb/initdb.sh
@@ -6,3 +6,11 @@ psql -h postgres -p 5432 -U postgres -c "CREATE ROLE bety WITH LOGIN CREATEDB NO
 
 # load database from dump
 pg_restore -c -C -v -h postgres -U postgres -d postgres -F c /db.dump
+
+# print some hints on what to do next
+echo ""
+echo "To fix the database id to be 77 instead of the default of 99:"
+echo "docker-compose run -e LOCAL_SERVER=77 bety fix"
+echo ""
+echo "To add a user, you can use:"
+echo "docker-compose run bety user 'login' 'password' 'full name' 'email' 1 1"
diff --git a/script/betyuser.sh b/script/betyuser.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+
+if [ $# != 6 ]; then
+  echo "$0 username password fullname email data_access page_access"
+  echo "data access : 1=Restricted, 2=Internal, 3=External, 4=Public"
+  echo "page_access : 1=Admin,      2=Manager,  3=Creator,  4=Viewer"
+  exit 1
+fi
+
+LOGIN="$1"
+PASSWORD="$2"
+NAME="$3"
+EMAIL="$4"
+ACCESS=${5:-2}
+PAGE=${6:-2}
+
+SALT="${LOGIN}"
+
+COUNT=10
+
+DIGEST="${SECRET_KEY_BASE}"
+for x in $(seq ${COUNT}); do
+  DIGEST=$(echo -n "${DIGEST}--${SALT}--${PASSWORD}--${SECRET_KEY_BASE}" | sha1sum | awk '{print $1}')
+done
+
+psql -q -h postgres -U bety -t -c "INSERT INTO users (login, name, email, crypted_password, salt, access_level, page_access_level, created_at, updated_at) VALUES ('${LOGIN}', '${NAME}', '${EMAIL}', '${DIGEST}', '${SALT}', ${ACCESS}, ${PAGE}, NOW(), NOW())"
+
+if [ $? == 0 ]; then
+  echo "User ($LOGIN) has been added to database"
+else
+  echo "Could not add user to database"
+fi
diff --git a/script/load.bety.sh b/script/load.bety.sh
@@ -238,27 +238,29 @@ fi
 
 # check the database for information first
 if [ -z "${DUMPURL}" ]; then
-  DUMPURL=$(psql ${PG_OPT} ${PG_USER} -q -d "${DATABASE}" -t -c "SELECT sync_url FROM machines WHERE sync_host_id = ${REMOTESITE};" | xargs )
+  DUMPURL=$(psql ${PG_OPT} ${PG_USER} -q -d "${DATABASE}" -t -c "SELECT sync_url FROM machines WHERE sync_host_id = ${REMOTESITE};" 2>/dev/null | xargs )
 fi
 if [ -z "${DUMPURL}" ]; then
-  echo "Did not find a sync_url in database, please update database, or provide script with DUMPURL."
-  exit -1
+  if [ "${FIXSEQUENCE}" != "YES" -o "${CREATE}" == "YES" ]; then
+    echo "Did not find a sync_url in database, please update database, or provide script with DUMPURL."
+    exit -1
+  fi
 fi
 
-MY_START_ID=$(psql ${PG_OPT} ${PG_USER} -q -d "${DATABASE}" -t -c "SELECT sync_start FROM machines WHERE sync_host_id = ${MYSITE};" | xargs )
+MY_START_ID=$(psql ${PG_OPT} ${PG_USER} -q -d "${DATABASE}" -t -c "SELECT sync_start FROM machines WHERE sync_host_id = ${MYSITE};" 2>/dev/null | xargs )
 if [ -z "${MY_START_ID}" ]; then
   MY_START_ID=$(( MYSITE * ID_RANGE + 1 ))
 fi
-MY_LAST_ID=$(psql ${PG_OPT} ${PG_USER} -q -d "${DATABASE}" -t -c "SELECT sync_end FROM machines WHERE sync_host_id = ${MYSITE};" | xargs )
+MY_LAST_ID=$(psql ${PG_OPT} ${PG_USER} -q -d "${DATABASE}" -t -c "SELECT sync_end FROM machines WHERE sync_host_id = ${MYSITE};" 2>/dev/null | xargs )
 if [ -z "${MY_LAST_ID}" ]; then
   MY_LAST_ID=$(( MY_START_ID + ID_RANGE - 2 ))
 fi
 
-REM_START_ID=$(psql ${PG_OPT} ${PG_USER} -q -d "${DATABASE}" -t -c "SELECT sync_start FROM machines WHERE sync_host_id = ${REMOTESITE};" | xargs )
+REM_START_ID=$(psql ${PG_OPT} ${PG_USER} -q -d "${DATABASE}" -t -c "SELECT sync_start FROM machines WHERE sync_host_id = ${REMOTESITE};" 2>/dev/null | xargs )
 if [ -z "${REM_START_ID}" ]; then
-  REM_START_ID=$(( MYSITE * ID_RANGE + 1 ))
+  REM_START_ID=$(( REMOTESITE * ID_RANGE + 1 ))
 fi
-REM_LAST_ID=$(psql ${PG_OPT} ${PG_USER} -q -d "${DATABASE}" -t -c "SELECT sync_end FROM machines WHERE sync_host_id = ${REMOTESITE};" | xargs )
+REM_LAST_ID=$(psql ${PG_OPT} ${PG_USER} -q -d "${DATABASE}" -t -c "SELECT sync_end FROM machines WHERE sync_host_id = ${REMOTESITE};" 2>/dev/null | xargs )
 if [ -z "${REM_LAST_ID}" ]; then
   REM_LAST_ID=$(( REM_START_ID + ID_RANGE - 2 ))
 fi
@@ -454,25 +456,11 @@ if [ "${USERS}" == "YES" ]; then
   echo "SELECT count(id) FROM users WHERE login='carya';" >&3 && read RESULT <&4
   if [ ${RESULT} -eq 0 ]; then
     echo "SELECT nextval('users_id_seq');" >&3 && read ID <&4
-    echo "INSERT INTO users (login, name, email, crypted_password, salt, city, state_prov, postal_code, country, area, access_level, page_access_level, created_at, updated_at, apikey, remember_token, remember_token_expires_at) VALUES ('carya', 'carya', 'betydb+${ID}@gmail.com', 'df8428063fb28d75841d719e3447c3f416860bb7', 'carya', 'Urbana', 'IL', '61801', 'USA', '', 1, 1, NOW(), NOW(), NULL, NULL, NULL);" >&3
+    $(dirname $0)/betyuser.sh "carya" "illinois" "carya" "betydb+${ID}@gmail.com" 1 1
     if [ "${QUIET}" != "YES" ]; then
       echo "Added carya with admin privileges with id=${ID}"
     fi
   fi
-
-  # add other users with specific rights
-  for f in 1 2 3 4; do
-    for g in 1 2 3 4; do
-      echo "SELECT count(id) FROM users WHERE login='carya${f}${g}';" >&3 && read RESULT <&4
-      if [ ${RESULT} -eq 0 ]; then
-        echo "SELECT nextval('users_id_seq');" >&3 && read ID <&4
-        echo "INSERT INTO users (login, name, email, crypted_password, salt, city, state_prov, postal_code, country, area, access_level, page_access_level, created_at, updated_at, apikey, remember_token, remember_token_expires_at) VALUES ('carya${f}${g}', 'carya${f}${g}', 'betydb+${ID}@gmail.com', 'df8428063fb28d75841d719e3447c3f416860bb7', 'carya', 'Urbana', 'IL', '61801', 'USA', '', $f, $g, NOW(), NOW(), NULL, NULL, NULL);" >&3
-        if [ "${QUIET}" != "YES" ]; then
-          echo "Added carya$f$g with access_level=$f and page_access_level=$g with id=${ID}"
-        fi
-      fi
-    done
-  done
 fi
 
 # Add guest user
@@ -481,7 +469,7 @@ if [ "${GUESTUSER}" == "YES" ]; then
   echo "SELECT count(id) FROM users WHERE login='guestuser';" >&3 && read RESULT <&4
   if [ ${RESULT} -eq 0 ]; then
     echo "SELECT nextval('users_id_seq');" >&3 && read ID <&4
-    echo "INSERT INTO users (login, name, email, crypted_password, salt, city, state_prov, postal_code, country, area, access_level, page_access_level, created_at, updated_at, apikey, remember_token, remember_token_expires_at) VALUES ('guestuser', 'guestuser', 'betydb+${ID}@gmail.com', '994363a949b6486fc7ea54bf40335127f5413318', 'bety', 'Urbana', 'IL', '61801', 'USA', '', 4, 4, NOW(), NOW(), NULL, NULL, NULL);" >&3
+    $(dirname $0)/betyuser.sh "guestuser" "guestuser" "guestuser" "betydb+${ID}@gmail.com" 4 4
     if [ "${QUIET}" != "YES" ]; then
       echo "Added guestuser with access_level=4 and page_access_level=4 with id=${ID}"
     fi
