feat(schematic): added nginx to schematic docker container (#2403)

.devcontainer/devcontainer.json

@@ -65,7 +65,7 @@
     }
   },
   "forwardPorts": [
-    2432, 3000, 3306, 4200, 4211, 5017, 5200, 5432, 5601, 7010, 7080, 7200, 7888, 8010, 8071, 8000,
+    2432, 3000, 3306, 4200, 4211, 5017, 5200, 5432, 5601, 7010, 7443, 7200, 7888, 8010, 8071, 8000,
     8080, 8081, 8082, 8083, 8084, 8085, 8086, 8090, 8091, 8092, 8200, 8787, 8888, 8889, 9090, 9104,
     9200, 9411, 27017
   ],
@@ -114,7 +114,7 @@
       "label": "schematic-api-docs",
       "onAutoForward": "silent"
     },
-    "7080": {
+    "7443": {
       "label": "schematic-api",
       "onAutoForward": "silent"
     },

apps/schematic/api/Dockerfile

@@ -1,41 +1,81 @@
-FROM python:3.10.12-slim-buster
+FROM tiangolo/uwsgi-nginx-flask:python3.10
 
-ENV APP_DIR=/opt/app
+# add label 
+LABEL org.opencontainers.image.authors='Milen Nikolov <[email protected]>, Andrew Lamb <[email protected]>, Mialy DeFelice <[email protected]>, Gianna Jordan <[email protected]>, Lingling Peng <[email protected]>'
 
-SHELL ["/bin/bash", "-euxo", "pipefail", "-c"]
+# the environment variables defined here are the default
+# and can be overwritten by docker run -e VARIABLE = XX
+# or can be overwritten by .env when using docker compose
+ENV PYTHONFAULTHANDLER=1 \
+    PYTHONUNBUFFERED=1 \
+    PYTHONHASHSEED=random \
+    PIP_NO_CACHE_DIR=off \
+    PIP_DISABLE_PIP_VERSION_CHECK=on \
+    PIP_DEFAULT_TIMEOUT=200 \
+    POETRY_VERSION=1.3.0 \
+    APP_PARENT_DIR=/app \ 
+    NGINX_CONFIG=/etc/nginx/conf.d \
+    APP_DIR=/app/app \
+    ROOT=/ \
+    UWSGI_INI=/app/uwsgi.ini \ 
+    NGINX_WORKER_PROCESSES=1 \
+    VERSION=$TAG
 
-# hadolint ignore=DL3008
-RUN apt-get update -qq -y \
-    && apt-get install --no-install-recommends -qq -y \
-        build-essential \
-        gosu \
-        libpcre3 \
-        libpcre3-dev \
-        python3-dev \
+# run open ssl and generate certificate
+RUN apt update \
     && apt-get -y autoclean \
     && apt-get -y autoremove \
-    && rm -rf /var/lib/apt/lists/*
+    && rm -rf /var/lib/apt/lists/* \
+    && apt-get update \ 
+    && apt-get install -y openssl jq 
 
-WORKDIR ${APP_DIR}
-COPY schematic_api schematic_api/
-COPY pyproject.toml poetry.lock uwsgi.ini ./
+# add dhparam.pem
+# dhparam.pem was used in ssl-params.conf
+COPY dhparam.pem /etc/ssl/dhparam.pem
+
+# copy all nginx config files
+WORKDIR ${NGINX_CONFIG}
+COPY ./self-signed.conf ./ssl-params.conf ./certificate.conf ./
+
+# copy to use custom uwsgi.ini
+COPY uwsgi.ini /app/uwsgi.ini
+
+# copy files relevant for schematic apis
+COPY schematic_api pyproject.toml poetry.lock /app/app/
 
+# install dependencies
+WORKDIR /app/app
 RUN pip install poetry \
   && poetry config --local virtualenvs.create false \
   && poetry run pip install "cython<3.0.0" \
   && poetry run pip install --no-build-isolation pyyaml==5.4.1 \
-  && poetry install --with prod --no-root --no-interaction --no-ansi \
-  && pip cache purge
+  && poetry install --with prod --no-root --no-interaction --no-ansi
 
-WORKDIR /
-COPY docker-entrypoint.sh ./
-RUN chmod +x docker-entrypoint.sh
+# Update file permission 
+RUN mkdir /root/.synapseCache /app/app/manifests
 
-EXPOSE 7080
+# temporary here to ensure .synapseCache is not empty
+RUN echo "This is a test file." > /root/.synapseCache/test.txt
+# temporary here until we move .synapseCache to a different path
+RUN chmod -R 777 /root /app
+
+# Modify entrypoint script to allow SSL private key and certificate to be saved
+WORKDIR ${ROOT}
+COPY ./uwsgi-nginx-entrypoint.sh ./entrypoint2.sh
+COPY ./uwsgi-nginx-entrypoint.sh ./uwsgi-nginx-entrypoint2.sh
+COPY ./save_key_certificate.py ./save_key_certificate.py
+
+RUN chmod +x uwsgi-nginx-entrypoint2.sh
+RUN chmod +x entrypoint2.sh
+RUN chown -R nginx /uwsgi-nginx-entrypoint2.sh
+RUN chown -R nginx /entrypoint2.sh
+
+WORKDIR ${APP_DIR}
 
-ENTRYPOINT ["/docker-entrypoint.sh"]
+# specify entrypoint again to generate config
+# have to respecify CMD too
+ENTRYPOINT ["/entrypoint2.sh"]
+CMD ["/start.sh"]
 
-# Run server in development mode
-# CMD ["python", "-m", "openapi_server"]
-# Run server in production mode
-CMD ["uwsgi", "--ini", "uwsgi.ini", "--lazy", "--http", ":7080"]
+# Expose ports
+EXPOSE 7443

apps/schematic/api/README.md

@@ -1,49 +1,46 @@
-# OpenAPI generated server
-
-## Overview
-This server was generated by the [OpenAPI Generator](https://openapi-generator.tech) project. By using the
-[OpenAPI-Spec](https://openapis.org) from a remote server, you can easily generate a server stub.  This
-is an example of building a OpenAPI-enabled Flask server.
-
-This example uses the [Connexion](https://github.com/zalando/connexion) library on top of Flask.
-
-## Requirements
-Python 3.5.2+
-
-## Usage
-To run the server, please execute the following from the root directory:
+# Run Schematic APIs 
 
+## Running without Docker
+To run the server, please execute the following from folder `apps/schematic/api`:
 ```
-pip3 install -r requirements.txt
-python3 -m schematic_api
+poetry shell
 ```
-
-and open your browser to here:
+To install dependencies: 
 
 ```
-http://localhost:8080/v1/ui/
+poetry install 
 ```
-
-Your OpenAPI definition lives here:
+And run schematic APIs: 
 
 ```
-http://localhost:8080/v1/openapi.json
+python3 -m schematic-api
 ```
+and open your browser to here:
 
-To launch the integration tests, use tox:
 ```
-sudo pip install tox
-tox
+http://127.0.0.1:7080/api/v1/ui/
 ```
 
 ## Running with Docker
 
 To run the server on a Docker container, please execute the following from the root directory:
 
 ```bash
+# Prepare the development environment of the project with nx prepare schematic-api. This will create a venv and install all the Python dependencies.
+nx prepare schematic-api
+
+# You only need to run this command one time
+# This step adds SSL private key and certificate as environment variable in .env file
+python3 apps/schematic/api/prepare_key_certificate.py
+
 # building the image
-docker build -t schematic_api .
+nx build-image schematic-api
 
-# starting up a container
-docker run -p 8080:8080 schematic_api
-```
+# Start the containerized REST API with: 
+nx serve-detach schematic-api
+```
+You could open your browser here: 
+```
+https://localhost:7443/api/v1/ui/
+```
+Note: When the OpenAPI description has changed, regenerate the REST API with nx run schematic-api:generate. Also, `dhparam.pem` was generated by using command: `RUN openssl dhparam -out dhparam.pem 4096`. If there's an issue with the `dhparam.pem` in the future, please re run the command. 

apps/schematic/api/certificate.conf

@@ -0,0 +1,28 @@
+server {
+    # listen to port 80 for http requests
+    listen 80 http2 default_server;
+    # listen to port 7443 for https requests
+    listen 7443 ssl http2 default_server;
+    listen [::]:7443 ssl http2 default_server;
+    include     /etc/nginx/conf.d/self-signed.conf;
+    include     /etc/nginx/conf.d/ssl-params.conf;
+    server_name 127.0.0.1;
+    proxy_read_timeout 300;
+    proxy_connect_timeout 300;
+    proxy_send_timeout 300;
+    error_page 497 https://$http_host$request_uri;
+    location / {
+        try_files $uri @app;
+    }
+    location @app {
+        include uwsgi_params;
+        uwsgi_pass unix:///tmp/uwsgi.sock;
+    }
+    location /static {
+        alias /app/static;
+    }
+    location /health {
+        return 200 'alive';
+        add_header Content-Type text/plain;
+    }
+}

apps/schematic/api/debug_key_cert.py

@@ -0,0 +1,76 @@
+from environs import Env
+import subprocess
+import json
+import base64
+import os
+
+# Create an instance of Env
+env = Env()
+
+# Load the .env file
+env.read_env(path=".env")
+
+# Define key and certificate file path
+test_private_key_file_path = "test_private_key.key"
+test_certificate_key_file_path = "test_certificate.crt"
+
+# Access the variables
+secret_manager_secrets = os.environ["SECRETS_MANAGER_SECRETS"]
+
+ssl_private_key = json.loads(secret_manager_secrets)["SSL_PRIVATE_KEY"]
+ssl_certificate_key = json.loads(secret_manager_secrets)["SSL_CERTIFICATE"]
+
+# delete preivous results if necessary
+if os.path.exists(test_private_key_file_path):
+    os.remove(test_private_key_file_path)
+if os.path.exists(test_certificate_key_file_path):
+    os.remove(test_certificate_key_file_path)
+
+# make sure that key and certificate can be decoded in correct format
+with open(test_private_key_file_path, "wb") as file:
+    decoded_private_key = base64.b64decode(ssl_private_key)
+    file.write(decoded_private_key)
+
+with open(test_certificate_key_file_path, "wb") as file:
+    decoded_ssl_certificate_key = base64.b64decode(ssl_certificate_key)
+    file.write(decoded_ssl_certificate_key)
+
+
+# Make sure that certificate and key match each other
+def get_md5_cert(file):
+    openssl_x509_command = ["openssl", "x509", "-noout", "-modulus", "-in", file]
+    openssl_md5_command = ["openssl", "md5"]
+
+    x509_process = subprocess.Popen(openssl_x509_command, stdout=subprocess.PIPE)
+    md5_process = subprocess.Popen(
+        openssl_md5_command, stdin=x509_process.stdout, stdout=subprocess.PIPE
+    )
+
+    output, error = md5_process.communicate()
+
+    if error:
+        print("error getting md5", error.decode("utf-8"))
+
+    return output.decode("utf-8").strip()
+
+
+def get_md5_private_key(file):
+    openssl_rsa_command = ["openssl", "rsa", "-noout", "-modulus", "-in", file]
+    openssl_md5_command = ["openssl", "md5"]
+
+    rsa_process = subprocess.Popen(openssl_rsa_command, stdout=subprocess.PIPE)
+    md5_process = subprocess.Popen(
+        openssl_md5_command, stdin=rsa_process.stdout, stdout=subprocess.PIPE
+    )
+
+    output, error = md5_process.communicate()
+
+    if error:
+        print(error.decode("utf-8"))
+    return output.decode("utf-8").strip()
+
+
+md5_key = get_md5_private_key(test_private_key_file_path)
+md5_cert = get_md5_cert(test_certificate_key_file_path)
+
+assert md5_key == md5_cert

apps/schematic/api/dhparam.pem

@@ -0,0 +1,13 @@
+-----BEGIN DH PARAMETERS-----
+MIICCAKCAgEAwzcUARDhjqW74+OsC8quGIJhkuL60o9GHuR3feEOky1cY1D/Qm0I
+s32FFsm5PutkJ6ZUFf86Uae+ARrQlsidsryHF+bHZpIBe12h/4pcQhH9ghdmHcq6
+lqJiD55Gb+uzqe/rmOCInEjl7WXxLvYDzndRbP9gY7I5tVQCdfMTYm7ZYK+Xt53X
+ufIkwwkxj3qXimFIEeTJEqK837u7VM9Q5H+nZR+W/lAw8IvYp3wH3IrOmBflWBi/
+AaHCemd+gwaS8nZFrNSWffOd9Gg+tuFehlnCSMb4FudEbLk+AyvDAq1RMI1bH1SO
++go/i/iX3u616eXi83/U7JUMNj655Iyoc5F02GlDjyvRRauV50S4nIB7t/mxgGEZ
+B7C1wce23PwhsRLxsT5xlti7T3QWgvO0w/P+jnCvwfyu9jUzP87qozAYAV8jCMzx
+Henya27o3Qewhr6IuMm7tqo5Bz28AJMm+/DL+XQfF4ceP4XzA4OJVKfQzDKXh4PI
+BSw8qY85esWJ3yQjwITygdOMHIxRVQA7Et7kKee3D3iDKeHdRu1m4hLtqRbHesrc
+QMdhbZBW6WwYQPeqDhkYKBFbAFrrQHkNAOsduq6/OpWoOuu4yL4K4Sfkd8wQ/3At
+e6lfZimHDEHlCIfKM3+MS91zkeOHlBVFVI+H2LiBk6tVcPm1hSK9c+sCAQI=
+-----END DH PARAMETERS-----

apps/schematic/api/docker-compose.yml

@@ -1,4 +1,4 @@
-version: "3.8"
+version: "3.10"
 
 services:
   schematic-api:
@@ -13,7 +13,7 @@ services:
     networks:
       - schematic
     ports:
-      - "${SERVER_PORT}:7080"
+      - "7443:7443"
 
 # volumes:
 #   openchallenges-mariadb: