Fixes arising from deployment (#49)

* Set timer variable to null before access

* Use https bokeh for CORS reasons

* Tidy up docker-compose file

* Add .env to gitignore for future updates

* Use simple node http-server for deployment

* Use gunicorn for deployment

* Remove yarn linting in lieu of pre-commit/prettier workflow

* Rename docker-compose service in CI

* Add gunicorn as a direct dev dependency

* Use correct URL for e2e testing

* Explicitly tell http-server to use localhost for e2e tests

- More attempts to force http-server to run

- Revert change to http-server options to not use localhost

* Daemonize docker-compose

* Install tree within API docker image

* Set MONGO_URI in docker-compose directly

* Remove minor versions of external GH actions and update Pipfile.lock

Author: ml-evs

.docker/app_dockerfile

@@ -2,7 +2,7 @@ FROM node:15.14.0-stretch
 SHELL ["/bin/bash", "--login", "-c"]
 
 WORKDIR /app
-CMD ["yarn", "serve"]
+RUN npm install -g http-server
 
 # copy repo contents and install deps
 #
@@ -11,3 +11,6 @@ COPY webapp/package.json /app/
 RUN yarn install
 
 COPY webapp /app
+
+RUN yarn build
+CMD [ "http-server", "dist", "--port", "8081" ]

.docker/server_dockerfile

@@ -1,13 +1,14 @@
 FROM python:3.8
 SHELL ["/bin/bash", "--login", "-c"]
+RUN apt update && apt install -y tree && apt clean
 
 # Copy all server code into workdir
 WORKDIR /app
 COPY ./pydatalab/ /app
 
 # Install Pipenv and use it to grab dependencies
 RUN pip install pipenv
-RUN pipenv install --deploy
+RUN pipenv install --deploy --dev
 
 # Run flask server
-CMD ["pipenv", "run", "python", "pydatalab/main.py"]
+CMD ["pipenv", "run", "gunicorn", "-w", "2", "--error-logfile", "logs/pydatalab_error.log", "--access-logfile", "logs/pydatalab_access.log", "-b", "0.0.0.0:5001", "pydatalab.main:create_app()"]

.github/workflows/ci.yml

@@ -11,10 +11,10 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v2.3.4
+      - uses: actions/checkout@v2
 
       - name: Set up Python 3.9
-        uses: actions/setup-python@v2.2.2
+        uses: actions/setup-python@v2
         with:
           python-version: 3.9
 
@@ -43,10 +43,10 @@ jobs:
         python-version: [3.8, 3.9]
 
     steps:
-      - uses: actions/checkout@v2.3.4
+      - uses: actions/checkout@v2
 
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2.2.2
+        uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
 
@@ -63,28 +63,24 @@ jobs:
 
       - name: Upload coverage to Codecov
         if: matrix.python-version == 3.8 && github.repository == 'the-grey-group/datalabvue'
-        uses: codecov/codecov-action@v2.0.2
+        uses: codecov/codecov-action@v2
 
   webapp:
     name: Test the webapp
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v2.3.4
+      - uses: actions/checkout@v2
 
       - name: Set up Node
         uses: actions/setup-node@v2
         with:
-          node-version: "14"
+          node-version: "15.14"
 
       - name: Install web app
         working-directory: ./webapp
         run: yarn install
 
-      - name: Lint web app
-        working-directory: ./webapp
-        run: yarn lint --no-fix --max-warnings 0
-
       - name: Unit and Component test web app
         working-directory: ./webapp
         run: yarn test:unit
@@ -97,19 +93,17 @@ jobs:
         run: docker-compose build app
 
       - name: Build the Docker server image
-        run: docker-compose build server
+        run: docker-compose build api
 
       - name: Build the Docker mongo image
         run: docker-compose build mongo
 
       - name: Start Docker images
         run: |
           docker-compose up --abort-on-container-exit &
-          # If there are no build errors after 1 minute, then exit successfully.
-          # In future, some additional tests will run here instead.
           sleep 60
           exit 0
 
       - name: Run end-to-end tests
         working-directory: ./webapp
-        run: yarn test:e2e  --headless --url http://localhost:8080
+        run: yarn test:e2e  --headless --url http://localhost:8081

.gitignore

@@ -56,6 +56,7 @@ coverage.xml
 
 # Envs
 .python-version
+.env
 .venv
 .env.local
 env/

docker-compose.yml

@@ -6,25 +6,26 @@ services:
     build:
       context: .
       dockerfile: .docker/app_dockerfile
+    depends_on:
+      - api
     volumes:
-      - ./logs:/var/logs/mongod/
-
+      - ./logs:/app/logs/
     ports:
-      - "8080:8080"
+      - "8081:8081"
 
-  server:
+  api:
     restart: always
     build:
       context: .
       dockerfile: .docker/server_dockerfile
     depends_on:
       - mongo
     volumes:
-      - sockets:/tmp/
       - ./logs:/app/logs/
-
     ports:
       - "5001:5001"
+    environment:
+      - PYDATALAB_MONGO_URI=mongodb://mongo:27017/datalabvue
 
   mongo:
     restart: always
@@ -33,9 +34,7 @@ services:
       dockerfile: .docker/mongo_dockerfile
     command: ["mongod", "-f", "/etc/mongod.conf"]
     volumes:
-      - sockets:/tmp/
       - ./logs:/var/logs/mongod/
-
     ports:
       - "27017:27017"
 

pydatalab/Pipfile

@@ -25,6 +25,7 @@ pytest-cov = "*"
 pytest-dependency = "*"
 pre-commit = "*"
 mongomock = "*"
+gunicorn = "*"
 
 [requires]
 python_version = "3"

pydatalab/Pipfile.lock

@@ -23,7 +23,9 @@ def list_remote_directories_cached():
         for d in all_directory_structures
         if d["last_updated"]
     ]
-    if len(last_update_datetimes):
+
+    seconds_since_last_update = None
+    if last_update_datetimes:
         print("Last updates from caches:")
         print(last_update_datetimes)
         seconds_since_last_update = (

pydatalab/pydatalab/routes/remotes.py

@@ -10,7 +10,7 @@
       src="https://cdn.bokeh.org/bokeh/release/bokeh-2.2.3.min.js"
       crossorigin="anonymous"
     ></script>
-    <script src="http://cdn.bokeh.org/bokeh/release/bokeh-widgets-2.2.3.min.js"></script>
+    <script src="https://cdn.bokeh.org/bokeh/release/bokeh-widgets-2.2.3.min.js"></script>
   </head>
   <body>
     <noscript>

webapp/public/index.html

@@ -99,7 +99,10 @@ export default {
       console.log(
         `loadCachedTree received, ${response_json.seconds_since_last_update} seconds out of date:`
       );
-      if (response_json.seconds_since_last_update > 3600) {
+      if (
+        response_json.seconds_since_last_update == null ||
+        response_json.seconds_since_last_update > 3600
+      ) {
         console.log("cache more than 1 hr out of date. Fetching new sample tree");
         this.fetchRemoteTree();
       }