add *dev_packages workflows for python/noteable/* (#29)

Author: shouples

.gitignore

@@ -19,6 +19,9 @@ requirements.in
 requirements.R
 run.sh
 
+# Ignore these even at the parent-level directories; only to be used as placeholders for local development
+dev_packages
+
 # ...except for these places where we care about changes happening
 # (NOTE: this is because the tasks should copy the files down into the build directories)
 !scripts/apt-install

Taskfile.python.yaml

@@ -4,6 +4,7 @@ version: 3
 vars:
   NBL_PYTHON_VERSION: 3.9
   IDENTIFIER: base
+  BUILD_TARGET: main
 
 # NOTE: When using `deps: []`, variables are inherited from the current task, but when calling them
 # directly in `cmds: []`, the variables have to be passed in explicitly.
@@ -59,7 +60,7 @@ tasks:
     desc: Build the Python 3.x base image after copying required files
     cmds:
       - task python:base:copy-files IDENTIFIER=base NBL_PYTHON_VERSION={{.NBL_PYTHON_VERSION}}
-      - task build LANGUAGE=python NBL_LANGUAGE_VERSION={{.NBL_PYTHON_VERSION}} IDENTIFIER=base
+      - task build LANGUAGE=python NBL_LANGUAGE_VERSION={{.NBL_PYTHON_VERSION}} IDENTIFIER=base BUILD_TARGET=base
 
   # Base GPU image
   base-gpu:lock-dependencies:
@@ -133,8 +134,10 @@ tasks:
       - task python:datascience:build IDENTIFIER=noteable-gpu NBL_PYTHON_VERSION={{.NBL_PYTHON_VERSION}}
       # copy over noteable-specific files
       - task python:noteable:copy-files IDENTIFIER=noteable NBL_PYTHON_VERSION={{.NBL_PYTHON_VERSION}}
+      # copy dev_packages for local builds, even if we aren't using the dev stage
+      - sudo cp -R python/noteable/dev_packages python/noteable/{{.NBL_PYTHON_VERSION}}/
       # build the noteable image off of the datascience image
-      - task build LANGUAGE=python NBL_LANGUAGE_VERSION={{.NBL_PYTHON_VERSION}}  IDENTIFIER=noteable BUILD_TARGET=main -- --build-context base=docker-image://local/kernel-python-{{.NBL_PYTHON_VERSION}}-datascience:dev
+      - task build LANGUAGE=python NBL_LANGUAGE_VERSION={{.NBL_PYTHON_VERSION}} IDENTIFIER=noteable BUILD_TARGET={{.BUILD_TARGET}} -- --build-context base=docker-image://local/kernel-python-{{.NBL_PYTHON_VERSION}}-datascience:dev
 
   # Noteable GPU image
   noteable-gpu:lock-dependencies:
@@ -149,8 +152,10 @@ tasks:
       - task python:datascience-gpu:build IDENTIFIER=noteable-gpu NBL_PYTHON_VERSION={{.NBL_PYTHON_VERSION}}
       # copy over noteable-specific files
       - task python:noteable:copy-files IDENTIFIER=noteable NBL_PYTHON_VERSION={{.NBL_PYTHON_VERSION}} FILE_PREFIX="gpu."
+      # copy dev_packages for local builds, even if we aren't using the dev stage
+      - sudo cp -R python/noteable/gpu_dev_packages python/noteable/{{.NBL_PYTHON_VERSION}}/
       # build the noteable-gpu image off of the datascience-gpu image
-      - task build LANGUAGE=python NBL_LANGUAGE_VERSION={{.NBL_PYTHON_VERSION}} IDENTIFIER=noteable BUILD_TARGET=gpu TAG_SUFFIX=-gpu -- --build-context base=docker-image://local/kernel-python-{{.NBL_PYTHON_VERSION}}-datascience-gpu:dev
+      - task build LANGUAGE=python NBL_LANGUAGE_VERSION={{.NBL_PYTHON_VERSION}} IDENTIFIER=noteable BUILD_TARGET={{.BUILD_TARGET}} TAG_SUFFIX=-gpu -- --build-context base=docker-image://local/kernel-python-{{.NBL_PYTHON_VERSION}}-datascience-gpu:dev
 
   # convenience functions for building multiple images in parallel
   base:lock-all-dependencies:
@@ -251,6 +256,18 @@ tasks:
       - task: noteable:build
         vars: { NBL_PYTHON_VERSION: 3.10 }
 
+  noteable:build-all-with-dev-packages:
+    desc: LOCAL DEV - Build all `noteable` images with `dev_packages` included
+    deps:
+      - task: noteable:build
+        vars:
+          NBL_PYTHON_VERSION: 3.9
+          BUILD_TARGET: dev
+      - task: noteable:build
+        vars:
+          NBL_PYTHON_VERSION: 3.10
+          BUILD_TARGET: dev
+
   noteable-gpu:lock-all-dependencies:
     desc: Lock Python dependencies for all Python 3.x GPU builds using pip-compile
     deps:
@@ -267,6 +284,18 @@ tasks:
       - task: noteable-gpu:build
         vars: { NBL_PYTHON_VERSION: 3.10 }
 
+  noteable-gpu:build-all-with-dev-packages:
+    desc: LOCAL DEV - Build all `noteable` images with `gpu_dev_packages` included
+    deps:
+      - task: noteable-gpu:build
+        vars:
+          NBL_PYTHON_VERSION: 3.9
+          BUILD_TARGET: gpu-dev
+      - task: noteable-gpu:build
+        vars:
+          NBL_PYTHON_VERSION: 3.10
+          BUILD_TARGET: gpu-dev
+
   # convenience functions for building all images
   lock-all-dependencies:
     desc: Lock Python dependencies for all Python 3.x builds using pip-compile

python/noteable/3.10/Dockerfile

@@ -37,8 +37,32 @@ COPY ipython_config.py /etc/ipython
 COPY git_credential_helper.py /git_credential_helper.py
 COPY git-wrapper.sh /usr/local/bin/git
 
+# hadolint ignore=DL3007
+FROM main AS dev
+
+# Copy any local packages into the image for development/testing
+COPY ./dev_packages /dev_packages
+RUN if [ "$(ls /dev_packages)" ]; then \
+    pip install --no-cache-dir /dev_packages/* && \
+    python -c "import dx, noteable, psutil, sidecar_comms"; \
+    else \
+    echo "No dev_packages to install"; \
+    fi
+
 # hadolint ignore=DL3006
 FROM main as gpu
 
 COPY gpu.requirements.txt /tmp/noteable_gpu_requirements.txt
 RUN pip install --no-cache-dir -r /tmp/noteable_gpu_requirements.txt
+
+# hadolint ignore=DL3007
+FROM gpu AS gpu-dev
+
+# Copy any local packages into the image for development/testing
+COPY ./gpu_dev_packages /gpu_dev_packages
+RUN if [ "$(ls /gpu_dev_packages)" ]; then \
+    pip install --no-cache-dir /gpu_dev_packages/* && \
+    python -c "import dx, noteable, psutil, sidecar_comms"; \
+    else \
+    echo "No gpu_dev_packages to install"; \
+    fi

python/noteable/3.9/Dockerfile

@@ -37,8 +37,32 @@ COPY ipython_config.py /etc/ipython
 COPY git_credential_helper.py /git_credential_helper.py
 COPY git-wrapper.sh /usr/local/bin/git
 
+# hadolint ignore=DL3007
+FROM main AS dev
+
+# Copy any local packages into the image for development/testing
+COPY ./dev_packages /dev_packages
+RUN if [ "$(ls /dev_packages)" ]; then \
+    pip install --no-cache-dir /dev_packages/* && \
+    python -c "import dx, noteable, psutil, sidecar_comms"; \
+    else \
+    echo "No dev_packages to install"; \
+    fi
+
 # hadolint ignore=DL3006
 FROM main as gpu
 
 COPY gpu.requirements.txt /tmp/noteable_gpu_requirements.txt
 RUN pip install --no-cache-dir -r /tmp/noteable_gpu_requirements.txt
+
+# hadolint ignore=DL3007
+FROM gpu AS gpu-dev
+
+# Copy any local packages into the image for development/testing
+COPY ./gpu_dev_packages /gpu_dev_packages
+RUN if [ "$(ls /gpu_dev_packages)" ]; then \
+    pip install --no-cache-dir /gpu_dev_packages/* && \
+    python -c "import dx, noteable, psutil, sidecar_comms"; \
+    else \
+    echo "No gpu_dev_packages to install"; \
+    fi