add intermediate R build for common packages (#20)

Author: shouples

.gitignore

@@ -12,6 +12,7 @@ git-wrapper.sh
 gpu.Aptfile
 gpu.requirements.in
 initial-condarc
+install_packages.R
 ipython_config.py
 secrets_helper.sh
 requirements.in
@@ -26,6 +27,9 @@ run.sh
 !python/base/Aptfile
 !python/datascience/Aptfile
 !python/noteable/Aptfile
+!r/Aptfile
+!r/datascience/Aptfile
+!r/noteable/Aptfile
 
 !python/base-gpu/gpu.Aptfile
 !python/base-gpu/environment.txt
@@ -45,5 +49,10 @@ run.sh
 !python/noteable/git_credential_helper.py
 !python/noteable/git-wrapper.sh
 
+!r/datascience/install_packages.R
+
+!r/datascience/.Rprofile
 !r/noteable/.Rprofile
+
+!r/datascience/requirements.R
 !r/noteable/requirements.R

Taskfile.R.yaml

@@ -9,31 +9,45 @@ vars:
 # directly in `cmds: []`, the variables have to be passed in explicitly.
 
 tasks:
-  core:build:
-    desc: Build the R 4.x image
-    cmds:
-      - >-
-        docker build R/{{.IDENTIFIER}}/{{.NBL_R_VERSION}} \
-          --build-arg "NBL_R_VERSION={{.NBL_R_VERSION}}" \
-          --build-arg "BASE_IMAGE={{.BASE_IMAGE}}" \
-          --tag "local/kernel-r-{{.NBL_R_VERSION}}-{{.IDENTIFIER}}:dev"
-
   base:copy-files:
     desc: Copy files from the R directory to the build directories
     cmds:
-      - task copy-root-files LANGUAGE=R IDENTIFIER={{.IDENTIFIER}} NBL_LANGUAGE_VERSION={{.NBL_R_VERSION}}
-      - task copy-language-files LANGUAGE=R IDENTIFIER={{.IDENTIFIER}} NBL_LANGUAGE_VERSION={{.NBL_R_VERSION}}
-      - cp R/noteable/.Rprofile R/noteable/{{.NBL_R_VERSION}}/.Rprofile
-      - cp R/noteable/requirements.R R/noteable/{{.NBL_R_VERSION}}/requirements.R
+      - task copy-root-files LANGUAGE=r IDENTIFIER={{.IDENTIFIER}} NBL_LANGUAGE_VERSION={{.NBL_LANGUAGE_VERSION}}
+      - task copy-language-files LANGUAGE=r IDENTIFIER={{.IDENTIFIER}} NBL_LANGUAGE_VERSION={{.NBL_LANGUAGE_VERSION}}
+      - cp r/datascience/Aptfile r/datascience/{{.NBL_LANGUAGE_VERSION}}/Aptfile
+      - cp r/datascience/install_packages.R r/datascience/{{.NBL_LANGUAGE_VERSION}}/install_packages.R
+      - cp r/datascience/requirements.R r/datascience/{{.NBL_LANGUAGE_VERSION}}/requirements.R
+      - cp r/noteable/Aptfile r/noteable/{{.NBL_LANGUAGE_VERSION}}/Aptfile
+      - cp r/noteable/.Rprofile r/noteable/{{.NBL_LANGUAGE_VERSION}}/.Rprofile
+      - cp r/noteable/requirements.R r/noteable/{{.NBL_LANGUAGE_VERSION}}/requirements.R
+
   base:build:
     desc: Build the R 4.x base image after copying required files
     cmds:
       - task r:base:copy-files IDENTIFIER=base NBL_LANGUAGE_VERSION={{.NBL_R_VERSION}}
-      - task r:core:build IDENTIFIER=base NBL_R_VERSION={{.NBL_R_VERSION}}
+      # build the base image
+      - task build LANGUAGE=r NBL_LANGUAGE_VERSION={{.NBL_R_VERSION}} IDENTIFIER=base -- --build-arg NBL_R_VERSION={{.NBL_R_VERSION}}
+
+  datascience:build:
+    desc: Build the R 4.3.x image with commonly-installed packages
+    cmds:
+      # ensure the base image is built first
+      - task r:base:build IDENTIFIER=datascience NBL_R_VERSION={{.NBL_R_VERSION}}
+      # copy datascience-specific files to the build directories
+      - cp r/datascience/Aptfile r/datascience/{{.NBL_R_VERSION}}/Aptfile
+      - cp r/datascience/install_packages.R r/datascience/{{.NBL_R_VERSION}}/install_packages.R
+      - cp r/datascience/requirements.R r/datascience/{{.NBL_R_VERSION}}/requirements.R
+      # build the datascience image off of the base image
+      - task build LANGUAGE=r NBL_LANGUAGE_VERSION={{.NBL_R_VERSION}} IDENTIFIER=datascience BUILD_TARGET=main -- --build-context base=docker-image://local/kernel-r-{{.NBL_R_VERSION}}-base:dev
 
   noteable:build:
     desc: Build the R 4.3.x image with data frame -> DEX support
     cmds:
-      - cp R/noteable/.Rprofile R/noteable/{{.NBL_R_VERSION}}/.Rprofile
-      - cp R/noteable/requirements.R R/noteable/{{.NBL_R_VERSION}}/requirements.R
-      - task r:core:build IDENTIFIER=noteable NBL_R_VERSION={{.NBL_R_VERSION}} BASE_IMAGE=local/kernel-r-{{.NBL_R_VERSION}}-base:dev
+      # ensure the datascience image is built first
+      - task r:datascience:build IDENTIFIER=noteable NBL_R_VERSION={{.NBL_R_VERSION}}
+      # copy noteable-specific files to the build directories
+      - cp r/noteable/Aptfile r/noteable/{{.NBL_R_VERSION}}/Aptfile
+      - cp r/noteable/.Rprofile r/noteable/{{.NBL_R_VERSION}}/.Rprofile
+      - cp r/noteable/requirements.R r/noteable/{{.NBL_R_VERSION}}/requirements.R
+      # build the noteable image off of the datascience image
+      - task build LANGUAGE=r NBL_LANGUAGE_VERSION={{.NBL_R_VERSION}} IDENTIFIER=noteable BUILD_TARGET=main -- --build-context base=docker-image://local/kernel-r-{{.NBL_R_VERSION}}-datascience:dev

Taskfile.python.yaml

@@ -9,21 +9,6 @@ vars:
 # directly in `cmds: []`, the variables have to be passed in explicitly.
 
 tasks:
-  core:build:
-    desc: Build the Python 3.x image
-    cmds:
-      - >-
-        docker build python/{{.IDENTIFIER}}/{{.NBL_PYTHON_VERSION}} \
-          --build-arg "NBL_PYTHON_VERSION={{.NBL_PYTHON_VERSION}}" \
-          --build-arg "BASE_IMAGE={{.BASE_IMAGE}}" \
-          --target "{{.BUILD_TARGET}}" \
-          --tag "local/kernel-python-{{.NBL_PYTHON_VERSION}}-{{.IDENTIFIER}}{{.TAG_SUFFIX}}:dev"
-    requires:
-      vars: [IDENTIFIER, NBL_PYTHON_VERSION]
-    vars:
-      BUILD_TARGET: '{{default "base" .BUILD_TARGET}}'
-      TAG_SUFFIX: '{{default "" .TAG_SUFFIX}}'
-
   base:copy-files:
     desc: Copy files from the Python directory to the build directories
     cmds:
@@ -74,7 +59,7 @@ tasks:
     desc: Build the Python 3.x base image after copying required files
     cmds:
       - task python:base:copy-files IDENTIFIER=base NBL_PYTHON_VERSION={{.NBL_PYTHON_VERSION}}
-      - task python:core:build IDENTIFIER=base NBL_PYTHON_VERSION={{.NBL_PYTHON_VERSION}}
+      - task build LANGUAGE=python NBL_LANGUAGE_VERSION={{.NBL_PYTHON_VERSION}} IDENTIFIER=base
 
   # Base GPU image
   base-gpu:lock-dependencies:
@@ -86,14 +71,16 @@ tasks:
   base-gpu:build:
     desc: Build the Python 3.x image with GPU support after copying required files
     cmds:
-      - task python:base:copy-files IDENTIFIER=base-gpu NBL_PYTHON_VERSION={{.NBL_PYTHON_VERSION}}
+      # ensure the base image is built first
+      - task python:base:build IDENTIFIER=datascience NBL_PYTHON_VERSION={{.NBL_PYTHON_VERSION}}
       # copy base-gpu specific files
+      - task python:base:copy-files IDENTIFIER=base-gpu NBL_PYTHON_VERSION={{.NBL_PYTHON_VERSION}}
       - cp python/base-gpu/environment.txt python/base-gpu/{{.NBL_PYTHON_VERSION}}/environment.txt
       - cp python/base-gpu/gpu.Aptfile python/base-gpu/{{.NBL_PYTHON_VERSION}}/gpu.Aptfile
       - cp python/base-gpu/run.sh python/base-gpu/{{.NBL_PYTHON_VERSION}}/run.sh
       - cp python/base-gpu/initial-condarc python/base-gpu/{{.NBL_PYTHON_VERSION}}/initial-condarc
-      # build off of the base image
-      - task python:core:build IDENTIFIER=base-gpu NBL_PYTHON_VERSION={{.NBL_PYTHON_VERSION}} BASE_IMAGE=local/kernel-python-{{.NBL_PYTHON_VERSION}}-base:dev
+      # build the base-gpu image off of the base image
+      - task build LANGUAGE=python NBL_LANGUAGE_VERSION={{.NBL_PYTHON_VERSION}} IDENTIFIER=base-gpu BUILD_TARGET=main -- --build-context base=docker-image://local/kernel-python-{{.NBL_PYTHON_VERSION}}-base:dev
 
   # Datascience image
   datascience:lock-dependencies:
@@ -106,8 +93,8 @@ tasks:
     cmds:
       # ensure the base image is built first
       - task python:base:build IDENTIFIER=datascience NBL_PYTHON_VERSION={{.NBL_PYTHON_VERSION}}
-      # build the datascience image
-      - task python:core:build IDENTIFIER=datascience NBL_PYTHON_VERSION={{.NBL_PYTHON_VERSION}} BASE_IMAGE=local/kernel-python-{{.NBL_PYTHON_VERSION}}-base:dev
+      # build the datascience image off of the base image
+      - task build LANGUAGE=python NBL_LANGUAGE_VERSION={{.NBL_PYTHON_VERSION}}  IDENTIFIER=datascience BUILD_TARGET=main -- --build-context base=docker-image://local/kernel-python-{{.NBL_PYTHON_VERSION}}-base:dev
 
   # Datascience GPU image
   datascience-gpu:lock-dependencies:
@@ -120,8 +107,8 @@ tasks:
     cmds:
       # ensure the base-gpu image is built first
       - task python:base-gpu:build IDENTIFIER=datascience NBL_PYTHON_VERSION={{.NBL_PYTHON_VERSION}}
-      # build the datascience-gpu image
-      - task python:core:build IDENTIFIER=datascience NBL_PYTHON_VERSION={{.NBL_PYTHON_VERSION}} BASE_IMAGE=local/kernel-python-{{.NBL_PYTHON_VERSION}}-base-gpu:dev BUILD_TARGET=gpu TAG_SUFFIX=-gpu
+      # build the datascience-gpu image off of the base-gpu image
+      - task build LANGUAGE=python NBL_LANGUAGE_VERSION={{.NBL_PYTHON_VERSION}} IDENTIFIER=datascience BUILD_TARGET=gpu TAG_SUFFIX=-gpu -- --build-context base=docker-image://local/kernel-python-{{.NBL_PYTHON_VERSION}}-base-gpu:dev
 
   # Noteable image
   noteable:copy-files:
@@ -144,9 +131,10 @@ tasks:
     cmds:
       # ensure the datascience image is built first
       - task python:datascience:build IDENTIFIER=noteable-gpu NBL_PYTHON_VERSION={{.NBL_PYTHON_VERSION}}
-      # build the noteable image after copying over noteable-specific files
+      # copy over noteable-specific files
       - task python:noteable:copy-files IDENTIFIER=noteable NBL_PYTHON_VERSION={{.NBL_PYTHON_VERSION}}
-      - task python:core:build IDENTIFIER=noteable NBL_PYTHON_VERSION={{.NBL_PYTHON_VERSION}} BASE_IMAGE=local/kernel-python-{{.NBL_PYTHON_VERSION}}-datascience:dev
+      # build the noteable image off of the datascience image
+      - task build LANGUAGE=python NBL_LANGUAGE_VERSION={{.NBL_PYTHON_VERSION}}  IDENTIFIER=noteable BUILD_TARGET=main -- --build-context base=docker-image://local/kernel-python-{{.NBL_PYTHON_VERSION}}-datascience:dev
 
   # Noteable GPU image
   noteable-gpu:lock-dependencies:
@@ -159,9 +147,10 @@ tasks:
     cmds:
       # ensure the datascience-gpu image is built first
       - task python:datascience-gpu:build IDENTIFIER=noteable-gpu NBL_PYTHON_VERSION={{.NBL_PYTHON_VERSION}}
-      # build the noteable-gpu image after copying over noteable-specific files
+      # copy over noteable-specific files
       - task python:noteable:copy-files IDENTIFIER=noteable NBL_PYTHON_VERSION={{.NBL_PYTHON_VERSION}} FILE_PREFIX="gpu."
-      - task python:core:build IDENTIFIER=noteable NBL_PYTHON_VERSION={{.NBL_PYTHON_VERSION}} BASE_IMAGE=local/kernel-python-{{.NBL_PYTHON_VERSION}}-datascience-gpu:dev BUILD_TARGET=gpu TAG_SUFFIX=-gpu
+      # build the noteable-gpu image off of the datascience-gpu image
+      - task build LANGUAGE=python NBL_LANGUAGE_VERSION={{.NBL_PYTHON_VERSION}} IDENTIFIER=noteable BUILD_TARGET=gpu TAG_SUFFIX=-gpu -- --build-context base=docker-image://local/kernel-python-{{.NBL_PYTHON_VERSION}}-datascience-gpu:dev
 
   # convenience functions for building multiple images in parallel
   base:lock-all-dependencies:

Taskfile.yaml

@@ -23,3 +23,18 @@ tasks:
     generates:
       - ./{{.LANGUAGE}}/{{.IDENTIFIER}}/{{.NBL_LANGUAGE_VERSION}}/Aptfile
       - ./{{.LANGUAGE}}/{{.IDENTIFIER}}/{{.NBL_LANGUAGE_VERSION}}/run.sh
+
+  build:
+    desc: Build the kernel image with docker buildx
+    cmds:
+      - >-
+        docker buildx build {{.LANGUAGE}}/{{.IDENTIFIER}}/{{.NBL_LANGUAGE_VERSION}} {{.CLI_ARGS}} \
+          --build-arg "NBL_LANGUAGE_VERSION={{.NBL_LANGUAGE_VERSION}}" \
+          --target "{{.BUILD_TARGET}}" \
+          --tag "local/kernel-{{.LANGUAGE}}-{{.NBL_LANGUAGE_VERSION}}-{{.IDENTIFIER}}{{.TAG_SUFFIX}}:{{.TAG}}"
+    requires:
+      vars: [LANGUAGE, IDENTIFIER, NBL_LANGUAGE_VERSION]
+    vars:
+      BUILD_TARGET: '{{default "base" .BUILD_TARGET}}'
+      TAG_SUFFIX: '{{default "" .TAG_SUFFIX}}'
+      TAG: '{{default "dev" .TAG}}'

docker-bake.hcl

@@ -27,7 +27,8 @@ variable "TIMESTAMP" {
 group "rlang_4_3_0" {
   targets = [
     "rlang_base_4_3_0",
-    "rlang_noteable_4_3_0"
+    "rlang_noteable_4_3_0",
+    "rlang_datascience_4_3_0",
   ]
 }
 
@@ -306,18 +307,31 @@ target "python_datascience_3_11_gpu" {
 # Base R 4.3.0 image
 target "rlang_base_4_3_0" {
   inherits = ["base"]
-  context = "R/base/4.3.0"
+  context = "r/base/4.3.0"
   tags = [
     "ghcr.io/noteable-io/kernel-r-4.3.0-base:${TAG}"
   ]
 }
 
+# R 4.3.0 Datascience variant
+target "rlang_datascience_4_3_0" {
+  context = "r/datascience/4.3.0"
+  contexts = {
+    base = "target:rlang_base_4_3_0"
+  }
+  target = "main"
+  tags = [
+    "ghcr.io/noteable-io/kernel-r-4.3.0-datascience:${TAG}"
+  ]
+}
+
 # R 4.3.0 Noteable variant
 target "rlang_noteable_4_3_0" {
-  context = "R/noteable/4.3.0"
+  context = "r/noteable/4.3.0"
   contexts = {
-    base = "target:rlang_base_4_3_0"
+    base = "target:rlang_datascience_4_3_0"
   }
+  target = "main"
   tags = [
     "ghcr.io/noteable-io/kernel-r-4.3.0-noteable:${TAG}"
   ]

R/Aptfile renamed to r/Aptfile

@@ -5,7 +5,7 @@
 # - no git, secrets, SQL, extensions, etc
 # ---
 ARG NBL_R_VERSION=4.3.0
-FROM r-base:${NBL_R_VERSION}
+FROM r-base:${NBL_R_VERSION} as base
 
 # User/group setup
 USER root
@@ -22,7 +22,8 @@ RUN groupadd --gid 4004 noteable && \
   --gid noteable noteable \
   --home-dir /srv/noteable && \
   chown --recursive noteable:noteable /srv/noteable && \
-  mkdir -p /etc/noteable && chown noteable:noteable /etc/noteable
+  mkdir -p /etc/noteable && chown noteable:noteable /etc/noteable && \
+  chown --recursive noteable:noteable /usr/local/lib/R/site-library
 
 # Install tini to manage passing signals to the child kernel process
 ENV TINI_VERSION v0.19.0
@@ -35,17 +36,7 @@ COPY apt-install /usr/bin/
 COPY Aptfile .
 RUN /usr/bin/apt-install Aptfile
 
-SHELL ["/bin/bash", "-o", "pipefail", "-c"]
-RUN wget -qO- https://micromamba.snakepit.net/api/micromamba/linux-64/latest | tar -xvj bin/micromamba && \
-  ./bin/micromamba shell init -s bash -p ~/micromamba
-
 USER noteable
-RUN micromamba create --name noteable-venv \
-  -c conda-forge \
-  -y \
-  r="${NBL_R_VERSION}"
-# make subsequent RUN commands use the virtualenv:
-SHELL ["micromamba", "run", "-n", "noteable-venv", "/bin/bash", "-c"]
 
 # hadolint ignore=SC2239
 RUN R -e "install.packages('IRkernel', repos='http://cran.us.r-project.org')"

R/base/4.3.0/Dockerfile renamed to r/base/4.3.0/Dockerfile

@@ -0,0 +1,19 @@
+# syntax = docker/dockerfile:1.5
+# Datascience build: adds packages based on the Jupyter r-notebook dependencies
+# (ref: https://github.com/jupyter/docker-stacks/blob/main/r-notebook/Dockerfile)
+# hadolint ignore=DL3006
+FROM base as main
+
+USER root
+
+# R package dependencies
+# hadolint ignore=DL3045
+COPY Aptfile .
+RUN /usr/bin/apt-install Aptfile
+
+USER noteable
+
+# install R packages
+COPY requirements.R /tmp/requirements.R
+COPY install_packages.R /tmp/install_packages.R
+RUN R -e "source('/tmp/install_packages.R')"

r/datascience/4.3.0/Dockerfile

@@ -0,0 +1 @@
+libgdal-dev

r/datascience/Aptfile

@@ -0,0 +1,2 @@
+packages <- readLines("/tmp/requirements.R")
+install.packages(packages, repos='http://cran.us.r-project.org')

r/datascience/install_packages.R

@@ -0,0 +1,23 @@
+arrow
+caret
+crayon
+devtools
+dplyr
+e1071
+forecast
+ggplot2
+ggpubr
+hexbin
+htmltools
+htmlwidgets
+nycflights13
+randomforest
+rcurl
+rgdal
+rmarkdown
+rodbc
+rsqlite
+shiny
+tidymodels
+tidyverse
+unixodbc

r/datascience/requirements.R

@@ -2,12 +2,16 @@
 # Noteable build: adds packages to enable Noteable-specific functionality:
 # - DEX support (via .Rprofile)
 # hadolint ignore=DL3006
-FROM base
+FROM base as main
 
-USER noteable
+USER root
+
+# install python for reticulate support
+# hadolint ignore=DL3045
+COPY Aptfile .
+RUN /usr/bin/apt-install Aptfile
 
-# Install python to use with Reticulate
-RUN micromamba install python=3.9 -y -c conda-forge
+USER noteable
 
 # R package dependencies and py_install
 COPY requirements.R /tmp/requirements.R

R/noteable/.Rprofile renamed to r/noteable/.Rprofile

@@ -0,0 +1,2 @@
+python3
+python3-venv

R/noteable/4.3.0/Dockerfile renamed to r/noteable/4.3.0/Dockerfile

@@ -1,5 +1,7 @@
 install.packages('reticulate', repos='http://cran.us.r-project.org')
 library(reticulate)
+# for whatever reason, reticulate doesn't work with the default python installation
+install_miniconda()
 # Python packages to be used in R via reticulate
 # ref: https://rstudio.github.io/reticulate/articles/python_packages.html
 py_install('pandas==1.5.3', pip=TRUE)

r/noteable/Aptfile

@@ -3,9 +3,8 @@ set -o pipefail
 set -o nounset
 set -o errexit
 
-echo "Local time: $(date)"
-
 set -x
+echo "Local time: $(date)"
 
 connection_file=/tmp/connection_file.json
 cp /etc/noteable/connections/connection_file.json ${connection_file}
@@ -20,4 +19,4 @@ echo "Done injecting Secrets, turning echoing back on"
 set -x
 
 echo "Starting R kernel"
-micromamba run -n noteable-venv R --slave -e "IRkernel::main()" --args  ${connection_file}
+exec R --slave -e "IRkernel::main()" --args  ${connection_file}