Skip to content

Commit 128bbb9

Browse files
trallardtrallard
authored
DEV: general Gitpod enhancements (numpy#18822)
1 parent 65a3e7d commit 128bbb9

File tree

9 files changed

+376
-63
lines changed

9 files changed

+376
-63
lines changed

Diff for: ‎.github/workflows/docker.yml

+59
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
name: Build Base Docker Image
2+
3+
on:
4+
push:
5+
branches:
6+
- main
7+
paths:
8+
- './environment.yml'
9+
tags:
10+
- '*'
11+
12+
jobs:
13+
build:
14+
name: Build base Docker image
15+
runs-on: ubuntu-latest
16+
environment: numpy-dev
17+
if: "github.repository_owner == 'numpy' && !contains(github.event.head_commit.message, '[ci skip]') && !contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, '[skip github]')"
18+
steps:
19+
- name: Clone repository
20+
uses: actions/checkout@v2
21+
- name: Lint Docker
22+
uses: brpaz/[email protected]
23+
with:
24+
dockerfile: ./tools/gitpod/Dockerfile
25+
- name: Get refs
26+
shell: bash
27+
run: |
28+
export raw_branch=${GITHUB_REF#refs/heads/}
29+
echo "::set-output name=branch::${raw_branch//\//-}"
30+
echo "::set-output name=date::$(date +'%Y%m%d')"
31+
echo "::set-output name=sha8::$(echo ${GITHUB_SHA} | cut -c1-8)"
32+
id: getrefs
33+
- name: Set up Docker Buildx
34+
uses: docker/setup-buildx-action@v1
35+
- name: Cache Docker layers
36+
uses: actions/cache@v2
37+
with:
38+
path: /tmp/.buildx-cache
39+
key: ${{ runner.os }}-buildx-${{ github.sha }}
40+
restore-keys: ${{ runner.os }}-buildx-
41+
- name: Login to Docker Hub
42+
uses: docker/login-action@v1
43+
with:
44+
username: ${{ secrets.DOCKERHUB_USERNAME }}
45+
password: ${{ secrets.DOCKERHUB_TOKEN }}
46+
- name: Build and push
47+
id: docker_build
48+
uses: docker/build-push-action@v2
49+
with:
50+
context: "."
51+
file: "./tools/gitpod/Dockerfile"
52+
push: ${{ github.event_name != 'pull_request' }}
53+
cache-from: type=local,src=/tmp/.buildx-cache
54+
cache-to: type=local,dest=/tmp/.buildx-cache
55+
tags: |
56+
numpy/numpy-dev:${{ steps.getrefs.outputs.date }}-${{ steps.getrefs.outputs.branch}}-${{ steps.getrefs.outputs.sha8 }}, numpy/numpy-dev:latest
57+
- name: Image digest
58+
# Return details of the image build: sha and shell
59+
run: echo ${{ steps.docker_build.outputs.digest }}

Diff for: ‎.github/workflows/gitpod.yml

+55
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
name: Build Gitpod Docker image
2+
3+
on:
4+
push:
5+
branches:
6+
- main
7+
8+
jobs:
9+
build:
10+
name: Build Gitpod Docker image
11+
runs-on: ubuntu-latest
12+
environment: numpy-dev
13+
if: "github.repository_owner == 'numpy' && !contains(github.event.head_commit.message, '[ci skip]') && !contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, '[skip github]')"
14+
steps:
15+
- name: Clone repository
16+
uses: actions/checkout@v2
17+
- name: Lint Docker
18+
uses: brpaz/[email protected]
19+
with:
20+
dockerfile: ./tools/gitpod/gitpod.Dockerfile
21+
- name: Get refs
22+
shell: bash
23+
run: |
24+
export raw_branch=${GITHUB_REF#refs/heads/}
25+
echo "::set-output name=branch::${raw_branch//\//-}"
26+
echo "::set-output name=date::$(date +'%Y%m%d')"
27+
echo "::set-output name=sha8::$(echo ${GITHUB_SHA} | cut -c1-8)"
28+
id: getrefs
29+
- name: Set up Docker Buildx
30+
uses: docker/setup-buildx-action@v1
31+
- name: Cache Docker layers
32+
uses: actions/cache@v2
33+
with:
34+
path: /tmp/.buildx-cache
35+
key: ${{ runner.os }}-buildx-${{ github.sha }}
36+
restore-keys: ${{ runner.os }}-buildx-
37+
- name: Login to Docker Hub
38+
uses: docker/login-action@v1
39+
with:
40+
username: ${{ secrets.DOCKERHUB_USERNAME }}
41+
password: ${{ secrets.DOCKERHUB_TOKEN }}
42+
- name: Build and push
43+
id: docker_build
44+
uses: docker/build-push-action@v2
45+
with:
46+
context: "."
47+
file: "./tools/gitpod/gitpod.Dockerfile"
48+
push: ${{ github.event_name != 'pull_request' }}
49+
cache-from: type=local,src=/tmp/.buildx-cache
50+
cache-to: type=local,dest=/tmp/.buildx-cache
51+
tags: |
52+
numpy/numpy-gitpod:${{ steps.getrefs.outputs.date }}-${{ steps.getrefs.outputs.branch}}-${{ steps.getrefs.outputs.sha8 }}, numpy/numpy-gitpod:latest
53+
- name: Image digest
54+
# Return details of the image build: sha and shell
55+
run: echo ${{ steps.docker_build.outputs.digest }}

Diff for: ‎.gitpod.yml

+58-5
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,63 @@
1-
image: thomasjpfan/numpy-gitpod:latest
1+
# Rebuilding NumPy on init - rather than on prebuild: this ensures
2+
# that even forks do have a usable freshly built NumPy
3+
# Might delegate this later to prebuild with Q2 improvements on gitpod
4+
# https://www.gitpod.io/docs/config-start-tasks/#configuring-the-terminal
5+
# -------------------------------------------------------------------------
6+
7+
image: numpy/numpy-gitpod:latest
28
tasks:
3-
# The base image complied numpy with ccache enabled. This second build
4-
# should be faster since it is using the cache.
5-
- init: python setup.py build_ext -i
9+
- name: Prepare development
10+
init: |
11+
mkdir -p .vscode
12+
cp tools/gitpod/settings.json .vscode/settings.json
13+
conda activate numpy-dev
14+
python setup.py build_ext --inplace
15+
echo "🛠 Completed rebuilding NumPy!! 🛠 "
16+
echo "📖 Building docs 📖 "
17+
git submodule update --init
18+
cd doc
19+
make html
20+
echo "✨ Pre-build complete! You can close this terminal ✨ "
21+
22+
23+
# --------------------------------------------------------
24+
# exposing ports for liveserve
25+
ports:
26+
- port: 5500
27+
onOpen: notify
28+
29+
# --------------------------------------------------------
30+
# some useful extensions to have
31+
vscode:
32+
extensions:
33+
- eamodio.gitlens
34+
- njpwerner.autodocstring
35+
- lextudio.restructuredtext
36+
- ritwickdey.liveserver
37+
- ms-python.python
38+
- yzhang.markdown-all-in-one
39+
- bungcip.better-toml
40+
- mhutchie.git-graph
641

42+
# --------------------------------------------------------
43+
# using prebuilds for the container - note: atm this only
44+
# works for the NumPy repo
45+
# With this configuration the prebuild will happen on push to master
746
github:
847
prebuilds:
48+
# enable for main/default branch
949
master: true
10-
branches: true
50+
# enable for other branches (defaults to false)
51+
branches: false
52+
# enable for pull requests coming from this repo (defaults to true)
53+
pullRequests: false
54+
# enable for pull requests coming from forks (defaults to false)
55+
pullRequestsFromForks: false
56+
# add a check to pull requests (defaults to true)
57+
addCheck: false
58+
# add a "Review in Gitpod" button as a comment to pull requests (defaults to false)
59+
addComment: false
60+
# add a "Review in Gitpod" button to the pull request's description (defaults to false)
61+
addBadge: false
62+
# add a label once the prebuild is ready to pull requests (defaults to false)
63+
addLabel: false

Diff for: ‎.hadolint.yaml

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
---
2+
ignored:
3+
- DL3006
4+
- DL3008
5+
- SC2016
6+
- DL3004
7+
- DL3007

Diff for: ‎environment.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ dependencies:
1919
- mypy=0.812
2020
- typing_extensions
2121
# For building docs
22-
- sphinx=3.5.2
22+
- sphinx=3.5.4
2323
- numpydoc=1.1.0
2424
- ipython
2525
- scipy

Diff for: ‎tools/gitpod/Dockerfile

+84-57
Original file line numberDiff line numberDiff line change
@@ -1,74 +1,101 @@
1-
# Builds a development environment for gitpod by building numpy with
2-
# ccache enabled. When gitpod is prebuilding or starting up it clones
3-
# a branch into `/workspace/numpy`. The gitpod clone will build numpy
4-
# faster because it is using compliers with ccache enabled.
5-
FROM gitpod/workspace-base as clone
6-
7-
COPY --chown=gitpod . /tmp/numpy_repo
8-
9-
# We use a multistage build to create a shallow clone of the repo to avoid
10-
# having the complete git history in the build stage and reducing the image
11-
# size. During the build stage, the shallow clone is used to install the
12-
# dependencies and build numpy to populate the cache used by ccache. Building
13-
# numpy with setup.py uses versioneer.py which requires a git history.
14-
RUN git clone --depth 1 file:////tmp/numpy_repo /tmp/numpy
15-
16-
FROM gitpod/workspace-base as build
17-
18-
# gitpod/workspace-base needs at least one file here
19-
RUN touch /home/gitpod/.bashrc.d/empty
1+
#
2+
# Dockerfile for NumPy development
3+
#
4+
# Usage:
5+
# -------
6+
#
7+
# To make a local build of the container, from the 'Docker-dev' directory:
8+
# docker build --rm -f "Dockerfile" -t <build-tag> "."
9+
#
10+
# To use the container use the following command. It assumes that you are in
11+
# the root folder of the NumPy git repository, making it available as
12+
# /home/numpy in the container. Whatever changes you make to that directory
13+
# are visible in the host and container.
14+
# The docker image is retrieved from the NumPy dockerhub repository
15+
#
16+
# docker run --rm -it -v $(pwd):/home/numpy numpy/numpy-dev:<image-tag>
17+
#
18+
# By default the container will activate the conda environment numpy-dev
19+
# which contains all the dependencies needed for NumPy development
20+
#
21+
# To build NumPy run: python setup.py build_ext --inplace
22+
#
23+
# To run the tests use: python runtests.py
24+
#
25+
# This image is based on: Ubuntu 20.04 (focal)
26+
# https://hub.docker.com/_/ubuntu/?tab=tags&name=focal
27+
# OS/ARCH: linux/amd64
28+
FROM gitpod/workspace-base:latest
2029

2130
ARG MAMBAFORGE_VERSION="4.10.0-0"
2231
ARG CONDA_ENV=numpy-dev
2332

24-
ENV CONDA_DIR=/home/gitpod/mambaforge3
25-
ENV PATH=$CONDA_DIR/bin:$PATH
2633

34+
# ---- Configure environment ----
35+
ENV CONDA_DIR=/home/gitpod/mambaforge3 \
36+
SHELL=/bin/bash
37+
ENV PATH=${CONDA_DIR}/bin:$PATH \
38+
WORKSPACE=/workspace/numpy
39+
40+
41+
# -----------------------------------------------------------------------------
42+
# ---- Creating as root - note: make sure to change to gitpod in the end ----
2743
USER root
28-
RUN install-packages texlive-latex-extra dvisvgm
29-
USER gitpod
44+
45+
# hadolint ignore=DL3008
46+
RUN apt-get update && \
47+
apt-get install -yq --no-install-recommends \
48+
ca-certificates \
49+
dirmngr \
50+
dvisvgm \
51+
gnupg \
52+
gpg-agent \
53+
texlive-latex-extra \
54+
vim && \
55+
# this needs to be done after installing dirmngr
56+
apt-key adv --keyserver keyserver.ubuntu.com --recv-key C99B11DEB97541F0 && \
57+
apt-add-repository https://cli.github.com/packages && \
58+
apt-get install -yq --no-install-recommends \
59+
gh && \
60+
locale-gen en_US.UTF-8 && \
61+
apt-get clean && \
62+
rm -rf /var/cache/apt/* &&\
63+
rm -rf /var/lib/apt/lists/* &&\
64+
rm -rf /tmp/*
3065

3166
# Allows this Dockerfile to activate conda environments
3267
SHELL ["/bin/bash", "--login", "-o", "pipefail", "-c"]
3368

34-
# Install mambaforge3
69+
# -----------------------------------------------------------------------------
70+
# ---- Installing mamba ----
3571
RUN wget -q -O mambaforge3.sh \
36-
https://github.com/conda-forge/miniforge/releases/download/$MAMBAFORGE_VERSION/Mambaforge-$MAMBAFORGE_VERSION-Linux-x86_64.sh && \
37-
bash mambaforge3.sh -p $CONDA_DIR -b && \
72+
"https://github.com/conda-forge/miniforge/releases/download/$MAMBAFORGE_VERSION/Mambaforge-$MAMBAFORGE_VERSION-Linux-x86_64.sh" && \
73+
bash mambaforge3.sh -p ${CONDA_DIR} -b && \
3874
rm mambaforge3.sh
3975

40-
# makes conda activate command for this Dockerfile
41-
RUN echo ". $CONDA_DIR/etc/profile.d/conda.sh" >> ~/.profile
42-
# enables conda for interactive sessions
43-
RUN conda init bash
76+
# -----------------------------------------------------------------------------
77+
# ---- Copy needed files ----
78+
# basic workspace configurations
79+
COPY ./tools/gitpod/workspace_config /usr/local/bin/workspace_config
4480

45-
# Install numpy dev dependencies
46-
COPY --from=clone --chown=gitpod /tmp/numpy /workspace/numpy
47-
RUN mamba env create -f /workspace/numpy/environment.yml -n $CONDA_ENV && \
48-
conda activate $CONDA_ENV && \
49-
mamba install ccache -y && \
50-
conda clean --all -f -y
81+
RUN chmod a+rx /usr/local/bin/workspace_config && \
82+
workspace_config
5183

52-
# Set up ccache for compilers for this Dockerfile and interactino sessions
53-
# Using `conda env config vars set` does not work with Docker
54-
# REF: https://github.com/conda-forge/compilers-feedstock/issues/31
55-
RUN echo "conda activate $CONDA_ENV" >> ~/.startuprc && \
56-
echo "export CC=\"ccache \$CC\"" >> ~/.startuprc && \
57-
echo "export CXX=\"ccache \$CXX\"" >> ~/.startuprc && \
58-
echo "export F77=\"ccache \$F77\"" >> ~/.startuprc && \
59-
echo "export F90=\"ccache \$F90\"" >> ~/.startuprc && \
60-
echo "export GFORTRAN=\"ccache \$GFORTRAN\"" >> ~/.startuprc && \
61-
echo "export FC=\"ccache \$FC\"" >> ~/.startuprc && \
62-
echo "source ~/.startuprc" >> ~/.profile && \
63-
echo "source ~/.startuprc" >> ~/.bashrc
84+
# Copy conda environment file into the container - this needs to exists inside
85+
# the container to create a conda environment from it
86+
COPY environment.yml /tmp/environment.yml
6487

65-
# Build numpy to populate the cache used by ccache
66-
RUN python /workspace/numpy/setup.py build_ext -i && \
67-
ccache -s
68-
69-
# .gitpod.yml is configured to install numpy from /workspace/numpy
70-
RUN echo "export PYTHONPATH=/workspace/numpy" >> ~/.bashrc
88+
# -----------------------------------------------------------------------------
89+
# ---- Create conda environment ----
90+
# Install NumPy dependencies
91+
RUN mamba env create -f /tmp/environment.yml && \
92+
conda activate ${CONDA_ENV} && \
93+
mamba install ccache -y && \
94+
# needed for docs rendering later on
95+
python -m pip install --no-cache-dir sphinx-autobuild && \
96+
conda clean --all -f -y && \
97+
rm -rf /tmp/*
7198

72-
# gitpod will load the repository into /workspace/numpy. We remove the
73-
# directoy from the image to prevent conflicts
74-
RUN sudo rm -rf /workspace/numpy
99+
# -----------------------------------------------------------------------------
100+
# Always make sure we are not root
101+
USER gitpod

0 commit comments

Comments
 (0)