Merge pull request #100 from cluster-apps-on-docker/feature/refactor-compose

[Feature] Refactor compose file

Author: andre-marcos-perez

CONTRIBUTING.md

@@ -23,4 +23,4 @@ parallel computing in distributed environments through our projects. :sparkles:
 - [x] JupyterLab R kernel;
 - [x] Jupyter notebook with Apache Spark R API examples;
 - [ ] Test coverage;
-- [ ] Ever growing examples.
+- [ ] Ever-growing examples.

README.md

@@ -9,7 +9,7 @@
 This project gives you an **Apache Spark** cluster in standalone mode with a **JupyterLab** interface built on top of **Docker**.
 Learn Apache Spark through its **Scala**, **Python** (PySpark) and **R** (SparkR) API by running the Jupyter [notebooks](build/workspace/) with examples on how to read, process and write data.
 
-<p align="center"><img src="docs/image/cluster-architecture.png"></p>
+<p align="center"><img alt="cluster-architecture" src="docs/image/cluster-architecture.png"></p>
 
 ![build-master](https://github.com/cluster-apps-on-docker/spark-standalone-cluster-on-docker/workflows/build-master/badge.svg)
 ![sponsor](https://img.shields.io/badge/patreon-sponsor-ff69b4)
@@ -22,7 +22,7 @@ Learn Apache Spark through its **Scala**, **Python** (PySpark) and **R** (SparkR
 ## TL;DR
 
 ```bash
-curl -LO https://raw.githubusercontent.com/cluster-apps-on-docker/spark-standalone-cluster-on-docker/master/docker-compose.yml
+curl -LO https://raw.githubusercontent.com/cluster-apps-on-docker/spark-standalone-cluster-on-docker/master/assets/docker-compose.yml
 docker-compose up
 ```
 
@@ -53,13 +53,13 @@ docker-compose up
 
 ### Download from Docker Hub (easier)
 
-1. Download the [docker compose](docker-compose.yml) file;
+1. Download the [docker compose](assets/docker-compose.yml) file;
 
 ```bash
-curl -LO https://raw.githubusercontent.com/cluster-apps-on-docker/spark-standalone-cluster-on-docker/master/docker-compose.yml
+curl -LO https://raw.githubusercontent.com/cluster-apps-on-docker/spark-standalone-cluster-on-docker/master/assets/docker-compose.yml
 ```
 
-2. Edit the [docker compose](docker-compose.yml) file with your favorite tech stack version, check **apps** [supported versions](#tech-stack);
+2. Edit the [docker compose](assets/docker-compose.yml) file with your favorite tech stack version, check **apps** [supported versions](#tech-stack);
 3. Start the cluster;
 
 ```bash
@@ -110,16 +110,16 @@ docker-compose up
 
 - Languages
 
-| Spark | Hadoop | Scala   | [Scala Kernel](https://almond.sh/) | Python | R     |
-|-------|--------|---------|------------------------------------|--------|-------|
-| 3.5.7 | 3      | 2.12.20 | 0.14.2                             | 3.12.3 | 4.3.3 |
+| Spark | Hadoop | Scala   | Python | R     |
+|-------|--------|---------|--------|-------|
+| 3.5.7 | 3      | 2.12.20 | 3.12.3 | 4.3.3 |
 
 - Apps
 
-| Component    | Version | Docker Tag                                           |
-|--------------|---------|------------------------------------------------------|
-| Apache Spark | 3.5.7   | **\<spark-version>**                                 |
-| JupyterLab   | 4.4.10  | **\<jupyterlab-version>**-spark-**\<spark-version>** |
+| Component    | Version | Docker Tag         |
+|--------------|---------|--------------------|
+| Apache Spark | 3.5.7   | 3.5.7              |
+| JupyterLab   | 4.4.10  | 4.4.10-spark-3.5.7 |
 
 ## <a name="metrics"></a>Metrics
 

docker-compose.yml assets/docker-compose.ymldocker-compose.yml renamed to assets/docker-compose.yml

@@ -8,22 +8,25 @@ volumes:
     driver: local
 services:
   jupyterlab:
+    pull_policy: missing
     image: andreper/jupyterlab:4.4.10-spark-3.5.7
     container_name: jupyterlab
     ports:
       - "8888:8888"
+      - "4040:4040"
     volumes:
       - shared-workspace:/opt/workspace
   spark-master:
+    pull_policy: missing
     image: andreper/spark-master:3.5.7
     container_name: spark-master
     ports:
       - "8080:8080"
       - "7077:7077"
-      - "4040:4040"
     volumes:
       - shared-workspace:/opt/workspace
   spark-worker-1:
+    pull_policy: missing
     image: andreper/spark-worker:3.5.7
     container_name: spark-worker-1
     environment:
@@ -36,6 +39,7 @@ services:
     depends_on:
       - spark-master
   spark-worker-2:
+    pull_policy: missing
     image: andreper/spark-worker:3.5.7
     container_name: spark-worker-2
     environment:
@@ -47,4 +51,4 @@ services:
       - shared-workspace:/opt/workspace
     depends_on:
       - spark-master
-...
+...

build/build.sh

@@ -12,12 +12,11 @@ SHOULD_BUILD_BASE="$(grep -m 1 build_base build.yml | sed -E 's/.*"([^"]*)".*/\1
 SHOULD_BUILD_SPARK="$(grep -m 1 build_spark build.yml | sed -E 's/.*"([^"]*)".*/\1/')"
 SHOULD_BUILD_JUPYTERLAB="$(grep -m 1 build_jupyter build.yml | sed -E 's/.*"([^"]*)".*/\1/')"
 
-SPARK_VERSION="$(grep -m 1 spark build.yml | sed -E 's/.*"([^"]*)".*/\1/')"
-JUPYTERLAB_VERSION="$(grep -m 1 jupyterlab build.yml | sed -E 's/.*"([^"]*)".*/\1/')"
-
+SPARK_VERSION="3.5.7"
 HADOOP_VERSION="3"
 SCALA_VERSION="2.12.20"
 SCALA_KERNEL_VERSION="0.14.2"
+JUPYTERLAB_VERSION="4.4.10"
 
 # ----------------------------------------------------------------------------------------------------------------------
 # -- Functions----------------------------------------------------------------------------------------------------------

build/build.yml

@@ -1,6 +1,3 @@
-applications:
-  spark: "3.5.7"
-  jupyterlab: "4.4.10"
 build:
   build_base: "true"
   build_spark: "true"

build/docker-compose.yml

@@ -8,22 +8,25 @@ volumes:
     driver: local
 services:
   jupyterlab:
+    pull_policy: never
     image: jupyterlab:4.4.10-spark-3.5.7
     container_name: jupyterlab
     ports:
       - "8888:8888"
+      - "4040:4040"
     volumes:
       - shared-workspace:/opt/workspace
   spark-master:
+    pull_policy: never
     image: spark-master:3.5.7
     container_name: spark-master
     ports:
       - "8080:8080"
       - "7077:7077"
-      - "4040:4040"
     volumes:
       - shared-workspace:/opt/workspace
   spark-worker-1:
+    pull_policy: never
     image: spark-worker:3.5.7
     container_name: spark-worker-1
     environment:
@@ -36,6 +39,7 @@ services:
     depends_on:
       - spark-master
   spark-worker-2:
+    pull_policy: never
     image: spark-worker:3.5.7
     container_name: spark-worker-2
     environment:

build/docker/base/Dockerfile

@@ -22,12 +22,14 @@ RUN mkdir -p ${shared_workspace}/data && \
 
 RUN apt-get -y update && \
     apt-get -y install curl python3 r-base && \
-    apt-get -y clean
+    apt-get -y clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 RUN curl -L https://github.com/scala/scala/releases/download/v${scala_version}/scala-${scala_version}.deb -o scala.deb && \
     apt-get -y install ./scala.deb && \
     rm -rf scala.deb && \
-    apt-get -y clean
+    apt-get -y clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 ENV SCALA_HOME="/usr/bin/scala"
 ENV PATH=${PATH}:${SCALA_HOME}/bin

build/docker/jupyterlab/Dockerfile

@@ -23,19 +23,20 @@ ARG jupyterlab_version
 RUN apt-get -y update && \
     apt-get -y install python3-pip python3-dev && \
     pip install wget==3.2 pyspark==${spark_version} jupyterlab==${jupyterlab_version} --break-system-packages && \
-    apt-get -y clean
+    apt-get -y clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 # -- Layer: Scala kernel for Spark
 
 ARG scala_version
 ARG scala_kernel_version
 
-RUN apt-get -y install ca-certificates-java --no-install-recommends && \
-    curl -Lo coursier https://git.io/coursier-cli && \
+RUN curl -Lo coursier https://git.io/coursier-cli && \
     chmod +x coursier && \
     ./coursier launch --fork almond:${scala_kernel_version} --scala ${scala_version} -- --display-name "Scala ${scala_version}" --install && \
     rm -f coursier && \
-    apt-get -y clean
+    apt-get -y clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 # -- Layer: R kernel for SparkR
 
@@ -45,11 +46,12 @@ RUN apt-get -y install r-base-dev && \
     curl -L https://archive.apache.org/dist/spark/spark-${spark_version}/SparkR_${spark_version}.tar.gz -o sparkr.tar.gz && \
     R CMD INSTALL sparkr.tar.gz && \
     rm -f sparkr.tar.gz && \
-    apt-get -y clean
+    apt-get -y clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 # -- Runtime
 
-EXPOSE 8888
+EXPOSE 8888 4040
 
 WORKDIR ${SHARED_WORKSPACE}
 CMD jupyter lab --ip=0.0.0.0 --port=8888 --no-browser --allow-root --NotebookApp.token=