Skip to content

Commit d908cb6

Browse files
andre-marcos-perezandre-marcos-perez
authored
Merge pull request #72 from cluster-apps-on-docker/staging
Staging
2 parents 0cda6be + 6a54dec commit d908cb6

5 files changed

Lines changed: 89 additions & 97 deletions

File tree

‎.github/workflows/master.yml‎

Lines changed: 37 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,8 @@ on:
66
branches:
77
- 'master'
88
jobs:
9-
spark-images:
10-
name: Spark Images
9+
spark-latest-images:
10+
name: Spark Images (latest)
1111
runs-on: ubuntu-latest
1212
env:
1313
DOCKERHUB_USR: ${{ secrets.DOCKERHUB_USR }}
@@ -24,32 +24,36 @@ jobs:
2424
SPARK_VERSION: "3.0.0"
2525
HADOOP_VERSION: "3.2"
2626
IS_LATEST: "true"
27-
- name: Build Spark v2.4.4
28-
uses: ./.github/actions/build-spark
29-
with:
30-
DOCKERHUB_USR: ${DOCKERHUB_USR}
31-
DOCKERHUB_PWD: ${DOCKERHUB_PWD}
32-
SCALA_VERSION: "2.11.12"
33-
SPARK_VERSION: "2.4.4"
34-
HADOOP_VERSION: "2.7"
35-
- name: Build Spark v2.4.0
27+
spark-images:
28+
name: Spark Images
29+
runs-on: ubuntu-latest
30+
env:
31+
DOCKERHUB_USR: ${{ secrets.DOCKERHUB_USR }}
32+
DOCKERHUB_PWD: ${{ secrets.DOCKERHUB_PWD }}
33+
strategy:
34+
matrix:
35+
spark_version: [ "2.4.4", "2.4.0" ]
36+
steps:
37+
- name: Checkout Code
38+
uses: actions/checkout@v2
39+
- name: Build Spark v2.4.4 + v2.4.0
3640
uses: ./.github/actions/build-spark
3741
with:
3842
DOCKERHUB_USR: ${DOCKERHUB_USR}
3943
DOCKERHUB_PWD: ${DOCKERHUB_PWD}
4044
SCALA_VERSION: "2.11.12"
41-
SPARK_VERSION: "2.4.0"
45+
SPARK_VERSION: ${{ matrix.spark_version }}
4246
HADOOP_VERSION: "2.7"
43-
jupyterlab-images:
44-
name: JupyterLab Images
47+
jupyterlab-latest-images:
48+
name: JupyterLab Images (latest)
4549
runs-on: ubuntu-latest
4650
env:
4751
DOCKERHUB_USR: ${{ secrets.DOCKERHUB_USR }}
4852
DOCKERHUB_PWD: ${{ secrets.DOCKERHUB_PWD }}
4953
steps:
5054
- name: Checkout Code
5155
uses: actions/checkout@v2
52-
- name: Build JupyterLab v3.0.0 + Spark v3.0.0 (latest)
56+
- name: Build JupyterLab v3.0.0 and Spark v3.0.0 (latest)
5357
uses: ./.github/actions/build-jupyterlab
5458
with:
5559
DOCKERHUB_USR: ${DOCKERHUB_USR}
@@ -59,7 +63,7 @@ jobs:
5963
SPARK_VERSION: "3.0.0"
6064
JUPYTERLAB_VERSION: "3.0.0"
6165
IS_LATEST: "true"
62-
- name: Build JupyterLab v2.1.4 + Spark v3.0.0
66+
- name: Build JupyterLab v2.1.4 and Spark v3.0.0
6367
uses: ./.github/actions/build-jupyterlab
6468
with:
6569
DOCKERHUB_USR: ${DOCKERHUB_USR}
@@ -68,39 +72,26 @@ jobs:
6872
SCALA_KERNEL_VERSION: "0.10.9"
6973
SPARK_VERSION: "3.0.0"
7074
JUPYTERLAB_VERSION: "2.1.4"
71-
- name: Build JupyterLab v3.0.0 + Spark v2.4.4
72-
uses: ./.github/actions/build-jupyterlab
73-
with:
74-
DOCKERHUB_USR: ${DOCKERHUB_USR}
75-
DOCKERHUB_PWD: ${DOCKERHUB_PWD}
76-
SCALA_VERSION: "2.11.12"
77-
SCALA_KERNEL_VERSION: "0.6.0"
78-
SPARK_VERSION: "2.4.4"
79-
JUPYTERLAB_VERSION: "3.0.0"
80-
- name: Build JupyterLab v2.1.4 + Spark v2.4.4
81-
uses: ./.github/actions/build-jupyterlab
82-
with:
83-
DOCKERHUB_USR: ${DOCKERHUB_USR}
84-
DOCKERHUB_PWD: ${DOCKERHUB_PWD}
85-
SCALA_VERSION: "2.11.12"
86-
SCALA_KERNEL_VERSION: "0.6.0"
87-
SPARK_VERSION: "2.4.4"
88-
JUPYTERLAB_VERSION: "2.1.4"
89-
- name: Build JupyterLab v3.0.0 + Spark v2.4.0
90-
uses: ./.github/actions/build-jupyterlab
91-
with:
92-
DOCKERHUB_USR: ${DOCKERHUB_USR}
93-
DOCKERHUB_PWD: ${DOCKERHUB_PWD}
94-
SCALA_VERSION: "2.11.12"
95-
SCALA_KERNEL_VERSION: "0.6.0"
96-
SPARK_VERSION: "2.4.0"
97-
JUPYTERLAB_VERSION: "3.0.0"
98-
- name: Build JupyterLab v2.1.4 + Spark v2.4.0
75+
IS_LATEST: "false"
76+
jupyterlab-images:
77+
name: JupyterLab Images
78+
runs-on: ubuntu-latest
79+
env:
80+
DOCKERHUB_USR: ${{ secrets.DOCKERHUB_USR }}
81+
DOCKERHUB_PWD: ${{ secrets.DOCKERHUB_PWD }}
82+
strategy:
83+
matrix:
84+
spark_version: [ "2.4.4", "2.4.0" ]
85+
jupyterlab_version: [ "3.0.0", "2.1.4" ]
86+
steps:
87+
- name: Checkout Code
88+
uses: actions/checkout@v2
89+
- name: Build JupyterLab v3.0.0 + v2.1.4 and Spark v2.4.4 + v2.4.0
9990
uses: ./.github/actions/build-jupyterlab
10091
with:
10192
DOCKERHUB_USR: ${DOCKERHUB_USR}
10293
DOCKERHUB_PWD: ${DOCKERHUB_PWD}
10394
SCALA_VERSION: "2.11.12"
10495
SCALA_KERNEL_VERSION: "0.6.0"
105-
SPARK_VERSION: "2.4.0"
106-
JUPYTERLAB_VERSION: "2.1.4"
96+
SPARK_VERSION: ${{ matrix.spark_version }}
97+
JUPYTERLAB_VERSION: ${{ matrix.jupyterlab_version }}

‎.github/workflows/staging.yml‎

Lines changed: 37 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,8 @@ on:
44
branches:
55
- 'staging'
66
jobs:
7-
spark-images:
8-
name: Spark Images
7+
spark-latest-images:
8+
name: Spark Images (latest)
99
runs-on: ubuntu-latest
1010
env:
1111
DOCKERHUB_USR: ${{ secrets.DOCKERHUB_USR }}
@@ -22,32 +22,36 @@ jobs:
2222
SPARK_VERSION: "3.0.0"
2323
HADOOP_VERSION: "3.2"
2424
IS_LATEST: "true"
25-
- name: Build Spark v2.4.4
26-
uses: ./.github/actions/build-spark
27-
with:
28-
DOCKERHUB_USR: ${DOCKERHUB_USR}
29-
DOCKERHUB_PWD: ${DOCKERHUB_PWD}
30-
SCALA_VERSION: "2.11.12"
31-
SPARK_VERSION: "2.4.4"
32-
HADOOP_VERSION: "2.7"
33-
- name: Build Spark v2.4.0
25+
spark-images:
26+
name: Spark Images
27+
runs-on: ubuntu-latest
28+
env:
29+
DOCKERHUB_USR: ${{ secrets.DOCKERHUB_USR }}
30+
DOCKERHUB_PWD: ${{ secrets.DOCKERHUB_PWD }}
31+
strategy:
32+
matrix:
33+
spark_version: [ "2.4.4", "2.4.0" ]
34+
steps:
35+
- name: Checkout Code
36+
uses: actions/checkout@v2
37+
- name: Build Spark v2.4.4 + v2.4.0
3438
uses: ./.github/actions/build-spark
3539
with:
3640
DOCKERHUB_USR: ${DOCKERHUB_USR}
3741
DOCKERHUB_PWD: ${DOCKERHUB_PWD}
3842
SCALA_VERSION: "2.11.12"
39-
SPARK_VERSION: "2.4.0"
43+
SPARK_VERSION: ${{ matrix.spark_version }}
4044
HADOOP_VERSION: "2.7"
41-
jupyterlab-images:
42-
name: JupyterLab Images
45+
jupyterlab-latest-images:
46+
name: JupyterLab Images (latest)
4347
runs-on: ubuntu-latest
4448
env:
4549
DOCKERHUB_USR: ${{ secrets.DOCKERHUB_USR }}
4650
DOCKERHUB_PWD: ${{ secrets.DOCKERHUB_PWD }}
4751
steps:
4852
- name: Checkout Code
4953
uses: actions/checkout@v2
50-
- name: Build JupyterLab v3.0.0 + Spark v3.0.0 (latest)
54+
- name: Build JupyterLab v3.0.0 and Spark v3.0.0 (latest)
5155
uses: ./.github/actions/build-jupyterlab
5256
with:
5357
DOCKERHUB_USR: ${DOCKERHUB_USR}
@@ -57,7 +61,7 @@ jobs:
5761
SPARK_VERSION: "3.0.0"
5862
JUPYTERLAB_VERSION: "3.0.0"
5963
IS_LATEST: "true"
60-
- name: Build JupyterLab v2.1.4 + Spark v3.0.0
64+
- name: Build JupyterLab v2.1.4 and Spark v3.0.0
6165
uses: ./.github/actions/build-jupyterlab
6266
with:
6367
DOCKERHUB_USR: ${DOCKERHUB_USR}
@@ -66,39 +70,26 @@ jobs:
6670
SCALA_KERNEL_VERSION: "0.10.9"
6771
SPARK_VERSION: "3.0.0"
6872
JUPYTERLAB_VERSION: "2.1.4"
69-
- name: Build JupyterLab v3.0.0 + Spark v2.4.4
70-
uses: ./.github/actions/build-jupyterlab
71-
with:
72-
DOCKERHUB_USR: ${DOCKERHUB_USR}
73-
DOCKERHUB_PWD: ${DOCKERHUB_PWD}
74-
SCALA_VERSION: "2.11.12"
75-
SCALA_KERNEL_VERSION: "0.6.0"
76-
SPARK_VERSION: "2.4.4"
77-
JUPYTERLAB_VERSION: "3.0.0"
78-
- name: Build JupyterLab v2.1.4 + Spark v2.4.4
79-
uses: ./.github/actions/build-jupyterlab
80-
with:
81-
DOCKERHUB_USR: ${DOCKERHUB_USR}
82-
DOCKERHUB_PWD: ${DOCKERHUB_PWD}
83-
SCALA_VERSION: "2.11.12"
84-
SCALA_KERNEL_VERSION: "0.6.0"
85-
SPARK_VERSION: "2.4.4"
86-
JUPYTERLAB_VERSION: "2.1.4"
87-
- name: Build JupyterLab v3.0.0 + Spark v2.4.0
88-
uses: ./.github/actions/build-jupyterlab
89-
with:
90-
DOCKERHUB_USR: ${DOCKERHUB_USR}
91-
DOCKERHUB_PWD: ${DOCKERHUB_PWD}
92-
SCALA_VERSION: "2.11.12"
93-
SCALA_KERNEL_VERSION: "0.6.0"
94-
SPARK_VERSION: "2.4.0"
95-
JUPYTERLAB_VERSION: "3.0.0"
96-
- name: Build JupyterLab v2.1.4 + Spark v2.4.0
73+
IS_LATEST: "false"
74+
jupyterlab-images:
75+
name: JupyterLab Images
76+
runs-on: ubuntu-latest
77+
env:
78+
DOCKERHUB_USR: ${{ secrets.DOCKERHUB_USR }}
79+
DOCKERHUB_PWD: ${{ secrets.DOCKERHUB_PWD }}
80+
strategy:
81+
matrix:
82+
spark_version: [ "2.4.4", "2.4.0" ]
83+
jupyterlab_version: [ "3.0.0", "2.1.4" ]
84+
steps:
85+
- name: Checkout Code
86+
uses: actions/checkout@v2
87+
- name: Build JupyterLab v3.0.0 + v2.1.4 and Spark v2.4.4 + v2.4.0
9788
uses: ./.github/actions/build-jupyterlab
9889
with:
9990
DOCKERHUB_USR: ${DOCKERHUB_USR}
10091
DOCKERHUB_PWD: ${DOCKERHUB_PWD}
10192
SCALA_VERSION: "2.11.12"
10293
SCALA_KERNEL_VERSION: "0.6.0"
103-
SPARK_VERSION: "2.4.0"
104-
JUPYTERLAB_VERSION: "2.1.4"
94+
SPARK_VERSION: ${{ matrix.spark_version }}
95+
JUPYTERLAB_VERSION: ${{ matrix.jupyterlab_version }}

‎CHANGELOG.md‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,16 @@
22

33
All notable changes to this project will be documented in this file.
44

5+
## [1.2.3](https://github.com/andre-marcos-perez/spark-standalone-cluster-on-docker/releases/tag/v1.2.3) (2020-12-31)
6+
7+
### Core
8+
9+
- Added wget Python package on JupyterLab image since it is been used on Medium article ([#65](https://github.com/cluster-apps-on-docker/spark-standalone-cluster-on-docker/issues/65)).
10+
11+
### Repository
12+
13+
- Enhanced Github actions with build matrix.
14+
515
## [1.2.2](https://github.com/andre-marcos-perez/spark-standalone-cluster-on-docker/releases/tag/v1.2.2) (2020-12-30)
616

717
### Repository

‎README.md‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,18 +11,18 @@ Learn Apache Spark through its **Scala**, **Python** (PySpark) and **R** (SparkR
1111

1212
<p align="center"><img src="docs/image/cluster-architecture.png"></p>
1313

14-
![build](https://github.com/andre-marcos-perez/spark-standalone-cluster-on-docker/workflows/build/badge.svg?branch=master)
14+
![build-master](https://github.com/cluster-apps-on-docker/spark-standalone-cluster-on-docker/workflows/build-master/badge.svg)
15+
![sponsor](https://img.shields.io/badge/patreon-sponsor-ff69b4)
1516
![jupyterlab-latest-version](https://img.shields.io/docker/v/andreper/jupyterlab/3.0.0-spark-3.0.0?color=yellow&label=jupyterlab-latest)
1617
![spark-latest-version](https://img.shields.io/docker/v/andreper/spark-master/3.0.0?color=yellow&label=spark-latest)
1718
![spark-scala-api](https://img.shields.io/badge/spark%20api-scala-red)
1819
![spark-pyspark-api](https://img.shields.io/badge/spark%20api-pyspark-red)
1920
![spark-sparkr-api](https://img.shields.io/badge/spark%20api-sparkr-red)
20-
![sponsor](https://img.shields.io/badge/patreon-sponsor-ff69b4)
2121

2222
## TL;DR
2323

2424
```bash
25-
curl -LO https://raw.githubusercontent.com/andre-marcos-perez/spark-standalone-cluster-on-docker/master/docker-compose.yml
25+
curl -LO https://raw.githubusercontent.com/cluster-apps-on-docker/spark-standalone-cluster-on-docker/master/docker-compose.yml
2626
docker-compose up
2727
```
2828

@@ -56,7 +56,7 @@ docker-compose up
5656
1. Download the [docker compose](docker-compose.yml) file;
5757

5858
```bash
59-
curl -LO https://raw.githubusercontent.com/andre-marcos-perez/spark-standalone-cluster-on-docker/master/docker-compose.yml
59+
curl -LO https://raw.githubusercontent.com/cluster-apps-on-docker/spark-standalone-cluster-on-docker/master/docker-compose.yml
6060
```
6161

6262
2. Edit the [docker compose](docker-compose.yml) file with your favorite tech stack version, check **apps** [supported versions](#tech-stack);

‎build/docker/jupyterlab/Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ ARG scala_kernel_version
2424
RUN apt-get update -y && \
2525
apt-get install -y python3-pip python3-dev && \
2626
pip3 install --upgrade pip && \
27-
pip3 install pyspark==${spark_version} jupyterlab==${jupyterlab_version}
27+
pip3 install wget==3.2 pyspark==${spark_version} jupyterlab==${jupyterlab_version}
2828

2929
# -- Layer: Scala kernel for Spark
3030

0 commit comments

Comments
 (0)