From 8a28290fde9a8f625ef3e900ff91819bf7a5417c Mon Sep 17 00:00:00 2001 From: Jarvis Date: Fri, 23 Aug 2024 22:45:43 +0800 Subject: [PATCH] [Feature] add dockerfile (#7346) --- .github/workflows/publish-docker.yaml | 43 +++++---- docs/en/start-v2/docker/docker.md | 103 +++++++++++++++++++++- pom.xml | 93 +++++++++++++++++++ seatunnel-dist/pom.xml | 21 ++++- seatunnel-dist/src/main/docker/Dockerfile | 12 +++ 5 files changed, 250 insertions(+), 22 deletions(-) create mode 100644 seatunnel-dist/src/main/docker/Dockerfile diff --git a/.github/workflows/publish-docker.yaml b/.github/workflows/publish-docker.yaml index 96b0b019368..e1041bca2c9 100644 --- a/.github/workflows/publish-docker.yaml +++ b/.github/workflows/publish-docker.yaml @@ -18,16 +18,17 @@ name: publish-docker on: push: + tags: + - '*' branches: - dev - - docker paths-ignore: - 'docs/**' - '**/*.md' - - 'seatunnel-ui/**' env: - HUB: ghcr.io/${{ github.repository }} + DOCKER_USERNAME: ${{ secrets.DOCKERHUB_USER }} + DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} jobs: build: @@ -36,11 +37,16 @@ jobs: permissions: contents: read packages: write - timeout-minutes: 30 + timeout-minutes: 60 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 + with: + submodules: true + - name: free disk space + run: tools/github/free_disk_space.sh + - uses: actions/checkout@v4 - name: Cache local Maven repository - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: ~/.m2/repository key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} @@ -51,26 +57,27 @@ jobs: with: java-version: 8 distribution: 'adopt' + - name: Log in to the Container registry - uses: docker/login-action@v1.10.0 + uses: docker/login-action@v3 with: - registry: ${{ env.HUB }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + username: ${{ env.DOCKER_USERNAME }} + password: ${{ env.DOCKER_PASSWORD }} - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v3 - name: Build and push docker images env: - MAVEN_OPTS: -Xmx2G -Xms2G + MAVEN_OPTS: -Xmx4096m run: | - ./mvnw -B clean deploy \ + ./mvnw -B clean install \ -Dmaven.test.skip \ -Dmaven.javadoc.skip \ -Dlicense.skipAddThirdParty=true \ + -D"docker.build.skip"=false \ + -D"docker.verify.skip"=false \ + -D"docker.push.skip"=false \ -Dmaven.deploy.skip \ - -Ddocker.tag=${{ github.sha }} \ - -Ddocker.hub=${{ env.HUB }} \ - -Pdocker \ - --no-snapshot-updates + --no-snapshot-updates \ + -Pdocker,seatunnel \ No newline at end of file diff --git a/docs/en/start-v2/docker/docker.md b/docs/en/start-v2/docker/docker.md index ad2e3b078ad..5c2c8422224 100644 --- a/docs/en/start-v2/docker/docker.md +++ b/docs/en/start-v2/docker/docker.md @@ -2,7 +2,104 @@ sidebar_position: 3 --- -# Set Up With Docker +# Set Up with Docker in local mode + +## Zeta Engine + +### Download + +```shell +docker pull apache/seatunnel: +``` + +How to submit job in local mode + +```shell +docker run --rm -it apache/seatunnel bash ./bin/seatunnel.sh -e local -c + + +# eg: a fake source to console sink +docker run --rm -it apache/seatunnel bash ./bin/seatunnel.sh -e local -c config/v2.batch.config.template + +``` + +### Build Image By Yourself + +```Dockerfile +FROM openjdk:8 + +ARG VERSION +# Build from Source Code And Copy it into image +COPY ./target/apache-seatunnel-${VERSION}-bin.tar.gz /opt/ + +# Download From Internet +# Please Note this file only include fake/console connector, You'll need to download the other connectors manually +# wget -P /opt https://dlcdn.apache.org/seatunnel/2.3.6/apache-seatunnel-${VERSION}-bin.tar.gz + +RUN cd /opt && \ + tar -zxvf apache-seatunnel-${VERSION}-bin.tar.gz && \ + mv apache-seatunnel-${VERSION} seatunnel && \ + rm apache-seatunnel-${VERSION}-bin.tar.gz + +WORKDIR /opt/seatunnel +``` + +## Spark or Flink Engine + +### Download And Install the connectors you needed + +refer the step as Zeta Engine + +### Mount Spark/Flink library + +By default, Spark home is `/opt/spark`, Flink home is `/opt/flink`. +If you need run with spark/flink, you can mount the related library to `/opt/spark` or `/opt/flink`. + +```shell +docker run \ + -v :/opt/spark \ + -v :/opt/flink \ + ... +``` + +Or you can change the `SPARK_HOME`, `FLINK_HOME` environment variable in Dockerfile and re-build your and mount the spark/flink to related path. + +```Dockerfile +FROM apache/seatunnel + +ENV SPARK_HOME= + +... + +``` + +```shell +docker run \ + -v : \ + ... +``` + +### Submit job + +The command is different for different engines and different versions of the same engine, please choose the correct command. + +- Spark + +```shell +# spark2 +docker run --rm -it apache/seatunnel bash ./bin/start-seatunnel-spark-2-connector-v2.sh -c config/v2.batch.config.template + +# spark3 +docker run --rm -it apache/seatunnel bash ./bin/start-seatunnel-spark-3-connector-v2.sh -c config/v2.batch.config.template +``` + +- Flink + before you submit job, you need start flink cluster first. + +```shell +# flink version between `1.12.x` and `1.14.x` +docker run --rm -it apache/seatunnel bash -c '/bin/start-cluster.sh && ./bin/start-seatunnel-flink-13-connector-v2.sh -c config/v2.streaming.conf.template' +# flink version between `1.15.x` and `1.16.x` +docker run --rm -it apache/seatunnel bash -c '/bin/start-cluster.sh && ./bin/start-seatunnel-flink-15-connector-v2.sh -c config/v2.streaming.conf.template' +``` - ---> diff --git a/pom.xml b/pom.xml index 38860eb2792..3815a5c393d 100644 --- a/pom.xml +++ b/pom.xml @@ -113,7 +113,11 @@ 1.13 3.0.0 apache + seatunnel ${project.version} + true + true + true 1.81 4.13.2 5.9.0 @@ -760,6 +764,95 @@ maven-dependency-plugin ${maven-dependency-plugin.version} + + org.codehaus.mojo + exec-maven-plugin + ${exec-maven-plugin.version} + + + docker-build + + exec + + package + + ${docker.build.skip} + + 1 + + docker + ${project.basedir} + + buildx + build + --load + --no-cache + -t + ${docker.hub}/${docker.repo}:${docker.tag} + ${project.basedir} + --build-arg + VERSION=${project.version} + --file=src/main/docker/Dockerfile + + + + + docker-verify + + exec + + verify + + ${docker.verify.skip} + + 1 + + docker + ${project.basedir} + + run + --rm + ${docker.hub}/${docker.repo}:${docker.tag} + bash + ./bin/seatunnel.sh + -e + local + -c + config/v2.batch.config.template + + + + + docker-push + + exec + + install + + ${docker.push.skip} + + 1 + + docker + ${project.basedir} + + buildx + build + --platform + linux/amd64,linux/arm64 + --no-cache + --push + -t + ${docker.hub}/${docker.repo}:${docker.tag} + ${project.basedir} + --build-arg + VERSION=${revision} + --file=src/main/docker/Dockerfile + + + + + diff --git a/seatunnel-dist/pom.xml b/seatunnel-dist/pom.xml index e919293aab8..fb6935f0894 100644 --- a/seatunnel-dist/pom.xml +++ b/seatunnel-dist/pom.xml @@ -66,7 +66,6 @@ true - @@ -83,6 +82,9 @@ + false + false + false 8.0.27 42.4.3 2.5.1 @@ -965,5 +967,22 @@ + + docker + + + release + false + + + + + + org.codehaus.mojo + exec-maven-plugin + + + + diff --git a/seatunnel-dist/src/main/docker/Dockerfile b/seatunnel-dist/src/main/docker/Dockerfile new file mode 100644 index 00000000000..7d040ee41aa --- /dev/null +++ b/seatunnel-dist/src/main/docker/Dockerfile @@ -0,0 +1,12 @@ +FROM openjdk:8 + +ARG VERSION + +COPY ./target/apache-seatunnel-${VERSION}-bin.tar.gz /opt/ + +RUN cd /opt && \ + tar -zxvf apache-seatunnel-${VERSION}-bin.tar.gz && \ + mv apache-seatunnel-${VERSION} seatunnel && \ + rm apache-seatunnel-${VERSION}-bin.tar.gz + +WORKDIR /opt/seatunnel