From eed6f1a1daf08dc71e37b9f86d4c2d98f3504b61 Mon Sep 17 00:00:00 2001 From: gdliu3 Date: Sun, 30 Jun 2024 16:58:39 +0800 Subject: [PATCH 01/59] 1 --- .github/workflows/backend.yml | 66 +++++++++++++------------- .github/workflows/schedule_backend.yml | 4 +- 2 files changed, 35 insertions(+), 35 deletions(-) diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index 9975d477dae..f573ffc34cc 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -274,7 +274,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '11' ,'17'] os: [ 'ubuntu-latest', 'windows-latest' ] timeout-minutes: 60 steps: @@ -297,7 +297,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '11' ,'17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -328,7 +328,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '11' ,'17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -359,7 +359,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '11' ,'17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -390,7 +390,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '11' ,'17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -420,7 +420,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '11' ,'17'] os: [ 'ubuntu-latest' ] timeout-minutes: 120 steps: @@ -450,7 +450,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '11' ,'17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -480,7 +480,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '11' ,'17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -511,7 +511,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '11' ,'17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -542,7 +542,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '11' ,'17'] os: [ 'ubuntu-latest' ] timeout-minutes: 45 steps: @@ -565,7 +565,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '11' ,'17'] os: [ 'ubuntu-latest' ] timeout-minutes: 45 steps: @@ -599,7 +599,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '11' ,'17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -623,7 +623,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '11' ,'17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -647,7 +647,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '11' ,'17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -674,7 +674,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '11' ,'17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -701,7 +701,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '11' ,'17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -728,7 +728,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '11' ,'17'] os: [ 'ubuntu-latest' ] timeout-minutes: 150 steps: @@ -755,7 +755,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '11' ,'17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -782,7 +782,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '11' ,'17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -809,7 +809,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '11' ,'17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -836,7 +836,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '11' ,'17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -862,7 +862,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '11' ,'17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -888,7 +888,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '11' ,'17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -912,7 +912,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '11' ,'17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -936,7 +936,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '11' ,'17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -960,7 +960,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '11' ,'17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -986,7 +986,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '11' ,'17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -1010,7 +1010,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '11' ,'17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -1033,7 +1033,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '11' ,'17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -1056,7 +1056,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '11' ,'17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -1080,7 +1080,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '11' ,'17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -1105,7 +1105,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '11' ,'17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -1131,7 +1131,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '11' ,'17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: diff --git a/.github/workflows/schedule_backend.yml b/.github/workflows/schedule_backend.yml index 503a3866df9..b41059b4e36 100644 --- a/.github/workflows/schedule_backend.yml +++ b/.github/workflows/schedule_backend.yml @@ -104,7 +104,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '11' ,'17'] os: [ 'ubuntu-latest', 'windows-latest' ] timeout-minutes: 90 steps: @@ -126,7 +126,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '11' ,'17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: From c8cbcba0520192062a42e5f23ae865b8f2a16108 Mon Sep 17 00:00:00 2001 From: gdliu3 Date: Thu, 11 Jul 2024 22:52:49 +0800 Subject: [PATCH 02/59] 1 --- docs/en/connector-v2/sink/Hive.md | 2 +- docs/en/connector-v2/sink/OssFile.md | 2 +- docs/en/connector-v2/source/Hive.md | 2 +- docs/en/connector-v2/source/OssFile.md | 2 +- pom.xml | 4 ++-- .../connector-file/connector-file-base/pom.xml | 4 ++-- .../connector-file/connector-file-ftp/pom.xml | 2 +- .../connector-file/connector-file-hadoop/pom.xml | 2 +- .../connector-file/connector-file-local/pom.xml | 2 +- .../connector-file/connector-file-oss/pom.xml | 2 +- .../connector-file/connector-file-s3/pom.xml | 2 +- .../connector-file/connector-file-sftp/pom.xml | 2 +- seatunnel-connectors-v2/connector-hive/pom.xml | 4 ++-- seatunnel-connectors-v2/connector-iceberg/pom.xml | 2 +- seatunnel-connectors-v2/connector-kudu/pom.xml | 2 +- seatunnel-connectors-v2/connector-paimon/pom.xml | 2 +- seatunnel-core/seatunnel-flink-starter/pom.xml | 2 +- seatunnel-core/seatunnel-spark-starter/pom.xml | 2 +- seatunnel-core/seatunnel-starter/pom.xml | 2 +- seatunnel-dist/pom.xml | 4 ++-- seatunnel-dist/src/main/assembly/assembly-bin-ci.xml | 2 +- seatunnel-dist/src/main/assembly/assembly-bin.xml | 2 +- .../connector-file-local-e2e/pom.xml | 2 +- .../connector-file-oss-e2e/pom.xml | 2 +- .../connector-file-s3-e2e/pom.xml | 2 +- .../connector-hive-e2e/pom.xml | 2 +- .../connector-iceberg-e2e/pom.xml | 2 +- .../e2e/connector/iceberg/s3/IcebergSourceIT.java | 2 +- .../connector-paimon-e2e/pom.xml | 2 +- .../seatunnel/ConnectorPackageServiceContainer.java | 12 ++++++------ .../container/seatunnel/SeaTunnelContainer.java | 8 ++++---- .../connector-seatunnel-e2e-base/pom.xml | 2 +- .../engine/e2e/ClusterSeaTunnelContainer.java | 4 ++-- .../seatunnel/engine/e2e/k8s/KubernetesIT.java | 4 ++-- .../src/test/resources/seatunnel_dockerfile | 2 +- seatunnel-engine/seatunnel-engine-client/pom.xml | 2 +- seatunnel-engine/seatunnel-engine-server/pom.xml | 2 +- .../checkpoint-storage-hdfs/pom.xml | 2 +- .../imap-storage-plugins/imap-storage-file/pom.xml | 2 +- seatunnel-examples/seatunnel-engine-examples/pom.xml | 2 +- seatunnel-shade/pom.xml | 2 +- .../pom.xml | 8 ++++---- 42 files changed, 59 insertions(+), 59 deletions(-) rename seatunnel-shade/{seatunnel-hadoop3-3.1.4-uber => seatunnel-hadoop3-3.3.4-uber}/pom.xml (96%) diff --git a/docs/en/connector-v2/sink/Hive.md b/docs/en/connector-v2/sink/Hive.md index 023bb38ddb1..cdbfbec5cdf 100644 --- a/docs/en/connector-v2/sink/Hive.md +++ b/docs/en/connector-v2/sink/Hive.md @@ -10,7 +10,7 @@ Write data to Hive. In order to use this connector, You must ensure your spark/flink cluster already integrated hive. The tested hive version is 2.3.9. -If you use SeaTunnel Engine, You need put seatunnel-hadoop3-3.1.4-uber.jar and hive-exec-3.1.3.jar and libfb303-0.9.3.jar in $SEATUNNEL_HOME/lib/ dir. +If you use SeaTunnel Engine, You need put seatunnel-hadoop3-3.3.4-uber.jar and hive-exec-3.1.3.jar and libfb303-0.9.3.jar in $SEATUNNEL_HOME/lib/ dir. ::: ## Key features diff --git a/docs/en/connector-v2/sink/OssFile.md b/docs/en/connector-v2/sink/OssFile.md index aef2bb11c09..8c0587aee12 100644 --- a/docs/en/connector-v2/sink/OssFile.md +++ b/docs/en/connector-v2/sink/OssFile.md @@ -17,7 +17,7 @@ ### For SeaTunnel Zeta Engine -1. You must ensure `seatunnel-hadoop3-3.1.4-uber.jar`, `aliyun-sdk-oss-3.4.1.jar`, `hadoop-aliyun-3.1.4.jar` and `jdom-1.1.jar` in `${SEATUNNEL_HOME}/lib/` dir. +1. You must ensure `seatunnel-hadoop3-3.3.4-uber.jar`, `aliyun-sdk-oss-3.4.1.jar`, `hadoop-aliyun-3.1.4.jar` and `jdom-1.1.jar` in `${SEATUNNEL_HOME}/lib/` dir. ## Key features diff --git a/docs/en/connector-v2/source/Hive.md b/docs/en/connector-v2/source/Hive.md index da70cf7aa34..0fb6dbc5211 100644 --- a/docs/en/connector-v2/source/Hive.md +++ b/docs/en/connector-v2/source/Hive.md @@ -10,7 +10,7 @@ Read data from Hive. In order to use this connector, You must ensure your spark/flink cluster already integrated hive. The tested hive version is 2.3.9. -If you use SeaTunnel Engine, You need put seatunnel-hadoop3-3.1.4-uber.jar and hive-exec-3.1.3.jar and libfb303-0.9.3.jar in $SEATUNNEL_HOME/lib/ dir. +If you use SeaTunnel Engine, You need put seatunnel-hadoop3-3.3.4-uber.jar and hive-exec-3.1.3.jar and libfb303-0.9.3.jar in $SEATUNNEL_HOME/lib/ dir. ::: ## Key features diff --git a/docs/en/connector-v2/source/OssFile.md b/docs/en/connector-v2/source/OssFile.md index 3f781eb11a9..3a9b7661d80 100644 --- a/docs/en/connector-v2/source/OssFile.md +++ b/docs/en/connector-v2/source/OssFile.md @@ -17,7 +17,7 @@ ### For SeaTunnel Zeta Engine -1. You must ensure `seatunnel-hadoop3-3.1.4-uber.jar`, `aliyun-sdk-oss-3.4.1.jar`, `hadoop-aliyun-3.1.4.jar` and `jdom-1.1.jar` in `${SEATUNNEL_HOME}/lib/` dir. +1. You must ensure `seatunnel-hadoop3-3.3.4-uber.jar`, `aliyun-sdk-oss-3.4.1.jar`, `hadoop-aliyun-3.1.4.jar` and `jdom-1.1.jar` in `${SEATUNNEL_HOME}/lib/` dir. ## Key features diff --git a/pom.xml b/pom.xml index 50c0d412008..d7d16782763 100644 --- a/pom.xml +++ b/pom.xml @@ -65,7 +65,7 @@ ${java.version} ${java.version} - 1.7.25 + 1.7.36 2.17.1 3.4.4 1.2.17 @@ -470,7 +470,7 @@ org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber ${project.version} optional provided diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/pom.xml b/seatunnel-connectors-v2/connector-file/connector-file-base/pom.xml index f091e7023d9..9d7f8897fd1 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-base/pom.xml +++ b/seatunnel-connectors-v2/connector-file/connector-file-base/pom.xml @@ -45,7 +45,7 @@ org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber ${project.version} optional provided @@ -116,7 +116,7 @@ org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber ${project.version} optional provided diff --git a/seatunnel-connectors-v2/connector-file/connector-file-ftp/pom.xml b/seatunnel-connectors-v2/connector-file/connector-file-ftp/pom.xml index 0ab14d7e04b..7a83d4b3777 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-ftp/pom.xml +++ b/seatunnel-connectors-v2/connector-file/connector-file-ftp/pom.xml @@ -37,7 +37,7 @@ org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber ${project.version} optional provided diff --git a/seatunnel-connectors-v2/connector-file/connector-file-hadoop/pom.xml b/seatunnel-connectors-v2/connector-file/connector-file-hadoop/pom.xml index 59da8bf8446..73748aa4cc1 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-hadoop/pom.xml +++ b/seatunnel-connectors-v2/connector-file/connector-file-hadoop/pom.xml @@ -37,7 +37,7 @@ org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber ${project.version} optional provided diff --git a/seatunnel-connectors-v2/connector-file/connector-file-local/pom.xml b/seatunnel-connectors-v2/connector-file/connector-file-local/pom.xml index 9c242cd3a2a..effb823611c 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-local/pom.xml +++ b/seatunnel-connectors-v2/connector-file/connector-file-local/pom.xml @@ -37,7 +37,7 @@ org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber ${project.version} optional provided diff --git a/seatunnel-connectors-v2/connector-file/connector-file-oss/pom.xml b/seatunnel-connectors-v2/connector-file/connector-file-oss/pom.xml index c904257b90d..330a9f45db7 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-oss/pom.xml +++ b/seatunnel-connectors-v2/connector-file/connector-file-oss/pom.xml @@ -44,7 +44,7 @@ org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber ${project.version} optional provided diff --git a/seatunnel-connectors-v2/connector-file/connector-file-s3/pom.xml b/seatunnel-connectors-v2/connector-file/connector-file-s3/pom.xml index fbf0016fced..9ad842fa1d5 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-s3/pom.xml +++ b/seatunnel-connectors-v2/connector-file/connector-file-s3/pom.xml @@ -54,7 +54,7 @@ org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber ${project.version} optional provided diff --git a/seatunnel-connectors-v2/connector-file/connector-file-sftp/pom.xml b/seatunnel-connectors-v2/connector-file/connector-file-sftp/pom.xml index 816184caf00..c1d744de7b7 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-sftp/pom.xml +++ b/seatunnel-connectors-v2/connector-file/connector-file-sftp/pom.xml @@ -37,7 +37,7 @@ org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber ${project.version} optional provided diff --git a/seatunnel-connectors-v2/connector-hive/pom.xml b/seatunnel-connectors-v2/connector-hive/pom.xml index a1e12d019de..16c7cd09fb0 100644 --- a/seatunnel-connectors-v2/connector-hive/pom.xml +++ b/seatunnel-connectors-v2/connector-hive/pom.xml @@ -42,7 +42,7 @@ org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber @@ -63,7 +63,7 @@ org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber ${project.version} optional provided diff --git a/seatunnel-connectors-v2/connector-iceberg/pom.xml b/seatunnel-connectors-v2/connector-iceberg/pom.xml index 309900b09c7..74c1356428b 100644 --- a/seatunnel-connectors-v2/connector-iceberg/pom.xml +++ b/seatunnel-connectors-v2/connector-iceberg/pom.xml @@ -138,7 +138,7 @@ org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber ${project.version} optional provided diff --git a/seatunnel-connectors-v2/connector-kudu/pom.xml b/seatunnel-connectors-v2/connector-kudu/pom.xml index 9dcdc87853a..2207614b95e 100644 --- a/seatunnel-connectors-v2/connector-kudu/pom.xml +++ b/seatunnel-connectors-v2/connector-kudu/pom.xml @@ -59,7 +59,7 @@ org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber ${project.version} optional provided diff --git a/seatunnel-connectors-v2/connector-paimon/pom.xml b/seatunnel-connectors-v2/connector-paimon/pom.xml index 80934e68a2b..34698108a02 100644 --- a/seatunnel-connectors-v2/connector-paimon/pom.xml +++ b/seatunnel-connectors-v2/connector-paimon/pom.xml @@ -56,7 +56,7 @@ org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber optional provided diff --git a/seatunnel-core/seatunnel-flink-starter/pom.xml b/seatunnel-core/seatunnel-flink-starter/pom.xml index 3b4f5442c32..b0f75fc2a82 100644 --- a/seatunnel-core/seatunnel-flink-starter/pom.xml +++ b/seatunnel-core/seatunnel-flink-starter/pom.xml @@ -92,7 +92,7 @@ org.apache.logging.log4j:log4j-slf4j-impl org.apache.logging.log4j:log4j-1.2-api org.apache.logging.log4j:log4j-to-slf4j - org.apache.seatunnel:seatunnel-hadoop3-3.1.4-uber + org.apache.seatunnel:seatunnel-hadoop3-3.3.4-uber diff --git a/seatunnel-core/seatunnel-spark-starter/pom.xml b/seatunnel-core/seatunnel-spark-starter/pom.xml index 4502d72c976..e4b77a59bf9 100644 --- a/seatunnel-core/seatunnel-spark-starter/pom.xml +++ b/seatunnel-core/seatunnel-spark-starter/pom.xml @@ -99,7 +99,7 @@ org.apache.logging.log4j:log4j-slf4j-impl org.apache.logging.log4j:log4j-to-slf4j - org.apache.seatunnel:seatunnel-hadoop3-3.1.4-uber + org.apache.seatunnel:seatunnel-hadoop3-3.3.4-uber diff --git a/seatunnel-core/seatunnel-starter/pom.xml b/seatunnel-core/seatunnel-starter/pom.xml index 85f9d7d4a13..88120743042 100644 --- a/seatunnel-core/seatunnel-starter/pom.xml +++ b/seatunnel-core/seatunnel-starter/pom.xml @@ -116,7 +116,7 @@ commons-logging:* ch.qos.logback:* org.apache.logging.log4j:log4j-to-slf4j - org.apache.seatunnel:seatunnel-hadoop3-3.1.4-uber + org.apache.seatunnel:seatunnel-hadoop3-3.3.4-uber diff --git a/seatunnel-dist/pom.xml b/seatunnel-dist/pom.xml index 37f1cbebf42..278829f2b01 100644 --- a/seatunnel-dist/pom.xml +++ b/seatunnel-dist/pom.xml @@ -718,7 +718,7 @@ org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber ${project.version} optional provided @@ -906,7 +906,7 @@ org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber ${project.version} optional provided diff --git a/seatunnel-dist/src/main/assembly/assembly-bin-ci.xml b/seatunnel-dist/src/main/assembly/assembly-bin-ci.xml index cc48ac86a2c..cb9b2b8b779 100644 --- a/seatunnel-dist/src/main/assembly/assembly-bin-ci.xml +++ b/seatunnel-dist/src/main/assembly/assembly-bin-ci.xml @@ -169,7 +169,7 @@ org.apache.seatunnel:seatunnel-transforms-v2:jar org.apache.hadoop:hadoop-aws:jar com.amazonaws:aws-java-sdk-bundle:jar - org.apache.seatunnel:seatunnel-hadoop3-3.1.4-uber:jar:*:optional + org.apache.seatunnel:seatunnel-hadoop3-3.3.4-uber:jar:*:optional org.apache.hadoop:hadoop-aliyun:jar com.aliyun.oss:aliyun-sdk-oss:jar diff --git a/seatunnel-dist/src/main/assembly/assembly-bin.xml b/seatunnel-dist/src/main/assembly/assembly-bin.xml index 30fc5a6336a..a67db42784f 100644 --- a/seatunnel-dist/src/main/assembly/assembly-bin.xml +++ b/seatunnel-dist/src/main/assembly/assembly-bin.xml @@ -168,7 +168,7 @@ false org.apache.seatunnel:seatunnel-transforms-v2:jar - org.apache.seatunnel:seatunnel-hadoop3-3.1.4-uber:jar:*:optional + org.apache.seatunnel:seatunnel-hadoop3-3.3.4-uber:jar:*:optional ${artifact.file.name} /lib diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/pom.xml index ea120abdd31..12e9469e9dd 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/pom.xml @@ -40,7 +40,7 @@ org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber ${project.version} optional test diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-oss-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-oss-e2e/pom.xml index cea8e9a4a9a..e4f7eff3f0a 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-oss-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-oss-e2e/pom.xml @@ -49,7 +49,7 @@ org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber ${project.version} optional test diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-s3-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-s3-e2e/pom.xml index e33b6273a89..71d0366fe6b 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-s3-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-s3-e2e/pom.xml @@ -55,7 +55,7 @@ org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber ${project.version} optional test diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hive-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hive-e2e/pom.xml index bd9a112b6ac..0ecddaf03df 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hive-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hive-e2e/pom.xml @@ -40,7 +40,7 @@ org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber ${project.version} optional test diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-e2e/pom.xml index 11c147432d0..26618e1332e 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-e2e/pom.xml @@ -63,7 +63,7 @@ org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber ${project.version} optional test diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-s3-e2e/src/test/java/org/apache/seatunnel/e2e/connector/iceberg/s3/IcebergSourceIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-s3-e2e/src/test/java/org/apache/seatunnel/e2e/connector/iceberg/s3/IcebergSourceIT.java index a6155bd68b0..807d3d776b3 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-s3-e2e/src/test/java/org/apache/seatunnel/e2e/connector/iceberg/s3/IcebergSourceIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-s3-e2e/src/test/java/org/apache/seatunnel/e2e/connector/iceberg/s3/IcebergSourceIT.java @@ -83,7 +83,7 @@ value = {TestContainerId.SPARK_2_4}, type = {EngineType.FLINK, EngineType.SEATUNNEL}, disabledReason = - "Needs hadoop-aws,aws-java-sdk jar for flink, spark2.4. For the seatunnel engine, it crashes on seatunnel-hadoop3-3.1.4-uber.jar.") + "Needs hadoop-aws,aws-java-sdk jar for flink, spark2.4. For the seatunnel engine, it crashes on seatunnel-hadoop3-3.3.4-uber.jar.") @Slf4j public class IcebergSourceIT extends TestSuiteBase implements TestResource { diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-paimon-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-paimon-e2e/pom.xml index 69ea9a9f74f..a026851058a 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-paimon-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-paimon-e2e/pom.xml @@ -40,7 +40,7 @@ org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber optional test diff --git a/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/container/seatunnel/ConnectorPackageServiceContainer.java b/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/container/seatunnel/ConnectorPackageServiceContainer.java index 4f5ea990295..8e20359e13c 100644 --- a/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/container/seatunnel/ConnectorPackageServiceContainer.java +++ b/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/container/seatunnel/ConnectorPackageServiceContainer.java @@ -84,8 +84,8 @@ public void startUp() throws Exception { server1.withCopyFileToContainer( MountableFile.forHostPath( PROJECT_ROOT_PATH - + "/seatunnel-shade/seatunnel-hadoop3-3.1.4-uber/target/seatunnel-hadoop3-3.1.4-uber.jar"), - Paths.get(SEATUNNEL_HOME, "lib/seatunnel-hadoop3-3.1.4-uber.jar").toString()); + + "/seatunnel-shade/seatunnel-hadoop3-3.3.4-uber/target/seatunnel-hadoop3-3.3.4-uber.jar"), + Paths.get(SEATUNNEL_HOME, "lib/seatunnel-hadoop3-3.3.4-uber.jar").toString()); server2 = new GenericContainer<>(getDockerImage()) @@ -110,8 +110,8 @@ public void startUp() throws Exception { server2.withCopyFileToContainer( MountableFile.forHostPath( PROJECT_ROOT_PATH - + "/seatunnel-shade/seatunnel-hadoop3-3.1.4-uber/target/seatunnel-hadoop3-3.1.4-uber.jar"), - Paths.get(SEATUNNEL_HOME, "lib/seatunnel-hadoop3-3.1.4-uber.jar").toString()); + + "/seatunnel-shade/seatunnel-hadoop3-3.3.4-uber/target/seatunnel-hadoop3-3.3.4-uber.jar"), + Paths.get(SEATUNNEL_HOME, "lib/seatunnel-hadoop3-3.3.4-uber.jar").toString()); server3 = new GenericContainer<>(getDockerImage()) @@ -136,8 +136,8 @@ public void startUp() throws Exception { server3.withCopyFileToContainer( MountableFile.forHostPath( PROJECT_ROOT_PATH - + "/seatunnel-shade/seatunnel-hadoop3-3.1.4-uber/target/seatunnel-hadoop3-3.1.4-uber.jar"), - Paths.get(SEATUNNEL_HOME, "lib/seatunnel-hadoop3-3.1.4-uber.jar").toString()); + + "/seatunnel-shade/seatunnel-hadoop3-3.3.4-uber/target/seatunnel-hadoop3-3.3.4-uber.jar"), + Paths.get(SEATUNNEL_HOME, "lib/seatunnel-hadoop3-3.3.4-uber.jar").toString()); Startables.deepStart(Stream.of(server1)).join(); Startables.deepStart(Stream.of(server2)).join(); diff --git a/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/container/seatunnel/SeaTunnelContainer.java b/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/container/seatunnel/SeaTunnelContainer.java index 2d9e76ea3b5..a255a36f6e2 100644 --- a/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/container/seatunnel/SeaTunnelContainer.java +++ b/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/container/seatunnel/SeaTunnelContainer.java @@ -107,8 +107,8 @@ private GenericContainer createSeaTunnelServer() throws IOException, Interrup server.withCopyFileToContainer( MountableFile.forHostPath( PROJECT_ROOT_PATH - + "/seatunnel-shade/seatunnel-hadoop3-3.1.4-uber/target/seatunnel-hadoop3-3.1.4-uber.jar"), - Paths.get(SEATUNNEL_HOME, "lib/seatunnel-hadoop3-3.1.4-uber.jar").toString()); + + "/seatunnel-shade/seatunnel-hadoop3-3.3.4-uber/target/seatunnel-hadoop3-3.3.4-uber.jar"), + Paths.get(SEATUNNEL_HOME, "lib/seatunnel-hadoop3-3.3.4-uber.jar").toString()); // execute extra commands executeExtraCommands(server); @@ -150,8 +150,8 @@ protected GenericContainer createSeaTunnelContainerWithFakeSourceAndInMemoryS server.withCopyFileToContainer( MountableFile.forHostPath( PROJECT_ROOT_PATH - + "/seatunnel-shade/seatunnel-hadoop3-3.1.4-uber/target/seatunnel-hadoop3-3.1.4-uber.jar"), - Paths.get(SEATUNNEL_HOME, "lib/seatunnel-hadoop3-3.1.4-uber.jar").toString()); + + "/seatunnel-shade/seatunnel-hadoop3-3.3.4-uber/target/seatunnel-hadoop3-3.3.4-uber.jar"), + Paths.get(SEATUNNEL_HOME, "lib/seatunnel-hadoop3-3.3.4-uber.jar").toString()); server.start(); // execute extra commands diff --git a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/pom.xml b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/pom.xml index 973fe6434c7..df971e18822 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/pom.xml +++ b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/pom.xml @@ -66,7 +66,7 @@ org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber ${project.version} optional test diff --git a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/ClusterSeaTunnelContainer.java b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/ClusterSeaTunnelContainer.java index bf16cab75d2..4c1acbbdc4a 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/ClusterSeaTunnelContainer.java +++ b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/ClusterSeaTunnelContainer.java @@ -64,7 +64,7 @@ public class ClusterSeaTunnelContainer extends SeaTunnelContainer { private static final Path binPath = Paths.get(SEATUNNEL_HOME, "bin", SERVER_SHELL); private static final Path config = Paths.get(SEATUNNEL_HOME, "config"); private static final Path hadoopJar = - Paths.get(SEATUNNEL_HOME, "lib/seatunnel-hadoop3-3.1.4-uber.jar"); + Paths.get(SEATUNNEL_HOME, "lib/seatunnel-hadoop3-3.3.4-uber.jar"); private static final long CUSTOM_JOB_ID = 862969647010611201L; @@ -364,7 +364,7 @@ private GenericContainer createServer(String networkAlias) server.withCopyFileToContainer( MountableFile.forHostPath( PROJECT_ROOT_PATH - + "/seatunnel-shade/seatunnel-hadoop3-3.1.4-uber/target/seatunnel-hadoop3-3.1.4-uber.jar"), + + "/seatunnel-shade/seatunnel-hadoop3-3.3.4-uber/target/seatunnel-hadoop3-3.3.4-uber.jar"), hadoopJar.toString()); server.start(); // execute extra commands diff --git a/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-k8s-e2e/src/test/java/org/apache/seatunnel/engine/e2e/k8s/KubernetesIT.java b/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-k8s-e2e/src/test/java/org/apache/seatunnel/engine/e2e/k8s/KubernetesIT.java index 2de7caeddb5..a4633626c61 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-k8s-e2e/src/test/java/org/apache/seatunnel/engine/e2e/k8s/KubernetesIT.java +++ b/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-k8s-e2e/src/test/java/org/apache/seatunnel/engine/e2e/k8s/KubernetesIT.java @@ -174,8 +174,8 @@ private void copyFileToCurrentResources(String targetPath) throws IOException { Files.copy( Paths.get( PROJECT_ROOT_PATH - + "/seatunnel-shade/seatunnel-hadoop3-3.1.4-uber/target/seatunnel-hadoop3-3.1.4-uber.jar"), - Paths.get(targetPath + "/jars/seatunnel-hadoop3-3.1.4-uber.jar"), + + "/seatunnel-shade/seatunnel-hadoop3-3.3.4-uber/target/seatunnel-hadoop3-3.3.4-uber.jar"), + Paths.get(targetPath + "/jars/seatunnel-hadoop3-3.3.4-uber.jar"), StandardCopyOption.REPLACE_EXISTING); Files.copy( Paths.get( diff --git a/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-k8s-e2e/src/test/resources/seatunnel_dockerfile b/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-k8s-e2e/src/test/resources/seatunnel_dockerfile index d0a230230f2..844b1be5682 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-k8s-e2e/src/test/resources/seatunnel_dockerfile +++ b/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-k8s-e2e/src/test/resources/seatunnel_dockerfile @@ -17,7 +17,7 @@ FROM openjdk:8u162-jdk ENV SEATUNNEL_HOME="/opt/seatunnel" -COPY /jars/seatunnel-hadoop3-3.1.4-uber.jar ${SEATUNNEL_HOME}/lib/seatunnel-hadoop3-3.1.4-uber.jar +COPY /jars/seatunnel-hadoop3-3.3.4-uber.jar ${SEATUNNEL_HOME}/lib/seatunnel-hadoop3-3.3.4-uber.jar COPY /jars/seatunnel-transforms-v2.jar ${SEATUNNEL_HOME}/lib/sseatunnel-transforms-v2.jar COPY /jars/seatunnel-starter.jar ${SEATUNNEL_HOME}/starter/seatunnel-starter.jar COPY /bin ${SEATUNNEL_HOME}/bin diff --git a/seatunnel-engine/seatunnel-engine-client/pom.xml b/seatunnel-engine/seatunnel-engine-client/pom.xml index 975c557303c..0e3288bd4ac 100644 --- a/seatunnel-engine/seatunnel-engine-client/pom.xml +++ b/seatunnel-engine/seatunnel-engine-client/pom.xml @@ -88,7 +88,7 @@ org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber ${project.version} optional test diff --git a/seatunnel-engine/seatunnel-engine-server/pom.xml b/seatunnel-engine/seatunnel-engine-server/pom.xml index abd296cb1e3..b570ae8c32c 100644 --- a/seatunnel-engine/seatunnel-engine-server/pom.xml +++ b/seatunnel-engine/seatunnel-engine-server/pom.xml @@ -85,7 +85,7 @@ org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber ${project.version} optional provided diff --git a/seatunnel-engine/seatunnel-engine-storage/checkpoint-storage-plugins/checkpoint-storage-hdfs/pom.xml b/seatunnel-engine/seatunnel-engine-storage/checkpoint-storage-plugins/checkpoint-storage-hdfs/pom.xml index f7107f9f32b..504f0de1885 100644 --- a/seatunnel-engine/seatunnel-engine-storage/checkpoint-storage-plugins/checkpoint-storage-hdfs/pom.xml +++ b/seatunnel-engine/seatunnel-engine-storage/checkpoint-storage-plugins/checkpoint-storage-hdfs/pom.xml @@ -37,7 +37,7 @@ org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber ${project.version} optional provided diff --git a/seatunnel-engine/seatunnel-engine-storage/imap-storage-plugins/imap-storage-file/pom.xml b/seatunnel-engine/seatunnel-engine-storage/imap-storage-plugins/imap-storage-file/pom.xml index c7eb61012e6..26d224f3490 100644 --- a/seatunnel-engine/seatunnel-engine-storage/imap-storage-plugins/imap-storage-file/pom.xml +++ b/seatunnel-engine/seatunnel-engine-storage/imap-storage-plugins/imap-storage-file/pom.xml @@ -47,7 +47,7 @@ org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber ${project.version} optional provided diff --git a/seatunnel-examples/seatunnel-engine-examples/pom.xml b/seatunnel-examples/seatunnel-engine-examples/pom.xml index 5ebe8ab7305..d80f5b6b5ac 100644 --- a/seatunnel-examples/seatunnel-engine-examples/pom.xml +++ b/seatunnel-examples/seatunnel-engine-examples/pom.xml @@ -44,7 +44,7 @@ org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber ${project.version} diff --git a/seatunnel-shade/pom.xml b/seatunnel-shade/pom.xml index d8c5ff00031..cfe472ddbf4 100644 --- a/seatunnel-shade/pom.xml +++ b/seatunnel-shade/pom.xml @@ -27,7 +27,7 @@ SeaTunnel : Shade : - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber seatunnel-jackson seatunnel-guava seatunnel-arrow-5.0 diff --git a/seatunnel-shade/seatunnel-hadoop3-3.1.4-uber/pom.xml b/seatunnel-shade/seatunnel-hadoop3-3.3.4-uber/pom.xml similarity index 96% rename from seatunnel-shade/seatunnel-hadoop3-3.1.4-uber/pom.xml rename to seatunnel-shade/seatunnel-hadoop3-3.3.4-uber/pom.xml index 322c1ed65b5..889512642a4 100644 --- a/seatunnel-shade/seatunnel-hadoop3-3.1.4-uber/pom.xml +++ b/seatunnel-shade/seatunnel-hadoop3-3.3.4-uber/pom.xml @@ -22,11 +22,11 @@ ${revision} - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber SeaTunnel : Shade : Hadoop3 - 3.1.4 + 3.3.4 27.0-jre @@ -66,7 +66,7 @@ package - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber true true false @@ -122,7 +122,7 @@ - ${basedir}/target/seatunnel-hadoop3-3.1.4-uber.jar + ${basedir}/target/seatunnel-hadoop3-3.3.4-uber.jar jar optional From 9ea4e6a4bf190e9c3ccec721c45b1136797bc68e Mon Sep 17 00:00:00 2001 From: gdliu3 Date: Fri, 12 Jul 2024 11:13:48 +0800 Subject: [PATCH 03/59] 1 --- pom.xml | 1 + .../seatunnel-engine-server/pom.xml | 11 ++++++++++ .../event/JobEventHttpReportHandler.java | 20 +++++++++---------- .../seatunnel-hadoop3-3.3.4-uber/pom.xml | 6 ++++++ 4 files changed, 28 insertions(+), 10 deletions(-) diff --git a/pom.xml b/pom.xml index d7d16782763..7e61d60e8d7 100644 --- a/pom.xml +++ b/pom.xml @@ -133,6 +133,7 @@ 2.29.0 4.5 2.7.0 + 4.12.0 false true diff --git a/seatunnel-engine/seatunnel-engine-server/pom.xml b/seatunnel-engine/seatunnel-engine-server/pom.xml index b570ae8c32c..f10f012bc74 100644 --- a/seatunnel-engine/seatunnel-engine-server/pom.xml +++ b/seatunnel-engine/seatunnel-engine-server/pom.xml @@ -90,6 +90,11 @@ optional provided + + com.squareup.okhttp3 + okhttp + ${okhttp.version} + org.apache.seatunnel seatunnel-e2e-common @@ -102,6 +107,12 @@ mockwebserver 2.7.5 test + + + com.squareup.okio + okio + + diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/event/JobEventHttpReportHandler.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/event/JobEventHttpReportHandler.java index 1182c726730..37dca3f5d20 100644 --- a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/event/JobEventHttpReportHandler.java +++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/event/JobEventHttpReportHandler.java @@ -28,13 +28,13 @@ import com.hazelcast.ringbuffer.ReadResultSet; import com.hazelcast.ringbuffer.Ringbuffer; import com.hazelcast.ringbuffer.impl.RingbufferProxy; -import com.squareup.okhttp.MediaType; -import com.squareup.okhttp.OkHttpClient; -import com.squareup.okhttp.Request; -import com.squareup.okhttp.RequestBody; -import com.squareup.okhttp.Response; -import com.squareup.okhttp.ResponseBody; import lombok.extern.slf4j.Slf4j; +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; +import okhttp3.ResponseBody; import java.io.IOException; import java.time.Duration; @@ -147,9 +147,9 @@ public void close() { } private OkHttpClient createHttpClient() { - OkHttpClient client = new OkHttpClient(); - client.setConnectTimeout(30, TimeUnit.SECONDS); - client.setWriteTimeout(10, TimeUnit.SECONDS); - return client; + OkHttpClient.Builder builder = new OkHttpClient.Builder(); + builder.connectTimeout(30, TimeUnit.SECONDS); + builder.writeTimeout(10, TimeUnit.SECONDS); + return builder.build(); } } diff --git a/seatunnel-shade/seatunnel-hadoop3-3.3.4-uber/pom.xml b/seatunnel-shade/seatunnel-hadoop3-3.3.4-uber/pom.xml index 889512642a4..ec8856a0e23 100644 --- a/seatunnel-shade/seatunnel-hadoop3-3.3.4-uber/pom.xml +++ b/seatunnel-shade/seatunnel-hadoop3-3.3.4-uber/pom.xml @@ -46,6 +46,12 @@ org.apache.hadoop hadoop-client ${hadoop3.version} + + + ch.qos.reload4j + reload4j + + org.xerial.snappy From 01b26065ffae4e10f4981da0708d3a57a02b45fe Mon Sep 17 00:00:00 2001 From: gdliu3 Date: Fri, 12 Jul 2024 12:11:33 +0800 Subject: [PATCH 04/59] 1 --- .../connectors/seatunnel/kudu/catalog/KuduCatalog.java | 2 +- .../seatunnel/connectors/seatunnel/kudu/util/KuduUtil.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/seatunnel-connectors-v2/connector-kudu/src/main/java/org/apache/seatunnel/connectors/seatunnel/kudu/catalog/KuduCatalog.java b/seatunnel-connectors-v2/connector-kudu/src/main/java/org/apache/seatunnel/connectors/seatunnel/kudu/catalog/KuduCatalog.java index b5ba121fb51..c56f89006e1 100644 --- a/seatunnel-connectors-v2/connector-kudu/src/main/java/org/apache/seatunnel/connectors/seatunnel/kudu/catalog/KuduCatalog.java +++ b/seatunnel-connectors-v2/connector-kudu/src/main/java/org/apache/seatunnel/connectors/seatunnel/kudu/catalog/KuduCatalog.java @@ -17,6 +17,7 @@ package org.apache.seatunnel.connectors.seatunnel.kudu.catalog; +import org.apache.commons.lang3.StringUtils; import org.apache.seatunnel.api.table.catalog.Catalog; import org.apache.seatunnel.api.table.catalog.CatalogTable; import org.apache.seatunnel.api.table.catalog.PhysicalColumn; @@ -34,7 +35,6 @@ import org.apache.seatunnel.connectors.seatunnel.kudu.kuduclient.KuduTypeMapper; import org.apache.seatunnel.connectors.seatunnel.kudu.util.KuduUtil; -import org.apache.commons.lang.StringUtils; import org.apache.kudu.ColumnSchema; import org.apache.kudu.Schema; import org.apache.kudu.client.KuduClient; diff --git a/seatunnel-connectors-v2/connector-kudu/src/main/java/org/apache/seatunnel/connectors/seatunnel/kudu/util/KuduUtil.java b/seatunnel-connectors-v2/connector-kudu/src/main/java/org/apache/seatunnel/connectors/seatunnel/kudu/util/KuduUtil.java index 27374d0a3a5..b02e654caac 100644 --- a/seatunnel-connectors-v2/connector-kudu/src/main/java/org/apache/seatunnel/connectors/seatunnel/kudu/util/KuduUtil.java +++ b/seatunnel-connectors-v2/connector-kudu/src/main/java/org/apache/seatunnel/connectors/seatunnel/kudu/util/KuduUtil.java @@ -17,6 +17,7 @@ package org.apache.seatunnel.connectors.seatunnel.kudu.util; +import org.apache.commons.lang3.StringUtils; import org.apache.seatunnel.common.exception.CommonErrorCodeDeprecated; import org.apache.seatunnel.connectors.seatunnel.kudu.config.CommonConfig; import org.apache.seatunnel.connectors.seatunnel.kudu.config.KuduSourceConfig; @@ -24,7 +25,6 @@ import org.apache.seatunnel.connectors.seatunnel.kudu.exception.KuduConnectorErrorCode; import org.apache.seatunnel.connectors.seatunnel.kudu.exception.KuduConnectorException; -import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authentication.util.KerberosName; From 1c5e9f14c4accd0145f575f049fc244cfb59c18c Mon Sep 17 00:00:00 2001 From: gdliu3 Date: Fri, 12 Jul 2024 12:12:24 +0800 Subject: [PATCH 05/59] 1 --- .../connectors/seatunnel/kudu/catalog/KuduCatalog.java | 2 +- .../seatunnel/connectors/seatunnel/kudu/util/KuduUtil.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/seatunnel-connectors-v2/connector-kudu/src/main/java/org/apache/seatunnel/connectors/seatunnel/kudu/catalog/KuduCatalog.java b/seatunnel-connectors-v2/connector-kudu/src/main/java/org/apache/seatunnel/connectors/seatunnel/kudu/catalog/KuduCatalog.java index c56f89006e1..930b84497ea 100644 --- a/seatunnel-connectors-v2/connector-kudu/src/main/java/org/apache/seatunnel/connectors/seatunnel/kudu/catalog/KuduCatalog.java +++ b/seatunnel-connectors-v2/connector-kudu/src/main/java/org/apache/seatunnel/connectors/seatunnel/kudu/catalog/KuduCatalog.java @@ -17,7 +17,6 @@ package org.apache.seatunnel.connectors.seatunnel.kudu.catalog; -import org.apache.commons.lang3.StringUtils; import org.apache.seatunnel.api.table.catalog.Catalog; import org.apache.seatunnel.api.table.catalog.CatalogTable; import org.apache.seatunnel.api.table.catalog.PhysicalColumn; @@ -35,6 +34,7 @@ import org.apache.seatunnel.connectors.seatunnel.kudu.kuduclient.KuduTypeMapper; import org.apache.seatunnel.connectors.seatunnel.kudu.util.KuduUtil; +import org.apache.commons.lang3.StringUtils; import org.apache.kudu.ColumnSchema; import org.apache.kudu.Schema; import org.apache.kudu.client.KuduClient; diff --git a/seatunnel-connectors-v2/connector-kudu/src/main/java/org/apache/seatunnel/connectors/seatunnel/kudu/util/KuduUtil.java b/seatunnel-connectors-v2/connector-kudu/src/main/java/org/apache/seatunnel/connectors/seatunnel/kudu/util/KuduUtil.java index b02e654caac..06963153e78 100644 --- a/seatunnel-connectors-v2/connector-kudu/src/main/java/org/apache/seatunnel/connectors/seatunnel/kudu/util/KuduUtil.java +++ b/seatunnel-connectors-v2/connector-kudu/src/main/java/org/apache/seatunnel/connectors/seatunnel/kudu/util/KuduUtil.java @@ -17,7 +17,6 @@ package org.apache.seatunnel.connectors.seatunnel.kudu.util; -import org.apache.commons.lang3.StringUtils; import org.apache.seatunnel.common.exception.CommonErrorCodeDeprecated; import org.apache.seatunnel.connectors.seatunnel.kudu.config.CommonConfig; import org.apache.seatunnel.connectors.seatunnel.kudu.config.KuduSourceConfig; @@ -25,6 +24,7 @@ import org.apache.seatunnel.connectors.seatunnel.kudu.exception.KuduConnectorErrorCode; import org.apache.seatunnel.connectors.seatunnel.kudu.exception.KuduConnectorException; +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authentication.util.KerberosName; From 919a3d9f7ee29b1dd5bc56f8951b5ee4464dae7e Mon Sep 17 00:00:00 2001 From: gdliu3 Date: Fri, 12 Jul 2024 12:46:03 +0800 Subject: [PATCH 06/59] 1 --- tools/dependencies/known-dependencies.txt | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tools/dependencies/known-dependencies.txt b/tools/dependencies/known-dependencies.txt index 7c802e0c235..174d92bf594 100755 --- a/tools/dependencies/known-dependencies.txt +++ b/tools/dependencies/known-dependencies.txt @@ -13,7 +13,7 @@ jackson-core-2.13.3.jar jackson-databind-2.13.3.jar jackson-dataformat-properties-2.13.3.jar jackson-datatype-jsr310-2.13.3.jar -jcl-over-slf4j-1.7.25.jar +jcl-over-slf4j-1.7.36.jar jcommander-1.81.jar log4j-api-2.17.1.jar log4j-core-2.17.1.jar @@ -27,7 +27,7 @@ scala-library-2.12.15.jar seatunnel-jackson-2.3.6-SNAPSHOT-optional.jar seatunnel-guava-2.3.6-SNAPSHOT-optional.jar seatunnel-hazelcast-shade-2.3.6-SNAPSHOT-optional.jar -slf4j-api-1.7.25.jar +slf4j-api-1.7.36.jar jsqlparser-4.5.jar animal-sniffer-annotations-1.17.jar checker-qual-3.10.0.jar @@ -42,4 +42,12 @@ json-path-2.7.0.jar json-smart-2.4.7.jar accessors-smart-2.4.7.jar asm-9.1.jar -avro-1.11.1.jar \ No newline at end of file +avro-1.11.1.jar +annotations-13.0.jar +kotlin-stdlib-1.8.21.jar +kotlin-stdlib-common-1.9.10.jar +kotlin-stdlib-jdk7-1.8.21.jar +kotlin-stdlib-jdk8-1.8.21.jar +okhttp-4.12.0.jar +okio-3.6.0.jar +okio-jvm-3.6.0.jar \ No newline at end of file From 76adc38a15e98720ad3f064965ef85ba1484190d Mon Sep 17 00:00:00 2001 From: gdliu3 Date: Fri, 12 Jul 2024 17:33:18 +0800 Subject: [PATCH 07/59] 1 --- config/seatunnel-env.cmd | 11 ++++++++++- config/seatunnel-env.sh | 10 ++++++++++ pom.xml | 20 ++++++++++++++++++++ 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/config/seatunnel-env.cmd b/config/seatunnel-env.cmd index 79c2d3c117c..e491034bbef 100644 --- a/config/seatunnel-env.cmd +++ b/config/seatunnel-env.cmd @@ -18,4 +18,13 @@ REM Home directory of spark distribution. if "%SPARK_HOME%" == "" set "SPARK_HOME=C:\Program Files\spark" REM Home directory of flink distribution. -if "%FLINK_HOME%" == "" set "FLINK_HOME=C:\Program Files\flink" \ No newline at end of file +if "%FLINK_HOME%" == "" set "FLINK_HOME=C:\Program Files\flink" + +REM Set JAVA_OPTS +REM These parameters are required for Java 17 support. +REM They can be safely removed when using Java 8/11. +if defined JAVA_OPTS ( + set "JAVA_OPTS=%JAVA_OPTS% --add-exports=java.base/sun.net.util=ALL-UNNAMED --add-exports=java.rmi/sun.rmi.registry=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED --add-exports=java.security.jgss/sun.security.krb5=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.text=ALL-UNNAMED --add-opens=java.base/java.time=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.locks=ALL-UNNAMED" +) else ( + set "JAVA_OPTS=--add-exports=java.base/sun.net.util=ALL-UNNAMED --add-exports=java.rmi/sun.rmi.registry=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED --add-exports=java.security.jgss/sun.security.krb5=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.text=ALL-UNNAMED --add-opens=java.base/java.time=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.locks=ALL-UNNAMED" +) \ No newline at end of file diff --git a/config/seatunnel-env.sh b/config/seatunnel-env.sh index 1bae8c76254..b855b701de6 100644 --- a/config/seatunnel-env.sh +++ b/config/seatunnel-env.sh @@ -20,3 +20,13 @@ SPARK_HOME=${SPARK_HOME:-/opt/spark} # Home directory of flink distribution. FLINK_HOME=${FLINK_HOME:-/opt/flink} + +# Set JAVA_OPTS. +# These parameters are required for Java 17 support. +# They can be safely removed when using Java 8/11. +if [ -n "$JAVA_OPTS" ]; then + JAVA_OPTS="$JAVA_OPTS --add-exports=java.base/sun.net.util=ALL-UNNAMED --add-exports=java.rmi/sun.rmi.registry=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED --add-exports=java.security.jgss/sun.security.krb5=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.text=ALL-UNNAMED --add-opens=java.base/java.time=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.locks=ALL-UNNAMED" +else + JAVA_OPTS="--add-exports=java.base/sun.net.util=ALL-UNNAMED --add-exports=java.rmi/sun.rmi.registry=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED --add-exports=java.security.jgss/sun.security.krb5=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.text=ALL-UNNAMED --add-opens=java.base/java.time=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.locks=ALL-UNNAMED" +fi +export JAVA_OPTS \ No newline at end of file diff --git a/pom.xml b/pom.xml index 7e61d60e8d7..e2e0e5fbfd4 100644 --- a/pom.xml +++ b/pom.xml @@ -586,6 +586,26 @@ ch.qos.logback:logback-core org.apache.logging.log4j:log4j-to-slf4j + --add-exports=java.base/sun.net.util=ALL-UNNAMED + --add-exports=java.rmi/sun.rmi.registry=ALL-UNNAMED - + -add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED + --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED + --add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED + --add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED + --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED + --add-exports=java.security.jgss/sun.security.krb5=ALL-UNNAMED + --add-opens=java.base/java.lang=ALL-UNNAMED + --add-opens=java.base/java.net=ALL-UNNAMED + --add-opens=java.base/java.io=ALL-UNNAMED + --add-opens=java.base/java.nio=ALL-UNNAMED + --add-opens=java.base/sun.nio.ch=ALL-UNNAMED + --add-opens=java.base/java.lang.reflect=ALL-UNNAMED + --add-opens=java.base/java.text=ALL-UNNAMED + --add-opens=java.base/java.time=ALL-UNNAMED + --add-opens=java.base/java.util=ALL-UNNAMED + --add-opens=java.base/java.util.concurrent=ALL-UNNAMED + --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED + --add-opens=java.base/java.util.concurrent.locks=ALL-UNNAMED From 4afcc47106d0a74a1eeb2c489f60f823f739438b Mon Sep 17 00:00:00 2001 From: liugddx Date: Sat, 13 Jul 2024 23:18:26 +0800 Subject: [PATCH 08/59] 1 --- pom.xml | 61 +++++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 40 insertions(+), 21 deletions(-) diff --git a/pom.xml b/pom.xml index e2e0e5fbfd4..4b28fb4c10d 100644 --- a/pom.xml +++ b/pom.xml @@ -91,7 +91,7 @@ false false false - 2.22.2 + 3.3.1 2.22.2 1.6.8 3.0.1 @@ -134,6 +134,7 @@ 4.5 2.7.0 4.12.0 + false true @@ -586,26 +587,7 @@ ch.qos.logback:logback-core org.apache.logging.log4j:log4j-to-slf4j - --add-exports=java.base/sun.net.util=ALL-UNNAMED - --add-exports=java.rmi/sun.rmi.registry=ALL-UNNAMED - - -add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED - --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED - --add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED - --add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED - --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED - --add-exports=java.security.jgss/sun.security.krb5=ALL-UNNAMED - --add-opens=java.base/java.lang=ALL-UNNAMED - --add-opens=java.base/java.net=ALL-UNNAMED - --add-opens=java.base/java.io=ALL-UNNAMED - --add-opens=java.base/java.nio=ALL-UNNAMED - --add-opens=java.base/sun.nio.ch=ALL-UNNAMED - --add-opens=java.base/java.lang.reflect=ALL-UNNAMED - --add-opens=java.base/java.text=ALL-UNNAMED - --add-opens=java.base/java.time=ALL-UNNAMED - --add-opens=java.base/java.util=ALL-UNNAMED - --add-opens=java.base/java.util.concurrent=ALL-UNNAMED - --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED - --add-opens=java.base/java.util.concurrent.locks=ALL-UNNAMED + ${surefire.argLine} @@ -958,6 +940,43 @@ false + + jdk8 + + 1.8 + + + + + + + jdk17 + + [17,) + + + --add-exports java.base/sun.net.util=ALL-UNNAMED + --add-exports java.rmi/sun.rmi.registry=ALL-UNNAMED + --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED + --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED + --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED + --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED + --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED + --add-exports java.security.jgss/sun.security.krb5=ALL-UNNAMED + --add-opens java.base/java.lang=ALL-UNNAMED + --add-opens java.base/java.net=ALL-UNNAMED + --add-opens java.base/java.io=ALL-UNNAMED + --add-opens java.base/java.nio=ALL-UNNAMED + --add-opens java.base/sun.nio.ch=ALL-UNNAMED + --add-opens java.base/java.lang.reflect=ALL-UNNAMED + --add-opens java.base/java.text=ALL-UNNAMED + --add-opens java.base/java.time=ALL-UNNAMED + --add-opens java.base/java.util=ALL-UNNAMED + --add-opens java.base/java.util.concurrent=ALL-UNNAMED + --add-opens java.base/java.util.concurrent.atomic=ALL-UNNAMED + --add-opens java.base/java.util.concurrent.locks=ALL-UNNAMED + + From d6ec68de78123d3f1b720bfcb50e9312c717bca4 Mon Sep 17 00:00:00 2001 From: liugddx Date: Sat, 13 Jul 2024 23:29:46 +0800 Subject: [PATCH 09/59] 1 --- .github/workflows/backend.yml | 66 +++++++++++++------------- .github/workflows/schedule_backend.yml | 4 +- pom.xml | 9 ++++ 3 files changed, 44 insertions(+), 35 deletions(-) diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index 953503d3f52..327928babca 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -287,7 +287,7 @@ jobs: cache: 'maven' - name: run all modules unit test run: | - ./mvnw -B -T 1 clean verify -D"maven.test.skip"=false -D"license.skipAddThirdParty"=true --no-snapshot-updates + ./mvnw -B -T 1 clean verify -Pjdk${{ matrix.java }} -D"maven.test.skip"=false -D"license.skipAddThirdParty"=true --no-snapshot-updates env: MAVEN_OPTS: -Xmx4096m @@ -315,7 +315,7 @@ jobs: run: | sub_modules=`python tools/update_modules_check/update_modules_check.py sub_update_it_module ${{needs.changes.outputs.it-modules}} 8 0` if [ ! -z $sub_modules ]; then - ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am -Pci + ./mvnw -T 1 -B verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am -Pci else echo "sub modules is empty, skipping" fi @@ -346,7 +346,7 @@ jobs: run: | sub_modules=`python tools/update_modules_check/update_modules_check.py sub_update_it_module ${{needs.changes.outputs.it-modules}} 8 1` if [ ! -z $sub_modules ]; then - ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am -Pci + ./mvnw -T 1 -B verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am -Pci else echo "sub modules is empty, skipping" fi @@ -377,7 +377,7 @@ jobs: run: | sub_modules=`python tools/update_modules_check/update_modules_check.py sub_update_it_module ${{needs.changes.outputs.it-modules}} 8 2` if [ ! -z $sub_modules ]; then - ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am -Pci + ./mvnw -T 1 -B verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am -Pci else echo "sub modules is empty, skipping" fi @@ -408,7 +408,7 @@ jobs: run: | sub_modules=`python tools/update_modules_check/update_modules_check.py sub_update_it_module ${{needs.changes.outputs.it-modules}} 8 3` if [ ! -z $sub_modules ]; then - ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am -Pci + ./mvnw -T 1 -B verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am -Pci else echo "sub modules is empty, skipping" fi @@ -438,7 +438,7 @@ jobs: run: | sub_modules=`python tools/update_modules_check/update_modules_check.py sub_update_it_module ${{needs.changes.outputs.it-modules}} 8 4` if [ ! -z $sub_modules ]; then - ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am -Pci + ./mvnw -T 1 -B verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am -Pci else echo "sub modules is empty, skipping" fi @@ -468,7 +468,7 @@ jobs: run: | sub_modules=`python tools/update_modules_check/update_modules_check.py sub_update_it_module ${{needs.changes.outputs.it-modules}} 8 5` if [ ! -z $sub_modules ]; then - ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am -Pci + ./mvnw -T 1 -B verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am -Pci else echo "sub modules is empty, skipping" fi @@ -498,7 +498,7 @@ jobs: run: | sub_modules=`python tools/update_modules_check/update_modules_check.py sub_update_it_module ${{needs.changes.outputs.it-modules}} 8 6` if [ ! -z $sub_modules ]; then - ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am -Pci + ./mvnw -T 1 -B verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am -Pci else echo "sub modules is empty, skipping" fi @@ -529,7 +529,7 @@ jobs: run: | sub_modules=`python tools/update_modules_check/update_modules_check.py sub_update_it_module ${{needs.changes.outputs.it-modules}} 8 7` if [ ! -z $sub_modules ]; then - ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am -Pci + ./mvnw -T 1 -B verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am -Pci else echo "sub modules is empty, skipping" fi @@ -556,7 +556,7 @@ jobs: - name: run seatunnel zeta integration test if: needs.changes.outputs.api == 'true' run: | - ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-seatunnel-e2e-base,:connector-console-seatunnel-e2e -am -Pci + ./mvnw -T 1 -B verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-seatunnel-e2e-base,:connector-console-seatunnel-e2e -am -Pci env: MAVEN_OPTS: -Xmx4096m engine-k8s-it: @@ -589,7 +589,7 @@ jobs: - name: run seatunnel zeta on k8s test if: needs.changes.outputs.api == 'true' run: | - ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :seatunnel-engine-k8s-e2e -am -Pci + ./mvnw -T 1 -B verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :seatunnel-engine-k8s-e2e -am -Pci env: MAVEN_OPTS: -Xmx4096m KUBECONFIG: /etc/rancher/k3s/k3s.yaml @@ -613,7 +613,7 @@ jobs: - name: run transform-v2 integration test (part-1) if: needs.changes.outputs.api == 'true' run: | - ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :seatunnel-transforms-v2-e2e-part-1 -am -Pci + ./mvnw -B -T 1 verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :seatunnel-transforms-v2-e2e-part-1 -am -Pci env: MAVEN_OPTS: -Xmx4096m @@ -637,7 +637,7 @@ jobs: - name: run transform-v2 integration test (part-2) if: needs.changes.outputs.api == 'true' run: | - ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :seatunnel-transforms-v2-e2e-part-2 -am -Pci + ./mvnw -B -T 1 verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :seatunnel-transforms-v2-e2e-part-2 -am -Pci env: MAVEN_OPTS: -Xmx4096m @@ -664,7 +664,7 @@ jobs: ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt` run_it_modules=`python tools/update_modules_check/update_modules_check.py sub_it_module "$sub_modules" 7 0` - ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $run_it_modules -am -Pci + ./mvnw -B -T 1 verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $run_it_modules -am -Pci env: MAVEN_OPTS: -Xmx4096m @@ -691,7 +691,7 @@ jobs: ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt` run_it_modules=`python tools/update_modules_check/update_modules_check.py sub_it_module "$sub_modules" 7 1` - ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $run_it_modules -am -Pci + ./mvnw -B -T 1 verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $run_it_modules -am -Pci env: MAVEN_OPTS: -Xmx4096m @@ -718,7 +718,7 @@ jobs: ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt` run_it_modules=`python tools/update_modules_check/update_modules_check.py sub_it_module "$sub_modules" 7 2` - ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $run_it_modules -am -Pci + ./mvnw -B -T 1 verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $run_it_modules -am -Pci env: MAVEN_OPTS: -Xmx4096m @@ -745,7 +745,7 @@ jobs: ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt` run_it_modules=`python tools/update_modules_check/update_modules_check.py sub_it_module "$sub_modules" 7 3` - ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $run_it_modules -am -Pci + ./mvnw -B -T 1 verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $run_it_modules -am -Pci env: MAVEN_OPTS: -Xmx4096m @@ -772,7 +772,7 @@ jobs: ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt` run_it_modules=`python tools/update_modules_check/update_modules_check.py sub_it_module "$sub_modules" 7 4` - ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $run_it_modules -am -Pci + ./mvnw -B -T 1 verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $run_it_modules -am -Pci env: MAVEN_OPTS: -Xmx4096m @@ -799,7 +799,7 @@ jobs: ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt` run_it_modules=`python tools/update_modules_check/update_modules_check.py sub_it_module "$sub_modules" 7 5` - ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $run_it_modules -am -Pci + ./mvnw -B -T 1 verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $run_it_modules -am -Pci env: MAVEN_OPTS: -Xmx4096m @@ -826,7 +826,7 @@ jobs: ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt` run_it_modules=`python tools/update_modules_check/update_modules_check.py sub_it_module "$sub_modules" 7 6` - ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $run_it_modules -am -Pci + ./mvnw -B -T 1 verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $run_it_modules -am -Pci env: MAVEN_OPTS: -Xmx4096m @@ -852,7 +852,7 @@ jobs: - name: run jdbc connectors integration test (part-1) if: needs.changes.outputs.api == 'true' run: | - ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-jdbc-e2e-part-1 -am -Pci + ./mvnw -B -T 1 verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-jdbc-e2e-part-1 -am -Pci env: MAVEN_OPTS: -Xmx4096m @@ -878,7 +878,7 @@ jobs: - name: run jdbc connectors integration test (part-2) if: needs.changes.outputs.api == 'true' run: | - ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-jdbc-e2e-part-2 -am -Pci + ./mvnw -B -T 1 verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-jdbc-e2e-part-2 -am -Pci env: MAVEN_OPTS: -Xmx4096m @@ -902,7 +902,7 @@ jobs: - name: run jdbc connectors integration test (part-3) if: needs.changes.outputs.api == 'true' run: | - ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-jdbc-e2e-part-3 -am -Pci + ./mvnw -B -T 1 verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-jdbc-e2e-part-3 -am -Pci env: MAVEN_OPTS: -Xmx4096m @@ -926,7 +926,7 @@ jobs: - name: run jdbc connectors integration test (part-4) if: needs.changes.outputs.api == 'true' run: | - ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-jdbc-e2e-part-4 -am -Pci + ./mvnw -B -T 1 verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-jdbc-e2e-part-4 -am -Pci env: MAVEN_OPTS: -Xmx4096m @@ -950,7 +950,7 @@ jobs: - name: run jdbc connectors integration test (part-5) if: needs.changes.outputs.api == 'true' run: | - ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-jdbc-e2e-part-5 -am -Pci + ./mvnw -B -T 1 verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-jdbc-e2e-part-5 -am -Pci env: MAVEN_OPTS: -Xmx4096m @@ -976,7 +976,7 @@ jobs: - name: run jdbc connectors integration test (part-6) if: needs.changes.outputs.api == 'true' run: | - ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-jdbc-e2e-part-6 -am -Pci + ./mvnw -B -T 1 verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-jdbc-e2e-part-6 -am -Pci env: MAVEN_OPTS: -Xmx4096m @@ -1000,7 +1000,7 @@ jobs: - name: run jdbc connectors integration test (part-7) if: needs.changes.outputs.api == 'true' run: | - ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-jdbc-e2e-part-7 -am -Pci + ./mvnw -B -T 1 verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-jdbc-e2e-part-7 -am -Pci env: MAVEN_OPTS: -Xmx4096m @@ -1023,7 +1023,7 @@ jobs: cache: 'maven' - name: run kudu connector integration test run: | - ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-kudu-e2e -am -Pci + ./mvnw -B -T 1 verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-kudu-e2e -am -Pci env: MAVEN_OPTS: -Xmx4096m @@ -1046,7 +1046,7 @@ jobs: cache: 'maven' - name: run amazonsqs connector integration test run: | - ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-amazonsqs-e2e -am -Pci + ./mvnw -B -T 1 verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-amazonsqs-e2e -am -Pci env: MAVEN_OPTS: -Xmx4096m @@ -1069,7 +1069,7 @@ jobs: cache: 'maven' - name: run kafka connector integration test run: | - ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-kafka-e2e -am -Pci + ./mvnw -B -T 1 verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-kafka-e2e -am -Pci env: MAVEN_OPTS: -Xmx4096m @@ -1092,7 +1092,7 @@ jobs: cache: 'maven' - name: run rocket connector integration test run: | - ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-rocketmq-e2e -am -Pci + ./mvnw -B -T 1 verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-rocketmq-e2e -am -Pci env: MAVEN_OPTS: -Xmx4096m @@ -1118,7 +1118,7 @@ jobs: run: tools/github/free_disk_space.sh - name: run doris connector integration test run: | - ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-doris-e2e -am -Pci + ./mvnw -B -T 1 verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-doris-e2e -am -Pci env: MAVEN_OPTS: -Xmx4096m @@ -1142,4 +1142,4 @@ jobs: cache: 'maven' - name: run oracle cdc connector integration test run: | - ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-cdc-oracle-e2e -am -Pci \ No newline at end of file + ./mvnw -B -T 1 verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-cdc-oracle-e2e -am -Pci \ No newline at end of file diff --git a/.github/workflows/schedule_backend.yml b/.github/workflows/schedule_backend.yml index b41059b4e36..f4f2d75fcbc 100644 --- a/.github/workflows/schedule_backend.yml +++ b/.github/workflows/schedule_backend.yml @@ -117,7 +117,7 @@ jobs: cache: 'maven' - name: run all modules unit test run: | - ./mvnw -B -T 1C clean verify -D"maven.test.skip"=false -D"scalastyle.skip"=true -D"license.skipAddThirdParty"=true --no-snapshot-updates + ./mvnw -B -T 1C clean verify -Pjdk${{ matrix.java }} -D"maven.test.skip"=false -D"scalastyle.skip"=true -D"license.skipAddThirdParty"=true --no-snapshot-updates env: MAVEN_OPTS: -Xmx2048m @@ -139,7 +139,7 @@ jobs: cache: 'maven' - name: run all modules integration test run: | - ./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false -D"scalastyle.skip"=true -D"license.skipAddThirdParty"=true --no-snapshot-updates + ./mvnw -T 1C -B verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"scalastyle.skip"=true -D"license.skipAddThirdParty"=true --no-snapshot-updates env: MAVEN_OPTS: -Xmx2048m diff --git a/pom.xml b/pom.xml index 4b28fb4c10d..c818d2926e3 100644 --- a/pom.xml +++ b/pom.xml @@ -949,6 +949,15 @@ + + jdk11 + + 11 + + + + + jdk17 From 41fc615ee62767d32a0b4004e6e8d4c11e730759 Mon Sep 17 00:00:00 2001 From: liugddx Date: Sun, 14 Jul 2024 09:57:55 +0800 Subject: [PATCH 10/59] 1 --- .../seatunnel/core/starter/spark/SparkCommandArgsTest.java | 2 +- .../core/starter/spark/multitable/MultiTableSinkTest.java | 2 +- .../org/apache/seatunnel/engine/e2e/CheckpointEnableIT.java | 4 +++- .../seatunnel/translation/spark/sink/SparkSinkTest.java | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-3-starter/src/test/java/org/apache/seatunnel/core/starter/spark/SparkCommandArgsTest.java b/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-3-starter/src/test/java/org/apache/seatunnel/core/starter/spark/SparkCommandArgsTest.java index da997e33559..c263b05b481 100644 --- a/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-3-starter/src/test/java/org/apache/seatunnel/core/starter/spark/SparkCommandArgsTest.java +++ b/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-3-starter/src/test/java/org/apache/seatunnel/core/starter/spark/SparkCommandArgsTest.java @@ -34,7 +34,7 @@ import static org.apache.seatunnel.api.common.CommonOptions.PLUGIN_NAME; @DisabledOnJre( - value = JRE.JAVA_11, + value = {JRE.JAVA_11, JRE.JAVA_17}, disabledReason = "We should update apache common lang3 version to 3.8 to avoid NPE, " + "see https://github.com/apache/commons-lang/commit/50ce8c44e1601acffa39f5568f0fc140aade0564") diff --git a/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-3-starter/src/test/java/org/apache/seatunnel/core/starter/spark/multitable/MultiTableSinkTest.java b/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-3-starter/src/test/java/org/apache/seatunnel/core/starter/spark/multitable/MultiTableSinkTest.java index 61ed44515c7..08734be5f98 100644 --- a/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-3-starter/src/test/java/org/apache/seatunnel/core/starter/spark/multitable/MultiTableSinkTest.java +++ b/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-3-starter/src/test/java/org/apache/seatunnel/core/starter/spark/multitable/MultiTableSinkTest.java @@ -46,7 +46,7 @@ public class MultiTableSinkTest { @Test @DisabledOnJre( - value = JRE.JAVA_11, + value = {JRE.JAVA_11, JRE.JAVA_17}, disabledReason = "We should update apache common lang3 version to 3.8 to avoid NPE, " + "see https://github.com/apache/commons-lang/commit/50ce8c44e1601acffa39f5568f0fc140aade0564") diff --git a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/CheckpointEnableIT.java b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/CheckpointEnableIT.java index 66785d5b649..4723d69eb7e 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/CheckpointEnableIT.java +++ b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/CheckpointEnableIT.java @@ -44,7 +44,9 @@ import static org.awaitility.Awaitility.await; @Slf4j -@DisabledOnJre(value = JRE.JAVA_11, disabledReason = "slf4j jar conflict, we should fix it later") +@DisabledOnJre( + value = {JRE.JAVA_11, JRE.JAVA_17}, + disabledReason = "slf4j jar conflict, we should fix it later") public class CheckpointEnableIT extends TestSuiteBase { @TestTemplate diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/test/java/org/apache/seatunnel/translation/spark/sink/SparkSinkTest.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/test/java/org/apache/seatunnel/translation/spark/sink/SparkSinkTest.java index b426f175480..cf8e6aa6ac0 100644 --- a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/test/java/org/apache/seatunnel/translation/spark/sink/SparkSinkTest.java +++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/test/java/org/apache/seatunnel/translation/spark/sink/SparkSinkTest.java @@ -59,7 +59,7 @@ public class SparkSinkTest { @Test @DisabledOnJre( - value = JRE.JAVA_11, + value = {JRE.JAVA_11, JRE.JAVA_17}, disabledReason = "We should update apache common lang3 version to 3.8 to avoid NPE, " + "see https://github.com/apache/commons-lang/commit/50ce8c44e1601acffa39f5568f0fc140aade0564") From 1ed9e9375fe86773cacf211f38ec5aba0dc8221f Mon Sep 17 00:00:00 2001 From: liugddx Date: Sun, 14 Jul 2024 11:44:09 +0800 Subject: [PATCH 11/59] 1 --- seatunnel-connectors-v2/connector-jdbc/pom.xml | 4 ++-- .../seatunnel-connector-v2-e2e/connector-kafka-e2e/pom.xml | 5 ----- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/seatunnel-connectors-v2/connector-jdbc/pom.xml b/seatunnel-connectors-v2/connector-jdbc/pom.xml index cfe0ed44ecd..7cddf8804e4 100644 --- a/seatunnel-connectors-v2/connector-jdbc/pom.xml +++ b/seatunnel-connectors-v2/connector-jdbc/pom.xml @@ -30,8 +30,8 @@ SeaTunnel : Connectors V2 : Jdbc - 8.0.27 - 42.4.3 + 8.0.32 + 42.7.2 8.1.2.141 9.2.1.jre8 5.2.5-HBase-2.x diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/pom.xml index fa2e1930cce..128f167fe8b 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/pom.xml @@ -109,11 +109,6 @@ ${testcontainer.version} test - - mysql - mysql-connector-java - test - From 1e7232115fc71dfd31718aaa4809e116a921d6e1 Mon Sep 17 00:00:00 2001 From: liugddx Date: Sun, 14 Jul 2024 12:59:12 +0800 Subject: [PATCH 12/59] 1 --- seatunnel-connectors-v2/connector-starrocks/pom.xml | 2 +- seatunnel-dist/pom.xml | 12 +++--------- .../connector-cdc-mongodb-e2e/pom.xml | 2 +- .../connector-doris-e2e/pom.xml | 2 +- .../connector-starrocks-e2e/pom.xml | 2 +- 5 files changed, 7 insertions(+), 13 deletions(-) diff --git a/seatunnel-connectors-v2/connector-starrocks/pom.xml b/seatunnel-connectors-v2/connector-starrocks/pom.xml index 08e49bc0f02..320f3acfc18 100644 --- a/seatunnel-connectors-v2/connector-starrocks/pom.xml +++ b/seatunnel-connectors-v2/connector-starrocks/pom.xml @@ -33,7 +33,7 @@ connector.starrocks 4.5.13 4.4.4 - 8.0.16 + 8.0.32 1.0.1 5.0.0 diff --git a/seatunnel-dist/pom.xml b/seatunnel-dist/pom.xml index 278829f2b01..e6eca3695eb 100644 --- a/seatunnel-dist/pom.xml +++ b/seatunnel-dist/pom.xml @@ -83,8 +83,8 @@ - 8.0.27 - 42.4.3 + 8.0.32 + 42.7.2 2.5.1 8.1.2.141 9.2.1.jre8 @@ -695,7 +695,7 @@ com.amazonaws aws-java-sdk-bundle - 1.11.271 + ${aws-java-sdk.version} provided @@ -876,12 +876,6 @@ ${project.version} provided - - com.amazonaws - aws-java-sdk-bundle - ${aws-java-sdk.version} - provided - org.apache.seatunnel diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mongodb-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mongodb-e2e/pom.xml index a8814c11ee6..45cd5632c82 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mongodb-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mongodb-e2e/pom.xml @@ -26,7 +26,7 @@ SeaTunnel : E2E : Connector V2 : CDC Mongodb - 8.0.16 + 8.0.32 diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/pom.xml index af85d92acef..8d9684648c0 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/pom.xml @@ -26,7 +26,7 @@ SeaTunnel : E2E : Connector V2 : Doris - 8.0.31 + 8.0.32 diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-starrocks-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-starrocks-e2e/pom.xml index 38e4f7eabee..f9b031215b8 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-starrocks-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-starrocks-e2e/pom.xml @@ -26,7 +26,7 @@ SeaTunnel : E2E : Connector V2 : StarRocks - 8.0.27 + 8.0.32 From d502478c5a0642d879fb2a40568ba1deb99242c6 Mon Sep 17 00:00:00 2001 From: liugddx Date: Sun, 14 Jul 2024 13:55:28 +0800 Subject: [PATCH 13/59] 1 --- config/seatunnel-env.cmd | 32 ++++++++++++++++++++++++-------- config/seatunnel-env.sh | 25 +++++++++++++++++++------ 2 files changed, 43 insertions(+), 14 deletions(-) diff --git a/config/seatunnel-env.cmd b/config/seatunnel-env.cmd index e491034bbef..5390c0d7641 100644 --- a/config/seatunnel-env.cmd +++ b/config/seatunnel-env.cmd @@ -20,11 +20,27 @@ if "%SPARK_HOME%" == "" set "SPARK_HOME=C:\Program Files\spark" REM Home directory of flink distribution. if "%FLINK_HOME%" == "" set "FLINK_HOME=C:\Program Files\flink" -REM Set JAVA_OPTS -REM These parameters are required for Java 17 support. -REM They can be safely removed when using Java 8/11. -if defined JAVA_OPTS ( - set "JAVA_OPTS=%JAVA_OPTS% --add-exports=java.base/sun.net.util=ALL-UNNAMED --add-exports=java.rmi/sun.rmi.registry=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED --add-exports=java.security.jgss/sun.security.krb5=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.text=ALL-UNNAMED --add-opens=java.base/java.time=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.locks=ALL-UNNAMED" -) else ( - set "JAVA_OPTS=--add-exports=java.base/sun.net.util=ALL-UNNAMED --add-exports=java.rmi/sun.rmi.registry=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED --add-exports=java.security.jgss/sun.security.krb5=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.text=ALL-UNNAMED --add-opens=java.base/java.time=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.locks=ALL-UNNAMED" -) \ No newline at end of file +REM Initialize JAVA_OPTS to avoid unbound variable error. +if not defined JAVA_OPTS ( + set "JAVA_OPTS=" +) + +REM Function to check if the current Java version is 17 or higher. +for /f "tokens=2 delims==" %%v in ('java -XshowSettings:properties -version 2>&1 ^| findstr "java.version"') do ( + set "java_version=%%v" +) +for /f "tokens=1 delims=." %%v in ("%java_version%") do ( + set "java_major_version=%%v" +) + +REM Set JAVA_OPTS if Java version is 17 or newer. +if %java_major_version% geq 17 ( + if defined JAVA_OPTS ( + set "JAVA_OPTS=%JAVA_OPTS% --add-exports=java.base/sun.net.util=ALL-UNNAMED --add-exports=java.rmi/sun.rmi.registry=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED --add-exports=java.security.jgss/sun.security.krb5=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.text=ALL-UNNAMED --add-opens=java.base/java.time=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.locks=ALL-UNNAMED" + ) else ( + set "JAVA_OPTS=--add-exports=java.base/sun.net.util=ALL-UNNAMED --add-exports=java.rmi/sun.rmi.registry=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED --add-exports=java.security.jgss/sun.security.krb5=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.text=ALL-UNNAMED --add-opens=java.base/java.time=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.locks=ALL-UNNAMED" + ) +) + +REM Export JAVA_OPTS +set JAVA_OPTS \ No newline at end of file diff --git a/config/seatunnel-env.sh b/config/seatunnel-env.sh index b855b701de6..20583f7dcb8 100644 --- a/config/seatunnel-env.sh +++ b/config/seatunnel-env.sh @@ -21,12 +21,25 @@ SPARK_HOME=${SPARK_HOME:-/opt/spark} # Home directory of flink distribution. FLINK_HOME=${FLINK_HOME:-/opt/flink} -# Set JAVA_OPTS. -# These parameters are required for Java 17 support. -# They can be safely removed when using Java 8/11. -if [ -n "$JAVA_OPTS" ]; then +# Initialize JAVA_OPTS to avoid unbound variable error. +JAVA_OPTS=${JAVA_OPTS:-} + +# Function to check if the current Java version is 17 or higher. +is_java_17_or_newer() { + java_version_output=$(java -version 2>&1) + if [[ $java_version_output == *"version \""* ]]; then + java_version=$(echo $java_version_output | awk -F '"' '/version/ {print $2}') + java_major_version=$(echo $java_version | cut -d'.' -f1) + if [[ "$java_major_version" -ge 17 ]]; then + return 0 + fi + fi + return 1 +} + +# Set JAVA_OPTS if Java version is 17 or newer. +if is_java_17_or_newer; then JAVA_OPTS="$JAVA_OPTS --add-exports=java.base/sun.net.util=ALL-UNNAMED --add-exports=java.rmi/sun.rmi.registry=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED --add-exports=java.security.jgss/sun.security.krb5=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.text=ALL-UNNAMED --add-opens=java.base/java.time=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.locks=ALL-UNNAMED" -else - JAVA_OPTS="--add-exports=java.base/sun.net.util=ALL-UNNAMED --add-exports=java.rmi/sun.rmi.registry=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED --add-exports=java.security.jgss/sun.security.krb5=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.text=ALL-UNNAMED --add-opens=java.base/java.time=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.locks=ALL-UNNAMED" fi + export JAVA_OPTS \ No newline at end of file From adc337e038483d584b6fe9099849086b7f84a6bf Mon Sep 17 00:00:00 2001 From: liugddx Date: Sun, 14 Jul 2024 15:01:31 +0800 Subject: [PATCH 14/59] 1 --- .github/workflows/backend.yml | 66 +++++++++++++++--------------- tools/dependencies/checkLicense.sh | 2 +- 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index 327928babca..3a37a7d8224 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -274,7 +274,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ,'17'] + java: [ '8','17'] os: [ 'ubuntu-latest', 'windows-latest' ] timeout-minutes: 60 steps: @@ -297,7 +297,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ,'17'] + java: [ '8','17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -328,7 +328,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ,'17'] + java: [ '8','17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -359,7 +359,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ,'17'] + java: [ '8','17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -390,7 +390,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ,'17'] + java: [ '8','17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -420,7 +420,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ,'17'] + java: [ '8','17'] os: [ 'ubuntu-latest' ] timeout-minutes: 120 steps: @@ -450,7 +450,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ,'17'] + java: [ '8','17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -480,7 +480,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ,'17'] + java: [ '8','17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -511,7 +511,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ,'17'] + java: [ '8','17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -542,7 +542,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ,'17'] + java: [ '8','17'] os: [ 'ubuntu-latest' ] timeout-minutes: 45 steps: @@ -565,7 +565,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ,'17'] + java: [ '8','17'] os: [ 'ubuntu-latest' ] timeout-minutes: 45 steps: @@ -599,7 +599,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ,'17'] + java: [ '8','17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -623,7 +623,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ,'17'] + java: [ '8','17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -647,7 +647,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ,'17'] + java: [ '8','17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -674,7 +674,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ,'17'] + java: [ '8','17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -701,7 +701,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ,'17'] + java: [ '8','17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -728,7 +728,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ,'17'] + java: [ '8','17'] os: [ 'ubuntu-latest' ] timeout-minutes: 150 steps: @@ -755,7 +755,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ,'17'] + java: [ '8','17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -782,7 +782,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ,'17'] + java: [ '8','17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -809,7 +809,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ,'17'] + java: [ '8','17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -836,7 +836,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ,'17'] + java: [ '8','17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -862,7 +862,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ,'17'] + java: [ '8','17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -888,7 +888,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ,'17'] + java: [ '8','17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -912,7 +912,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ,'17'] + java: [ '8','17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -936,7 +936,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ,'17'] + java: [ '8','17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -960,7 +960,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ,'17'] + java: [ '8','17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -986,7 +986,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ,'17'] + java: [ '8','17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -1010,7 +1010,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ,'17'] + java: [ '8','17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -1033,7 +1033,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ,'17'] + java: [ '8','17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -1056,7 +1056,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ,'17'] + java: [ '8','17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -1079,7 +1079,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ,'17'] + java: [ '8','17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -1103,7 +1103,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ,'17'] + java: [ '8','17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -1129,7 +1129,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ,'17'] + java: [ '8','17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: diff --git a/tools/dependencies/checkLicense.sh b/tools/dependencies/checkLicense.sh index 836ca91d354..99f5baa842f 100755 --- a/tools/dependencies/checkLicense.sh +++ b/tools/dependencies/checkLicense.sh @@ -23,7 +23,7 @@ if [ -d "/tmp/seatunnel-dependencies" ]; then rm -rf /tmp/seatunnel-dependencies/* fi -./mvnw clean -pl '!seatunnel-dist' --batch-mode --no-snapshot-updates dependency:copy-dependencies -DincludeScope=runtime -DoutputDirectory=/tmp/seatunnel-dependencies +./mvnw clean -pl '!seatunnel-dist' --batch-mode --no-snapshot-updates dependency:copy-dependencies -Prelease -DincludeScope=runtime -DoutputDirectory=/tmp/seatunnel-dependencies # List all modules(jars) that belong to the SeaTunnel itself, these will be ignored when checking the dependency ls /tmp/seatunnel-dependencies | sort > all-dependencies.txt From 85cfe75cc455762811c0ea442899279b8dfa5b7a Mon Sep 17 00:00:00 2001 From: liugddx Date: Sun, 14 Jul 2024 16:26:23 +0800 Subject: [PATCH 15/59] 1 --- .github/workflows/backend.yml | 66 +++++++++---------- .github/workflows/schedule_backend.yml | 4 +- .../DefaultSeaTunnelRowDeserializer.java | 46 +++++++++++-- 3 files changed, 74 insertions(+), 42 deletions(-) diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index 3a37a7d8224..79d2b03d9ef 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -287,7 +287,7 @@ jobs: cache: 'maven' - name: run all modules unit test run: | - ./mvnw -B -T 1 clean verify -Pjdk${{ matrix.java }} -D"maven.test.skip"=false -D"license.skipAddThirdParty"=true --no-snapshot-updates + ./mvnw -B -T 1 clean verify -D"maven.test.skip"=false -D"license.skipAddThirdParty"=true --no-snapshot-updates env: MAVEN_OPTS: -Xmx4096m @@ -315,7 +315,7 @@ jobs: run: | sub_modules=`python tools/update_modules_check/update_modules_check.py sub_update_it_module ${{needs.changes.outputs.it-modules}} 8 0` if [ ! -z $sub_modules ]; then - ./mvnw -T 1 -B verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am -Pci + ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am -Pci else echo "sub modules is empty, skipping" fi @@ -346,7 +346,7 @@ jobs: run: | sub_modules=`python tools/update_modules_check/update_modules_check.py sub_update_it_module ${{needs.changes.outputs.it-modules}} 8 1` if [ ! -z $sub_modules ]; then - ./mvnw -T 1 -B verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am -Pci + ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am -Pci else echo "sub modules is empty, skipping" fi @@ -377,7 +377,7 @@ jobs: run: | sub_modules=`python tools/update_modules_check/update_modules_check.py sub_update_it_module ${{needs.changes.outputs.it-modules}} 8 2` if [ ! -z $sub_modules ]; then - ./mvnw -T 1 -B verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am -Pci + ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am -Pci else echo "sub modules is empty, skipping" fi @@ -408,7 +408,7 @@ jobs: run: | sub_modules=`python tools/update_modules_check/update_modules_check.py sub_update_it_module ${{needs.changes.outputs.it-modules}} 8 3` if [ ! -z $sub_modules ]; then - ./mvnw -T 1 -B verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am -Pci + ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am -Pci else echo "sub modules is empty, skipping" fi @@ -438,7 +438,7 @@ jobs: run: | sub_modules=`python tools/update_modules_check/update_modules_check.py sub_update_it_module ${{needs.changes.outputs.it-modules}} 8 4` if [ ! -z $sub_modules ]; then - ./mvnw -T 1 -B verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am -Pci + ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am -Pci else echo "sub modules is empty, skipping" fi @@ -468,7 +468,7 @@ jobs: run: | sub_modules=`python tools/update_modules_check/update_modules_check.py sub_update_it_module ${{needs.changes.outputs.it-modules}} 8 5` if [ ! -z $sub_modules ]; then - ./mvnw -T 1 -B verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am -Pci + ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am -Pci else echo "sub modules is empty, skipping" fi @@ -498,7 +498,7 @@ jobs: run: | sub_modules=`python tools/update_modules_check/update_modules_check.py sub_update_it_module ${{needs.changes.outputs.it-modules}} 8 6` if [ ! -z $sub_modules ]; then - ./mvnw -T 1 -B verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am -Pci + ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am -Pci else echo "sub modules is empty, skipping" fi @@ -529,7 +529,7 @@ jobs: run: | sub_modules=`python tools/update_modules_check/update_modules_check.py sub_update_it_module ${{needs.changes.outputs.it-modules}} 8 7` if [ ! -z $sub_modules ]; then - ./mvnw -T 1 -B verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am -Pci + ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am -Pci else echo "sub modules is empty, skipping" fi @@ -556,7 +556,7 @@ jobs: - name: run seatunnel zeta integration test if: needs.changes.outputs.api == 'true' run: | - ./mvnw -T 1 -B verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-seatunnel-e2e-base,:connector-console-seatunnel-e2e -am -Pci + ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-seatunnel-e2e-base,:connector-console-seatunnel-e2e -am -Pci env: MAVEN_OPTS: -Xmx4096m engine-k8s-it: @@ -589,7 +589,7 @@ jobs: - name: run seatunnel zeta on k8s test if: needs.changes.outputs.api == 'true' run: | - ./mvnw -T 1 -B verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :seatunnel-engine-k8s-e2e -am -Pci + ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :seatunnel-engine-k8s-e2e -am -Pci env: MAVEN_OPTS: -Xmx4096m KUBECONFIG: /etc/rancher/k3s/k3s.yaml @@ -613,7 +613,7 @@ jobs: - name: run transform-v2 integration test (part-1) if: needs.changes.outputs.api == 'true' run: | - ./mvnw -B -T 1 verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :seatunnel-transforms-v2-e2e-part-1 -am -Pci + ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :seatunnel-transforms-v2-e2e-part-1 -am -Pci env: MAVEN_OPTS: -Xmx4096m @@ -637,7 +637,7 @@ jobs: - name: run transform-v2 integration test (part-2) if: needs.changes.outputs.api == 'true' run: | - ./mvnw -B -T 1 verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :seatunnel-transforms-v2-e2e-part-2 -am -Pci + ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :seatunnel-transforms-v2-e2e-part-2 -am -Pci env: MAVEN_OPTS: -Xmx4096m @@ -664,7 +664,7 @@ jobs: ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt` run_it_modules=`python tools/update_modules_check/update_modules_check.py sub_it_module "$sub_modules" 7 0` - ./mvnw -B -T 1 verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $run_it_modules -am -Pci + ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $run_it_modules -am -Pci env: MAVEN_OPTS: -Xmx4096m @@ -691,7 +691,7 @@ jobs: ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt` run_it_modules=`python tools/update_modules_check/update_modules_check.py sub_it_module "$sub_modules" 7 1` - ./mvnw -B -T 1 verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $run_it_modules -am -Pci + ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $run_it_modules -am -Pci env: MAVEN_OPTS: -Xmx4096m @@ -718,7 +718,7 @@ jobs: ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt` run_it_modules=`python tools/update_modules_check/update_modules_check.py sub_it_module "$sub_modules" 7 2` - ./mvnw -B -T 1 verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $run_it_modules -am -Pci + ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $run_it_modules -am -Pci env: MAVEN_OPTS: -Xmx4096m @@ -745,7 +745,7 @@ jobs: ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt` run_it_modules=`python tools/update_modules_check/update_modules_check.py sub_it_module "$sub_modules" 7 3` - ./mvnw -B -T 1 verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $run_it_modules -am -Pci + ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $run_it_modules -am -Pci env: MAVEN_OPTS: -Xmx4096m @@ -772,7 +772,7 @@ jobs: ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt` run_it_modules=`python tools/update_modules_check/update_modules_check.py sub_it_module "$sub_modules" 7 4` - ./mvnw -B -T 1 verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $run_it_modules -am -Pci + ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $run_it_modules -am -Pci env: MAVEN_OPTS: -Xmx4096m @@ -799,7 +799,7 @@ jobs: ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt` run_it_modules=`python tools/update_modules_check/update_modules_check.py sub_it_module "$sub_modules" 7 5` - ./mvnw -B -T 1 verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $run_it_modules -am -Pci + ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $run_it_modules -am -Pci env: MAVEN_OPTS: -Xmx4096m @@ -826,7 +826,7 @@ jobs: ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt` run_it_modules=`python tools/update_modules_check/update_modules_check.py sub_it_module "$sub_modules" 7 6` - ./mvnw -B -T 1 verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $run_it_modules -am -Pci + ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $run_it_modules -am -Pci env: MAVEN_OPTS: -Xmx4096m @@ -852,7 +852,7 @@ jobs: - name: run jdbc connectors integration test (part-1) if: needs.changes.outputs.api == 'true' run: | - ./mvnw -B -T 1 verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-jdbc-e2e-part-1 -am -Pci + ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-jdbc-e2e-part-1 -am -Pci env: MAVEN_OPTS: -Xmx4096m @@ -878,7 +878,7 @@ jobs: - name: run jdbc connectors integration test (part-2) if: needs.changes.outputs.api == 'true' run: | - ./mvnw -B -T 1 verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-jdbc-e2e-part-2 -am -Pci + ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-jdbc-e2e-part-2 -am -Pci env: MAVEN_OPTS: -Xmx4096m @@ -902,7 +902,7 @@ jobs: - name: run jdbc connectors integration test (part-3) if: needs.changes.outputs.api == 'true' run: | - ./mvnw -B -T 1 verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-jdbc-e2e-part-3 -am -Pci + ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-jdbc-e2e-part-3 -am -Pci env: MAVEN_OPTS: -Xmx4096m @@ -926,7 +926,7 @@ jobs: - name: run jdbc connectors integration test (part-4) if: needs.changes.outputs.api == 'true' run: | - ./mvnw -B -T 1 verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-jdbc-e2e-part-4 -am -Pci + ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-jdbc-e2e-part-4 -am -Pci env: MAVEN_OPTS: -Xmx4096m @@ -950,7 +950,7 @@ jobs: - name: run jdbc connectors integration test (part-5) if: needs.changes.outputs.api == 'true' run: | - ./mvnw -B -T 1 verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-jdbc-e2e-part-5 -am -Pci + ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-jdbc-e2e-part-5 -am -Pci env: MAVEN_OPTS: -Xmx4096m @@ -976,7 +976,7 @@ jobs: - name: run jdbc connectors integration test (part-6) if: needs.changes.outputs.api == 'true' run: | - ./mvnw -B -T 1 verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-jdbc-e2e-part-6 -am -Pci + ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-jdbc-e2e-part-6 -am -Pci env: MAVEN_OPTS: -Xmx4096m @@ -1000,7 +1000,7 @@ jobs: - name: run jdbc connectors integration test (part-7) if: needs.changes.outputs.api == 'true' run: | - ./mvnw -B -T 1 verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-jdbc-e2e-part-7 -am -Pci + ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-jdbc-e2e-part-7 -am -Pci env: MAVEN_OPTS: -Xmx4096m @@ -1023,7 +1023,7 @@ jobs: cache: 'maven' - name: run kudu connector integration test run: | - ./mvnw -B -T 1 verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-kudu-e2e -am -Pci + ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-kudu-e2e -am -Pci env: MAVEN_OPTS: -Xmx4096m @@ -1046,7 +1046,7 @@ jobs: cache: 'maven' - name: run amazonsqs connector integration test run: | - ./mvnw -B -T 1 verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-amazonsqs-e2e -am -Pci + ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-amazonsqs-e2e -am -Pci env: MAVEN_OPTS: -Xmx4096m @@ -1069,7 +1069,7 @@ jobs: cache: 'maven' - name: run kafka connector integration test run: | - ./mvnw -B -T 1 verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-kafka-e2e -am -Pci + ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-kafka-e2e -am -Pci env: MAVEN_OPTS: -Xmx4096m @@ -1092,7 +1092,7 @@ jobs: cache: 'maven' - name: run rocket connector integration test run: | - ./mvnw -B -T 1 verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-rocketmq-e2e -am -Pci + ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-rocketmq-e2e -am -Pci env: MAVEN_OPTS: -Xmx4096m @@ -1118,7 +1118,7 @@ jobs: run: tools/github/free_disk_space.sh - name: run doris connector integration test run: | - ./mvnw -B -T 1 verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-doris-e2e -am -Pci + ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-doris-e2e -am -Pci env: MAVEN_OPTS: -Xmx4096m @@ -1142,4 +1142,4 @@ jobs: cache: 'maven' - name: run oracle cdc connector integration test run: | - ./mvnw -B -T 1 verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-cdc-oracle-e2e -am -Pci \ No newline at end of file + ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-cdc-oracle-e2e -am -Pci \ No newline at end of file diff --git a/.github/workflows/schedule_backend.yml b/.github/workflows/schedule_backend.yml index f4f2d75fcbc..b41059b4e36 100644 --- a/.github/workflows/schedule_backend.yml +++ b/.github/workflows/schedule_backend.yml @@ -117,7 +117,7 @@ jobs: cache: 'maven' - name: run all modules unit test run: | - ./mvnw -B -T 1C clean verify -Pjdk${{ matrix.java }} -D"maven.test.skip"=false -D"scalastyle.skip"=true -D"license.skipAddThirdParty"=true --no-snapshot-updates + ./mvnw -B -T 1C clean verify -D"maven.test.skip"=false -D"scalastyle.skip"=true -D"license.skipAddThirdParty"=true --no-snapshot-updates env: MAVEN_OPTS: -Xmx2048m @@ -139,7 +139,7 @@ jobs: cache: 'maven' - name: run all modules integration test run: | - ./mvnw -T 1C -B verify -Pjdk${{ matrix.java }} -DskipUT=true -DskipIT=false -D"scalastyle.skip"=true -D"license.skipAddThirdParty"=true --no-snapshot-updates + ./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false -D"scalastyle.skip"=true -D"license.skipAddThirdParty"=true --no-snapshot-updates env: MAVEN_OPTS: -Xmx2048m diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/serialize/DefaultSeaTunnelRowDeserializer.java b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/serialize/DefaultSeaTunnelRowDeserializer.java index a531befe9a6..2cee95a61ee 100644 --- a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/serialize/DefaultSeaTunnelRowDeserializer.java +++ b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/serialize/DefaultSeaTunnelRowDeserializer.java @@ -23,6 +23,8 @@ import org.apache.seatunnel.api.table.type.SeaTunnelRow; import org.apache.seatunnel.api.table.type.SeaTunnelRowType; import org.apache.seatunnel.common.exception.CommonError; +import org.apache.seatunnel.common.utils.DateTimeUtils; +import org.apache.seatunnel.common.utils.DateUtils; import lombok.AllArgsConstructor; import software.amazon.awssdk.core.SdkBytes; @@ -34,6 +36,11 @@ import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; +import java.time.temporal.ChronoField; +import java.time.temporal.TemporalAccessor; +import java.time.temporal.TemporalQueries; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -44,6 +51,14 @@ public class DefaultSeaTunnelRowDeserializer implements SeaTunnelRowDeserializer private final SeaTunnelRowType typeInfo; + public static final DateTimeFormatter TIME_FORMAT = + new DateTimeFormatterBuilder() + .appendPattern("HH:mm:ss") + .appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, true) + .toFormatter(); + + public Map fieldFormatterMap = new HashMap<>(); + @Override public SeaTunnelRow deserialize(Map item) { SeaTunnelDataType[] seaTunnelDataTypes = typeInfo.getFieldTypes(); @@ -63,7 +78,9 @@ private List convertRow( } private Object convert( - String field, SeaTunnelDataType seaTunnelDataType, AttributeValue attributeValue) { + String fieldName, + SeaTunnelDataType seaTunnelDataType, + AttributeValue attributeValue) { if (attributeValue.type().equals(AttributeValue.Type.NUL)) { return null; } @@ -90,11 +107,26 @@ private Object convert( case STRING: return attributeValue.s(); case TIME: - return LocalTime.parse(attributeValue.s()); + return TIME_FORMAT.parse(attributeValue.s()); case DATE: - return LocalDate.parse(attributeValue.s()); + DateTimeFormatter dateFormatter = fieldFormatterMap.get(fieldName); + if (dateFormatter == null) { + dateFormatter = DateUtils.matchDateFormatter(attributeValue.s()); + fieldFormatterMap.put(fieldName, dateFormatter); + } + + return dateFormatter.parse(attributeValue.s()).query(TemporalQueries.localDate()); case TIMESTAMP: - return LocalDateTime.parse(attributeValue.s()); + DateTimeFormatter dateTimeFormatter = fieldFormatterMap.get(fieldName); + if (dateTimeFormatter == null) { + dateTimeFormatter = DateTimeUtils.matchDateTimeFormatter(attributeValue.s()); + fieldFormatterMap.put(fieldName, dateTimeFormatter); + } + + TemporalAccessor parsedTimestamp = dateTimeFormatter.parse(attributeValue.s()); + LocalTime localTime = parsedTimestamp.query(TemporalQueries.localTime()); + LocalDate localDate = parsedTimestamp.query(TemporalQueries.localDate()); + return LocalDateTime.of(localDate, localTime); case BYTES: return attributeValue.b().asByteArray(); case MAP: @@ -106,7 +138,7 @@ private Object convert( seatunnelMap.put( s, convert( - field, + fieldName, ((MapType) seaTunnelDataType).getValueType(), attributeValueInfo)); }); @@ -126,7 +158,7 @@ private Object convert( array, index, convert( - field, + fieldName, ((ArrayType) seaTunnelDataType).getElementType(), datas.get(index))); } @@ -149,7 +181,7 @@ private Object convert( return array; default: throw CommonError.convertToSeaTunnelTypeError( - "AmazonDynamodb", seaTunnelDataType.getSqlType().toString(), field); + "AmazonDynamodb", seaTunnelDataType.getSqlType().toString(), fieldName); } } } From b04c449f9c4ca53b270e909999849ac2a710b975 Mon Sep 17 00:00:00 2001 From: liugddx Date: Sun, 14 Jul 2024 16:49:37 +0800 Subject: [PATCH 16/59] 1 --- .../serialize/DefaultSeaTunnelRowDeserializer.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/serialize/DefaultSeaTunnelRowDeserializer.java b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/serialize/DefaultSeaTunnelRowDeserializer.java index 2cee95a61ee..a764a38d88f 100644 --- a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/serialize/DefaultSeaTunnelRowDeserializer.java +++ b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/serialize/DefaultSeaTunnelRowDeserializer.java @@ -26,7 +26,7 @@ import org.apache.seatunnel.common.utils.DateTimeUtils; import org.apache.seatunnel.common.utils.DateUtils; -import lombok.AllArgsConstructor; +import lombok.RequiredArgsConstructor; import software.amazon.awssdk.core.SdkBytes; import software.amazon.awssdk.services.dynamodb.model.AttributeValue; @@ -46,12 +46,12 @@ import java.util.List; import java.util.Map; -@AllArgsConstructor +@RequiredArgsConstructor public class DefaultSeaTunnelRowDeserializer implements SeaTunnelRowDeserializer { private final SeaTunnelRowType typeInfo; - public static final DateTimeFormatter TIME_FORMAT = + public static DateTimeFormatter TIME_FORMAT = new DateTimeFormatterBuilder() .appendPattern("HH:mm:ss") .appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, true) From 93c681717dee7daba02d24d20d88b9af11be1423 Mon Sep 17 00:00:00 2001 From: liugddx Date: Sun, 14 Jul 2024 19:38:55 +0800 Subject: [PATCH 17/59] 1 --- pom.xml | 3 --- 1 file changed, 3 deletions(-) diff --git a/pom.xml b/pom.xml index c818d2926e3..3fa34cd1d20 100644 --- a/pom.xml +++ b/pom.xml @@ -567,9 +567,6 @@ ${project.build.directory}/jacoco.exec - - **/*IT.java - org.apache.hadoop:hadoop-aliyun:jar diff --git a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/joblog/JobLogIT.java b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/joblog/JobLogIT.java index 18d08b7506e..d385571299b 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/joblog/JobLogIT.java +++ b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/joblog/JobLogIT.java @@ -66,7 +66,7 @@ public class JobLogIT extends SeaTunnelContainer { private static final Path BIN_PATH = Paths.get(SEATUNNEL_HOME, "bin", SERVER_SHELL); private static final Path CONFIG_PATH = Paths.get(SEATUNNEL_HOME, "config"); private static final Path HADOOP_JAR_PATH = - Paths.get(SEATUNNEL_HOME, "lib/seatunnel-hadoop3-3.1.4-uber.jar"); + Paths.get(SEATUNNEL_HOME, "lib/seatunnel-hadoop3-3.3.4-uber.jar"); private GenericContainer secondServer; private final Network NETWORK = Network.newNetwork(); @@ -310,7 +310,7 @@ private GenericContainer createServer(String networkAlias) server.withCopyFileToContainer( MountableFile.forHostPath( PROJECT_ROOT_PATH - + "/seatunnel-shade/seatunnel-hadoop3-3.1.4-uber/target/seatunnel-hadoop3-3.1.4-uber.jar"), + + "/seatunnel-shade/seatunnel-hadoop3-3.3.4-uber/target/seatunnel-hadoop3-3.3.4-uber.jar"), HADOOP_JAR_PATH.toString()); server.withCopyFileToContainer( MountableFile.forHostPath( diff --git a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/telemetry/MasterWorkerClusterSeaTunnelWithTelemetryIT.java b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/telemetry/MasterWorkerClusterSeaTunnelWithTelemetryIT.java index 2eca7f5e84b..d0c28519f24 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/telemetry/MasterWorkerClusterSeaTunnelWithTelemetryIT.java +++ b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/telemetry/MasterWorkerClusterSeaTunnelWithTelemetryIT.java @@ -67,7 +67,7 @@ public class MasterWorkerClusterSeaTunnelWithTelemetryIT extends SeaTunnelContai private static final Path binPath = Paths.get(SEATUNNEL_HOME, "bin", SERVER_SHELL); private static final Path config = Paths.get(SEATUNNEL_HOME, "config"); private static final Path hadoopJar = - Paths.get(SEATUNNEL_HOME, "lib/seatunnel-hadoop3-3.1.4-uber.jar"); + Paths.get(SEATUNNEL_HOME, "lib/seatunnel-hadoop3-3.3.4-uber.jar"); @Test public void testSubmitJobs() throws InterruptedException { @@ -695,7 +695,7 @@ private GenericContainer createServer(String networkAlias, String role) server.withCopyFileToContainer( MountableFile.forHostPath( PROJECT_ROOT_PATH - + "/seatunnel-shade/seatunnel-hadoop3-3.1.4-uber/target/seatunnel-hadoop3-3.1.4-uber.jar"), + + "/seatunnel-shade/seatunnel-hadoop3-3.3.4-uber/target/seatunnel-hadoop3-3.3.4-uber.jar"), hadoopJar.toString()); server.start(); // execute extra commands From a19632b78c9c077c783a44e19361731e6cf07dd2 Mon Sep 17 00:00:00 2001 From: liugddx Date: Sun, 1 Dec 2024 21:06:19 +0800 Subject: [PATCH 53/59] 1 --- pom.xml | 2 +- seatunnel-connectors-v2/connector-hudi/pom.xml | 2 +- .../seatunnel-connector-v2-e2e/connector-file-s3-e2e/pom.xml | 2 +- .../imap-storage-plugins/imap-storage-file/pom.xml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index 75f598400f9..aeb9bff915c 100644 --- a/pom.xml +++ b/pom.xml @@ -160,7 +160,7 @@ 0.16.0 true - 3.1.4 + 3.3.4 diff --git a/seatunnel-connectors-v2/connector-hudi/pom.xml b/seatunnel-connectors-v2/connector-hudi/pom.xml index 1a11d34f47e..48e6462a755 100644 --- a/seatunnel-connectors-v2/connector-hudi/pom.xml +++ b/seatunnel-connectors-v2/connector-hudi/pom.xml @@ -35,7 +35,7 @@ 1.12.2 1.1.10.4 4.0.2 - 3.1.4 + 3.3.4 connector-hudi diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-s3-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-s3-e2e/pom.xml index 71d0366fe6b..af63e69e894 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-s3-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-s3-e2e/pom.xml @@ -25,7 +25,7 @@ connector-file-s3-e2e SeaTunnel : E2E : Connector V2 : File S3 - 3.1.4 + 3.3.4 diff --git a/seatunnel-engine/seatunnel-engine-storage/imap-storage-plugins/imap-storage-file/pom.xml b/seatunnel-engine/seatunnel-engine-storage/imap-storage-plugins/imap-storage-file/pom.xml index 26d224f3490..751890ce136 100644 --- a/seatunnel-engine/seatunnel-engine-storage/imap-storage-plugins/imap-storage-file/pom.xml +++ b/seatunnel-engine/seatunnel-engine-storage/imap-storage-plugins/imap-storage-file/pom.xml @@ -34,7 +34,7 @@ 3.0.0 2.4.7 - 3.1.4 + 3.3.4 4.1.60.Final From 68c5bbcdf7af7757d9d17f104a1912ab8a7162ff Mon Sep 17 00:00:00 2001 From: liugddx Date: Mon, 2 Dec 2024 22:15:56 +0800 Subject: [PATCH 54/59] 1 --- seatunnel-core/seatunnel-starter/pom.xml | 2 +- .../org/apache/seatunnel/e2e/connector/file/s3/S3FileIT.java | 2 +- .../e2e/connector/file/s3/S3FileWithMultipleTableIT.java | 2 +- .../java/org/apache/seatunnel/e2e/connector/hive/HiveIT.java | 2 +- .../apache/seatunnel/e2e/connector/hive/HiveKerberosIT.java | 4 ++-- .../e2e/connector/hudi/HudiSeatunnelS3MultiTableIT.java | 2 +- .../apache/seatunnel/e2e/connector/paimon/PaimonWithS3IT.java | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/seatunnel-core/seatunnel-starter/pom.xml b/seatunnel-core/seatunnel-starter/pom.xml index 88120743042..418b35ed0cb 100644 --- a/seatunnel-core/seatunnel-starter/pom.xml +++ b/seatunnel-core/seatunnel-starter/pom.xml @@ -28,7 +28,7 @@ SeaTunnel : Core : Starter - 3.1.4 + 3.3.4 diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-s3-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/s3/S3FileIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-s3-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/s3/S3FileIT.java index 3ef03aad700..8e051264e25 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-s3-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/s3/S3FileIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-s3-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/s3/S3FileIT.java @@ -44,7 +44,7 @@ public class S3FileIT extends TestSuiteBase { public static final String S3_SDK_DOWNLOAD = "https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/1.11.271/aws-java-sdk-bundle-1.11.271.jar"; public static final String HADOOP_S3_DOWNLOAD = - "https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.1.4/hadoop-aws-3.1.4.jar"; + "https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.3.4/hadoop-aws-3.3.4.jar"; @TestContainerExtension private final ContainerExtendedFactory extendedFactory = diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-s3-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/s3/S3FileWithMultipleTableIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-s3-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/s3/S3FileWithMultipleTableIT.java index 34fd443146c..5038ffc5495 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-s3-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/s3/S3FileWithMultipleTableIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-s3-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/s3/S3FileWithMultipleTableIT.java @@ -36,7 +36,7 @@ public class S3FileWithMultipleTableIT extends TestSuiteBase { public static final String S3_SDK_DOWNLOAD = "https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/1.11.271/aws-java-sdk-bundle-1.11.271.jar"; public static final String HADOOP_S3_DOWNLOAD = - "https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.1.4/hadoop-aws-3.1.4.jar"; + "https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.3.4/hadoop-aws-3.3.4.jar"; @TestContainerExtension private final ContainerExtendedFactory extendedFactory = diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hive-e2e/src/test/java/org/apache/seatunnel/e2e/connector/hive/HiveIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hive-e2e/src/test/java/org/apache/seatunnel/e2e/connector/hive/HiveIT.java index bfa83dfb3b9..aaec0da4587 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hive-e2e/src/test/java/org/apache/seatunnel/e2e/connector/hive/HiveIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hive-e2e/src/test/java/org/apache/seatunnel/e2e/connector/hive/HiveIT.java @@ -72,7 +72,7 @@ private String libFb303Url() { } private String hadoopAwsUrl() { - return "https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.1.4/hadoop-aws-3.1.4.jar"; + return "https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.3.4/hadoop-aws-3.3.4.jar"; } private String aliyunSdkOssUrl() { diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hive-e2e/src/test/java/org/apache/seatunnel/e2e/connector/hive/HiveKerberosIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hive-e2e/src/test/java/org/apache/seatunnel/e2e/connector/hive/HiveKerberosIT.java index c2fca452fa8..9724d270112 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hive-e2e/src/test/java/org/apache/seatunnel/e2e/connector/hive/HiveKerberosIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hive-e2e/src/test/java/org/apache/seatunnel/e2e/connector/hive/HiveKerberosIT.java @@ -87,7 +87,7 @@ private String libFb303Url() { } private String hadoopAwsUrl() { - return "https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.1.4/hadoop-aws-3.1.4.jar"; + return "https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.3.4/hadoop-aws-3.3.4.jar"; } private String aliyunSdkOssUrl() { @@ -99,7 +99,7 @@ private String jdomUrl() { } private String hadoopAliyunUrl() { - return "https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aliyun/3.1.4/hadoop-aliyun-3.1.4.jar"; + return "https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aliyun/3.3.4/hadoop-aliyun-3.3.4.jar"; } private String hadoopCosUrl() { diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/src/test/java/org/apache/seatunnel/e2e/connector/hudi/HudiSeatunnelS3MultiTableIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/src/test/java/org/apache/seatunnel/e2e/connector/hudi/HudiSeatunnelS3MultiTableIT.java index 237fd100d26..48e5e5d77dc 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/src/test/java/org/apache/seatunnel/e2e/connector/hudi/HudiSeatunnelS3MultiTableIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/src/test/java/org/apache/seatunnel/e2e/connector/hudi/HudiSeatunnelS3MultiTableIT.java @@ -72,7 +72,7 @@ public class HudiSeatunnelS3MultiTableIT extends SeaTunnelContainer { protected static final String AWS_SDK_DOWNLOAD = "https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/1.11.271/aws-java-sdk-bundle-1.11.271.jar"; protected static final String HADOOP_AWS_DOWNLOAD = - "https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.1.4/hadoop-aws-3.1.4.jar"; + "https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.3.4/hadoop-aws-3.3.4.jar"; @Override @BeforeAll diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-paimon-e2e/src/test/java/org/apache/seatunnel/e2e/connector/paimon/PaimonWithS3IT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-paimon-e2e/src/test/java/org/apache/seatunnel/e2e/connector/paimon/PaimonWithS3IT.java index 2df1a5e49b2..a1251c07de8 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-paimon-e2e/src/test/java/org/apache/seatunnel/e2e/connector/paimon/PaimonWithS3IT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-paimon-e2e/src/test/java/org/apache/seatunnel/e2e/connector/paimon/PaimonWithS3IT.java @@ -52,7 +52,7 @@ public class PaimonWithS3IT extends SeaTunnelContainer { protected static final String AWS_SDK_DOWNLOAD = "https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/1.11.271/aws-java-sdk-bundle-1.11.271.jar"; protected static final String HADOOP_AWS_DOWNLOAD = - "https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.1.4/hadoop-aws-3.1.4.jar"; + "https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.3.4/hadoop-aws-3.3.4.jar"; @Override @BeforeAll From 74305b83d32c1c2dd48ffa9e52170f47d1c54bc1 Mon Sep 17 00:00:00 2001 From: liugddx Date: Tue, 3 Dec 2024 10:18:14 +0800 Subject: [PATCH 55/59] 1 --- tools/dependencies/known-dependencies.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/dependencies/known-dependencies.txt b/tools/dependencies/known-dependencies.txt index 6e29c914133..a3558177882 100755 --- a/tools/dependencies/known-dependencies.txt +++ b/tools/dependencies/known-dependencies.txt @@ -73,7 +73,6 @@ jetty-io-9.4.56.v20240826.jar jetty-security-9.4.56.v20240826.jar jetty-server-9.4.56.v20240826.jar jetty-servlet-9.4.56.v20240826.jar -jetty-util-9.4.20.v20190813.jar jetty-util-9.4.56.v20240826.jar jetty-util-ajax-9.4.56.v20240826.jar javax.servlet-api-3.1.0.jar From a18d38179d6c9de3fafe71867a200301735c96c0 Mon Sep 17 00:00:00 2001 From: liugddx Date: Tue, 3 Dec 2024 19:47:27 +0800 Subject: [PATCH 56/59] 1 --- tools/dependencies/known-dependencies.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/dependencies/known-dependencies.txt b/tools/dependencies/known-dependencies.txt index a3558177882..3b2d5c80fbf 100755 --- a/tools/dependencies/known-dependencies.txt +++ b/tools/dependencies/known-dependencies.txt @@ -74,6 +74,7 @@ jetty-security-9.4.56.v20240826.jar jetty-server-9.4.56.v20240826.jar jetty-servlet-9.4.56.v20240826.jar jetty-util-9.4.56.v20240826.jar +jetty-util-9.4.43.v20210629.jar jetty-util-ajax-9.4.56.v20240826.jar javax.servlet-api-3.1.0.jar seatunnel-jetty9-9.4.56-2.3.9-SNAPSHOT-optional.jar From ff059117e625d530a3b94747001ab3e24d50c2d6 Mon Sep 17 00:00:00 2001 From: liugddx Date: Wed, 4 Dec 2024 19:20:02 +0800 Subject: [PATCH 57/59] 1 --- .../org/apache/seatunnel/e2e/connector/tidb/TiDBCDCIT.java | 2 ++ .../apache/seatunnel/connectors/seatunnel/jdbc/JdbcIrisIT.java | 3 +++ 2 files changed, 5 insertions(+) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-tidb-e2e/src/test/java/org/apache/seatunnel/e2e/connector/tidb/TiDBCDCIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-tidb-e2e/src/test/java/org/apache/seatunnel/e2e/connector/tidb/TiDBCDCIT.java index 78c7077940c..ecd3fe61a86 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-tidb-e2e/src/test/java/org/apache/seatunnel/e2e/connector/tidb/TiDBCDCIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-tidb-e2e/src/test/java/org/apache/seatunnel/e2e/connector/tidb/TiDBCDCIT.java @@ -30,6 +30,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.TestTemplate; +import org.junit.jupiter.api.condition.DisabledOnJre; import org.testcontainers.containers.Container; import lombok.extern.slf4j.Slf4j; @@ -50,6 +51,7 @@ value = {}, type = {EngineType.SPARK}, disabledReason = "Currently SPARK do not support cdc") +@DisabledOnJre(JRE.JAVA_17) public class TiDBCDCIT extends TiDBTestBase implements TestResource { private static final String TIDB_DATABASE = "tidb_cdc"; diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-7/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcIrisIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-7/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcIrisIT.java index 28c813a54c7..403979fa130 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-7/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcIrisIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-7/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcIrisIT.java @@ -38,6 +38,8 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestTemplate; +import org.junit.jupiter.api.condition.DisabledOnJre; +import org.junit.jupiter.api.condition.JRE; import org.testcontainers.containers.Container; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.output.Slf4jLogConsumer; @@ -67,6 +69,7 @@ import java.util.stream.Collectors; @Slf4j +@DisabledOnJre(JRE.JAVA_17) public class JdbcIrisIT extends AbstractJdbcIT { private static final String IRIS_IMAGE = "intersystems/iris-community:2023.1"; private static final String IRIS_NETWORK_ALIASES = "e2e_irisDb"; From d5508fb8d1eace140502302d515999f120d73524 Mon Sep 17 00:00:00 2001 From: liugddx Date: Thu, 5 Dec 2024 10:29:40 +0800 Subject: [PATCH 58/59] 1 --- .../java/org/apache/seatunnel/e2e/connector/tidb/TiDBCDCIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-tidb-e2e/src/test/java/org/apache/seatunnel/e2e/connector/tidb/TiDBCDCIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-tidb-e2e/src/test/java/org/apache/seatunnel/e2e/connector/tidb/TiDBCDCIT.java index ecd3fe61a86..aa7300152f7 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-tidb-e2e/src/test/java/org/apache/seatunnel/e2e/connector/tidb/TiDBCDCIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-tidb-e2e/src/test/java/org/apache/seatunnel/e2e/connector/tidb/TiDBCDCIT.java @@ -31,6 +31,7 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.TestTemplate; import org.junit.jupiter.api.condition.DisabledOnJre; +import org.junit.jupiter.api.condition.JRE; import org.testcontainers.containers.Container; import lombok.extern.slf4j.Slf4j; From c213e29e4fd53f714e6b6d6f6a3a0dee7d24ae54 Mon Sep 17 00:00:00 2001 From: liugddx Date: Sun, 8 Dec 2024 12:06:25 +0800 Subject: [PATCH 59/59] 1 --- .../starter/seatunnel/command/ServerExecuteCommandTest.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/seatunnel-core/seatunnel-starter/src/test/java/org/apache/seatunnel/core/starter/seatunnel/command/ServerExecuteCommandTest.java b/seatunnel-core/seatunnel-starter/src/test/java/org/apache/seatunnel/core/starter/seatunnel/command/ServerExecuteCommandTest.java index 0962ed5ada2..b386066711e 100644 --- a/seatunnel-core/seatunnel-starter/src/test/java/org/apache/seatunnel/core/starter/seatunnel/command/ServerExecuteCommandTest.java +++ b/seatunnel-core/seatunnel-starter/src/test/java/org/apache/seatunnel/core/starter/seatunnel/command/ServerExecuteCommandTest.java @@ -25,7 +25,9 @@ public class ServerExecuteCommandTest { @Test - @DisabledOnJre(value = JRE.JAVA_11, disabledReason = "the test case only works on Java 8") + @DisabledOnJre( + value = {JRE.JAVA_11, JRE.JAVA_17}, + disabledReason = "the test case only works on Java 8") public void testJavaVersionCheck() { String realVersion = System.getProperty("java.version"); System.setProperty("java.version", "1.8.0_191");