diff --git a/bin/install-plugin.cmd b/bin/install-plugin.cmd index 2fe2a340f9a..50dd542f18c 100644 --- a/bin/install-plugin.cmd +++ b/bin/install-plugin.cmd @@ -22,8 +22,8 @@ REM Get seatunnel home set "SEATUNNEL_HOME=%~dp0..\" echo Set SEATUNNEL_HOME to [%SEATUNNEL_HOME%] -REM Connector default version is 2.3.9, you can also choose a custom version. eg: 2.3.9: install-plugin.bat 2.3.9 -set "version=2.3.9" +REM Connector default version is 2.3.10, you can also choose a custom version. eg: 2.3.10: install-plugin.bat 2.3.10 +set "version=2.3.10" if not "%~1"=="" set "version=%~1" REM Create the lib directory diff --git a/bin/install-plugin.sh b/bin/install-plugin.sh index 51afda5ad8a..db253de4674 100755 --- a/bin/install-plugin.sh +++ b/bin/install-plugin.sh @@ -23,8 +23,8 @@ # get seatunnel home SEATUNNEL_HOME=$(cd $(dirname $0);cd ../;pwd) -# connector default version is 2.3.9, you can also choose a custom version. eg: 2.3.9: sh install-plugin.sh 2.3.9 -version=2.3.9 +# connector default version is 2.3.10, you can also choose a custom version. eg: 2.3.10: sh install-plugin.sh 2.3.10 +version=2.3.10 if [ -n "$1" ]; then version="$1" diff --git a/deploy/kubernetes/seatunnel/Chart.yaml b/deploy/kubernetes/seatunnel/Chart.yaml index dc843bef76f..d5b1fffe592 100644 --- a/deploy/kubernetes/seatunnel/Chart.yaml +++ b/deploy/kubernetes/seatunnel/Chart.yaml @@ -35,8 +35,8 @@ keywords: type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 2.3.9 +version: 2.3.10 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 2.3.9 \ No newline at end of file +appVersion: 2.3.10 \ No newline at end of file diff --git a/docs/en/connector-v2/source/MySQL-CDC.md b/docs/en/connector-v2/source/MySQL-CDC.md index 42d3db09c91..9a95b3e566f 100644 --- a/docs/en/connector-v2/source/MySQL-CDC.md +++ b/docs/en/connector-v2/source/MySQL-CDC.md @@ -78,10 +78,9 @@ mysql> show variables where variable_name in ('log_bin', 'binlog_format', 'binlo | gtid_mode | ON | | log_bin | ON | +--------------------------+----------------+ -5 rows in set (0.00 sec) ``` -2. If inconsistent with the above results, configure your MySQL server configuration file(`$MYSQL_HOME/mysql.cnf`) with the following properties, which are described in the table below: +2. If the value of `log_bin` is not `on`, configure your MySQL server configuration file(`$MYSQL_HOME/mysql.cnf`) with the following properties, which are described in the table below: ``` # Enable binary replication log and set the prefix, expiration, and log format. @@ -95,8 +94,8 @@ binlog_format = row # mysql 5.6+ requires binlog_row_image to be set to FULL binlog_row_image = FULL -# enable gtid mode -# mysql 5.6+ requires gtid_mode to be set to ON +# optional enable gtid mode +# mysql 5.6+ requires gtid_mode to be set to ON, but not required by mysql 8.0+ gtid_mode = on enforce_gtid_consistency = on ``` @@ -119,7 +118,6 @@ mysql> show variables where variable_name in ('log_bin', 'binlog_format', 'binlo | binlog_format | ROW | | log_bin | ON | +--------------------------+----------------+ -5 rows in set (0.00 sec) ``` MySQL 5.6+: @@ -135,8 +133,22 @@ mysql> show variables where variable_name in ('log_bin', 'binlog_format', 'binlo | gtid_mode | ON | | log_bin | ON | +--------------------------+----------------+ -5 rows in set (0.00 sec) ``` +MySQL 8.0+: +```sql +show variables where variable_name in ('log_bin', 'binlog_format', 'binlog_row_image', 'gtid_mode', 'enforce_gtid_consistency') ++--------------------------+----------------+ +| Variable_name | Value | ++--------------------------+----------------+ +| binlog_format | ROW | +| binlog_row_image | FULL | +| enforce_gtid_consistency | OFF | +| gtid_mode | OFF | +| log_bin | ON | ++--------------------------+----------------+ + +``` + ### Notes @@ -169,38 +181,38 @@ When an initial consistent snapshot is made for large databases, your establishe ## Source Options -| Name | Type | Required | Default | Description | -|------------------------------------------------|----------|----------|---------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| base-url | String | Yes | - | The URL of the JDBC connection. Refer to a case: `jdbc:mysql://localhost:3306:3306/test`. | -| username | String | Yes | - | Name of the database to use when connecting to the database server. | -| password | String | Yes | - | Password to use when connecting to the database server. | -| database-names | List | No | - | Database name of the database to monitor. | -| database-pattern | String | No | .* | The database names RegEx of the database to capture, for example: `database_prefix.*`. | -| table-names | List | Yes | - | Table name of the database to monitor. The table name needs to include the database name, for example: `database_name.table_name` | -| table-pattern | String | Yes | - | The table names RegEx of the database to capture. The table name needs to include the database name, for example: `database.*\\.table_.*` | -| table-names-config | List | No | - | Table config list. for example: [{"table": "db1.schema1.table1","primaryKeys": ["key1"],"snapshotSplitColumn": "key2"}] | -| startup.mode | Enum | No | INITIAL | Optional startup mode for MySQL CDC consumer, valid enumerations are `initial`, `earliest`, `latest` and `specific`.
`initial`: Synchronize historical data at startup, and then synchronize incremental data.
`earliest`: Startup from the earliest offset possible.
`latest`: Startup from the latest offset.
`specific`: Startup from user-supplied specific offsets. | -| startup.specific-offset.file | String | No | - | Start from the specified binlog file name. **Note, This option is required when the `startup.mode` option used `specific`.** | -| startup.specific-offset.pos | Long | No | - | Start from the specified binlog file position. **Note, This option is required when the `startup.mode` option used `specific`.** | -| stop.mode | Enum | No | NEVER | Optional stop mode for MySQL CDC consumer, valid enumerations are `never`, `latest` or `specific`.
`never`: Real-time job don't stop the source.
`latest`: Stop from the latest offset.
`specific`: Stop from user-supplied specific offset. | -| stop.specific-offset.file | String | No | - | Stop from the specified binlog file name. **Note, This option is required when the `stop.mode` option used `specific`.** | -| stop.specific-offset.pos | Long | No | - | Stop from the specified binlog file position. **Note, This option is required when the `stop.mode` option used `specific`.** | -| snapshot.split.size | Integer | No | 8096 | The split size (number of rows) of table snapshot, captured tables are split into multiple splits when read the snapshot of table. | -| snapshot.fetch.size | Integer | No | 1024 | The maximum fetch size for per poll when read table snapshot. | -| server-id | String | No | - | A numeric ID or a numeric ID range of this database client, The numeric ID syntax is like `5400`, the numeric ID range syntax is like '5400-5408'.
Every ID must be unique across all currently-running database processes in the MySQL cluster. This connector joins the
MySQL cluster as another server (with this unique ID) so it can read the binlog.
By default, a random number is generated between 6500 and 2,148,492,146, though we recommend setting an explicit value. | -| server-time-zone | String | No | UTC | The session time zone in database server. If not set, then ZoneId.systemDefault() is used to determine the server time zone. | -| connect.timeout.ms | Duration | No | 30000 | The maximum time that the connector should wait after trying to connect to the database server before timing out. | -| connect.max-retries | Integer | No | 3 | The max retry times that the connector should retry to build database server connection. | -| connection.pool.size | Integer | No | 20 | The jdbc connection pool size. | +| Name | Type | Required | Default | Description | +|------------------------------------------------|----------|----------|---------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| base-url | String | Yes | - | The URL of the JDBC connection. Refer to a case: `jdbc:mysql://localhost:3306/test`. | +| username | String | Yes | - | Name of the database to use when connecting to the database server. | +| password | String | Yes | - | Password to use when connecting to the database server. | +| database-names | List | No | - | Database name of the database to monitor. | +| database-pattern | String | No | .* | The database names RegEx of the database to capture, for example: `database_prefix.*`. | +| table-names | List | Yes | - | Table name of the database to monitor. The table name needs to include the database name, for example: `database_name.table_name` | +| table-pattern | String | Yes | - | The table names RegEx of the database to capture. The table name needs to include the database name, for example: `database.*\\.table_.*` | +| table-names-config | List | No | - | Table config list. for example: [{"table": "db1.schema1.table1","primaryKeys": ["key1"],"snapshotSplitColumn": "key2"}] | +| startup.mode | Enum | No | INITIAL | Optional startup mode for MySQL CDC consumer, valid enumerations are `initial`, `earliest`, `latest` and `specific`.
`initial`: Synchronize historical data at startup, and then synchronize incremental data.
`earliest`: Startup from the earliest offset possible.
`latest`: Startup from the latest offset.
`specific`: Startup from user-supplied specific offsets. | +| startup.specific-offset.file | String | No | - | Start from the specified binlog file name. **Note, This option is required when the `startup.mode` option used `specific`.** | +| startup.specific-offset.pos | Long | No | - | Start from the specified binlog file position. **Note, This option is required when the `startup.mode` option used `specific`.** | +| stop.mode | Enum | No | NEVER | Optional stop mode for MySQL CDC consumer, valid enumerations are `never`, `latest` or `specific`.
`never`: Real-time job don't stop the source.
`latest`: Stop from the latest offset.
`specific`: Stop from user-supplied specific offset. | +| stop.specific-offset.file | String | No | - | Stop from the specified binlog file name. **Note, This option is required when the `stop.mode` option used `specific`.** | +| stop.specific-offset.pos | Long | No | - | Stop from the specified binlog file position. **Note, This option is required when the `stop.mode` option used `specific`.** | +| snapshot.split.size | Integer | No | 8096 | The split size (number of rows) of table snapshot, captured tables are split into multiple splits when read the snapshot of table. | +| snapshot.fetch.size | Integer | No | 1024 | The maximum fetch size for per poll when read table snapshot. | +| server-id | String | No | - | A numeric ID or a numeric ID range of this database client, The numeric ID syntax is like `5400`, the numeric ID range syntax is like '5400-5408'.
Every ID must be unique across all currently-running database processes in the MySQL cluster. This connector joins the
MySQL cluster as another server (with this unique ID) so it can read the binlog.
By default, a random number is generated between 6500 and 2,148,492,146, though we recommend setting an explicit value. | +| server-time-zone | String | No | UTC | The session time zone in database server. If not set, then ZoneId.systemDefault() is used to determine the server time zone. | +| connect.timeout.ms | Duration | No | 30000 | The maximum time that the connector should wait after trying to connect to the database server before timing out. | +| connect.max-retries | Integer | No | 3 | The max retry times that the connector should retry to build database server connection. | +| connection.pool.size | Integer | No | 20 | The jdbc connection pool size. | | chunk-key.even-distribution.factor.upper-bound | Double | No | 100 | The upper bound of the chunk key distribution factor. This factor is used to determine whether the table data is evenly distributed. If the distribution factor is calculated to be less than or equal to this upper bound (i.e., (MAX(id) - MIN(id) + 1) / row count), the table chunks would be optimized for even distribution. Otherwise, if the distribution factor is greater, the table will be considered as unevenly distributed and the sampling-based sharding strategy will be used if the estimated shard count exceeds the value specified by `sample-sharding.threshold`. The default value is 100.0. | -| chunk-key.even-distribution.factor.lower-bound | Double | No | 0.05 | The lower bound of the chunk key distribution factor. This factor is used to determine whether the table data is evenly distributed. If the distribution factor is calculated to be greater than or equal to this lower bound (i.e., (MAX(id) - MIN(id) + 1) / row count), the table chunks would be optimized for even distribution. Otherwise, if the distribution factor is less, the table will be considered as unevenly distributed and the sampling-based sharding strategy will be used if the estimated shard count exceeds the value specified by `sample-sharding.threshold`. The default value is 0.05. | -| sample-sharding.threshold | Integer | No | 1000 | This configuration specifies the threshold of estimated shard count to trigger the sample sharding strategy. When the distribution factor is outside the bounds specified by `chunk-key.even-distribution.factor.upper-bound` and `chunk-key.even-distribution.factor.lower-bound`, and the estimated shard count (calculated as approximate row count / chunk size) exceeds this threshold, the sample sharding strategy will be used. This can help to handle large datasets more efficiently. The default value is 1000 shards. | -| inverse-sampling.rate | Integer | No | 1000 | The inverse of the sampling rate used in the sample sharding strategy. For example, if this value is set to 1000, it means a 1/1000 sampling rate is applied during the sampling process. This option provides flexibility in controlling the granularity of the sampling, thus affecting the final number of shards. It's especially useful when dealing with very large datasets where a lower sampling rate is preferred. The default value is 1000. | -| exactly_once | Boolean | No | false | Enable exactly once semantic. | -| format | Enum | No | DEFAULT | Optional output format for MySQL CDC, valid enumerations are `DEFAULT`、`COMPATIBLE_DEBEZIUM_JSON`. | -| schema-changes.enabled | Boolean | No | false | Schema evolution is disabled by default. Now we only support `add column`、`drop column`、`rename column` and `modify column`. | -| debezium | Config | No | - | Pass-through [Debezium's properties](https://github.com/debezium/debezium/blob/v1.9.8.Final/documentation/modules/ROOT/pages/connectors/mysql.adoc#connector-properties) to Debezium Embedded Engine which is used to capture data changes from MySQL server. | -| common-options | | no | - | Source plugin common parameters, please refer to [Source Common Options](../source-common-options.md) for details | +| chunk-key.even-distribution.factor.lower-bound | Double | No | 0.05 | The lower bound of the chunk key distribution factor. This factor is used to determine whether the table data is evenly distributed. If the distribution factor is calculated to be greater than or equal to this lower bound (i.e., (MAX(id) - MIN(id) + 1) / row count), the table chunks would be optimized for even distribution. Otherwise, if the distribution factor is less, the table will be considered as unevenly distributed and the sampling-based sharding strategy will be used if the estimated shard count exceeds the value specified by `sample-sharding.threshold`. The default value is 0.05. | +| sample-sharding.threshold | Integer | No | 1000 | This configuration specifies the threshold of estimated shard count to trigger the sample sharding strategy. When the distribution factor is outside the bounds specified by `chunk-key.even-distribution.factor.upper-bound` and `chunk-key.even-distribution.factor.lower-bound`, and the estimated shard count (calculated as approximate row count / chunk size) exceeds this threshold, the sample sharding strategy will be used. This can help to handle large datasets more efficiently. The default value is 1000 shards. | +| inverse-sampling.rate | Integer | No | 1000 | The inverse of the sampling rate used in the sample sharding strategy. For example, if this value is set to 1000, it means a 1/1000 sampling rate is applied during the sampling process. This option provides flexibility in controlling the granularity of the sampling, thus affecting the final number of shards. It's especially useful when dealing with very large datasets where a lower sampling rate is preferred. The default value is 1000. | +| exactly_once | Boolean | No | false | Enable exactly once semantic. | +| format | Enum | No | DEFAULT | Optional output format for MySQL CDC, valid enumerations are `DEFAULT`、`COMPATIBLE_DEBEZIUM_JSON`. | +| schema-changes.enabled | Boolean | No | false | Schema evolution is disabled by default. Now we only support `add column`、`drop column`、`rename column` and `modify column`. | +| debezium | Config | No | - | Pass-through [Debezium's properties](https://github.com/debezium/debezium/blob/v1.9.8.Final/documentation/modules/ROOT/pages/connectors/mysql.adoc#connector-properties) to Debezium Embedded Engine which is used to capture data changes from MySQL server. | +| common-options | | no | - | Source plugin common parameters, please refer to [Source Common Options](../source-common-options.md) for details | ## Task Example diff --git a/docs/en/faq.md b/docs/en/faq.md index e85b2a2a227..23edc7b0703 100644 --- a/docs/en/faq.md +++ b/docs/en/faq.md @@ -38,7 +38,7 @@ Before starting an integration task, you can select different handling schemes f - **`CREATE_SCHEMA_WHEN_NOT_EXIST`**: Creates the table if it does not exist; skips creation if the table already exists. - **`ERROR_WHEN_SCHEMA_NOT_EXIST`**: Throws an error if the table does not exist. - **`IGNORE`**: Ignores table handling. - Many connectors currently support automatic table creation. Refer to the specific connector documentation, such as [Jdbc sink](https://seatunnel.apache.org/docs/2.3.8/connector-v2/sink/Jdbc#schema_save_mode-enum), for more information. + Many connectors currently support automatic table creation. Refer to the specific connector documentation, such as [Jdbc sink](https://seatunnel.apache.org/docs/connector-v2/sink/Jdbc/#schema_save_mode-enum), for more information. ## Does SeaTunnel support handling existing data before starting a data integration task? Yes, you can specify different processing schemes for existing data on the target side before starting an integration task, controlled via the `data_save_mode` parameter. Available options include: diff --git a/docs/en/seatunnel-engine/download-seatunnel.md b/docs/en/seatunnel-engine/download-seatunnel.md index 12b169e482c..28ab24cbae3 100644 --- a/docs/en/seatunnel-engine/download-seatunnel.md +++ b/docs/en/seatunnel-engine/download-seatunnel.md @@ -20,7 +20,7 @@ Go to the [Seatunnel Download Page](https://seatunnel.apache.org/download) to do Or you can also download it through the terminal. ```shell -export version="2.3.9" +export version="2.3.10" wget "https://archive.apache.org/dist/seatunnel/${version}/apache-seatunnel-${version}-bin.tar.gz" tar -xzvf "apache-seatunnel-${version}-bin.tar.gz" ``` @@ -33,10 +33,10 @@ Starting from the 2.2.0-beta version, the binary package no longer provides the sh bin/install-plugin.sh ``` -If you need a specific connector version, taking 2.3.9 as an example, you need to execute the following command. +If you need a specific connector version, taking 2.3.10 as an example, you need to execute the following command. ```bash -sh bin/install-plugin.sh 2.3.9 +sh bin/install-plugin.sh 2.3.10 ``` Usually you don't need all the connector plugins, so you can specify the plugins you need through configuring `config/plugin_config`, for example, if you only need the `connector-console` plugin, then you can modify the plugin.properties configuration file as follows. diff --git a/docs/en/seatunnel-engine/rest-api-v1.md b/docs/en/seatunnel-engine/rest-api-v1.md index 8859faa32f1..f9b5f69aa7b 100644 --- a/docs/en/seatunnel-engine/rest-api-v1.md +++ b/docs/en/seatunnel-engine/rest-api-v1.md @@ -55,7 +55,7 @@ network: ```json { - "projectVersion":"2.3.5-SNAPSHOT", + "projectVersion":"2.3.10-SNAPSHOT", "gitCommitAbbrev":"DeadD0d0", "totalSlot":"0", "unassignedSlot":"0", diff --git a/docs/en/seatunnel-engine/rest-api-v2.md b/docs/en/seatunnel-engine/rest-api-v2.md index 01dc9479117..72eb9044431 100644 --- a/docs/en/seatunnel-engine/rest-api-v2.md +++ b/docs/en/seatunnel-engine/rest-api-v2.md @@ -54,7 +54,7 @@ seatunnel: ```json { - "projectVersion":"2.3.5-SNAPSHOT", + "projectVersion":"2.3.10-SNAPSHOT", "gitCommitAbbrev":"DeadD0d0", "totalSlot":"0", "unassignedSlot":"0", diff --git a/docs/en/start-v2/docker/docker.md b/docs/en/start-v2/docker/docker.md index fba6d2e4732..0a0d375f5e4 100644 --- a/docs/en/start-v2/docker/docker.md +++ b/docs/en/start-v2/docker/docker.md @@ -40,7 +40,7 @@ You can download the source code from the [download page](https://seatunnel.apac ```shell cd seatunnel # Use already sett maven profile -sh ./mvnw -B clean install -Dmaven.test.skip=true -Dmaven.javadoc.skip=true -Dlicense.skipAddThirdParty=true -D"docker.build.skip"=false -D"docker.verify.skip"=false -D"docker.push.skip"=true -D"docker.tag"=2.3.9 -Dmaven.deploy.skip -D"skip.spotless"=true --no-snapshot-updates -Pdocker,seatunnel +sh ./mvnw -B clean install -Dmaven.test.skip=true -Dmaven.javadoc.skip=true -Dlicense.skipAddThirdParty=true -D"docker.build.skip"=false -D"docker.verify.skip"=false -D"docker.push.skip"=true -D"docker.tag"=2.3.10 -Dmaven.deploy.skip -D"skip.spotless"=true --no-snapshot-updates -Pdocker,seatunnel # Check the docker image docker images | grep apache/seatunnel @@ -53,10 +53,10 @@ sh ./mvnw clean package -DskipTests -Dskip.spotless=true # Build docker image cd seatunnel-dist -docker build -f src/main/docker/Dockerfile --build-arg VERSION=2.3.9 -t apache/seatunnel:2.3.9 . +docker build -f src/main/docker/Dockerfile --build-arg VERSION=2.3.10 -t apache/seatunnel:2.3.10 . # If you build from dev branch, you should add SNAPSHOT suffix to the version -docker build -f src/main/docker/Dockerfile --build-arg VERSION=2.3.9-SNAPSHOT -t apache/seatunnel:2.3.9-SNAPSHOT . +docker build -f src/main/docker/Dockerfile --build-arg VERSION=2.3.10-SNAPSHOT -t apache/seatunnel:2.3.10-SNAPSHOT . # Check the docker image docker images | grep apache/seatunnel @@ -72,7 +72,7 @@ COPY ./target/apache-seatunnel-${VERSION}-bin.tar.gz /opt/ # Download From Internet # Please Note this file only include fake/console connector, You'll need to download the other connectors manually -# wget -P /opt https://dlcdn.apache.org/seatunnel/2.3.6/apache-seatunnel-${VERSION}-bin.tar.gz +# wget -P /opt https://dlcdn.apache.org/seatunnel/${VERSION}/apache-seatunnel-${VERSION}-bin.tar.gz RUN cd /opt && \ tar -zxvf apache-seatunnel-${VERSION}-bin.tar.gz && \ diff --git a/docs/en/start-v2/kubernetes/helm.md b/docs/en/start-v2/kubernetes/helm.md index f33b4af453b..39bd643476f 100644 --- a/docs/en/start-v2/kubernetes/helm.md +++ b/docs/en/start-v2/kubernetes/helm.md @@ -30,7 +30,7 @@ minikube start --kubernetes-version=v1.23.3 Install with default settings. ```bash # Choose the corresponding version yourself -export VERSION=2.3.9 +export VERSION=2.3.10 helm pull oci://registry-1.docker.io/apache/seatunnel-helm --version ${VERSION} tar -xvf seatunnel-helm-${VERSION}.tgz cd seatunnel-helm diff --git a/docs/en/start-v2/kubernetes/kubernetes.mdx b/docs/en/start-v2/kubernetes/kubernetes.mdx index be0548f3d95..3173755f422 100644 --- a/docs/en/start-v2/kubernetes/kubernetes.mdx +++ b/docs/en/start-v2/kubernetes/kubernetes.mdx @@ -44,7 +44,7 @@ To run the image with SeaTunnel, first create a `Dockerfile`: ```Dockerfile FROM flink:1.13 -ENV SEATUNNEL_VERSION="2.3.9" +ENV SEATUNNEL_VERSION="2.3.10" ENV SEATUNNEL_HOME="/opt/seatunnel" RUN wget https://dlcdn.apache.org/seatunnel/${SEATUNNEL_VERSION}/apache-seatunnel-${SEATUNNEL_VERSION}-bin.tar.gz @@ -56,13 +56,13 @@ RUN cd ${SEATUNNEL_HOME} && sh bin/install-plugin.sh ${SEATUNNEL_VERSION} Then run the following commands to build the image: ```bash -docker build -t seatunnel:2.3.9-flink-1.13 -f Dockerfile . +docker build -t seatunnel:2.3.10-flink-1.13 -f Dockerfile . ``` -Image `seatunnel:2.3.9-flink-1.13` needs to be present in the host (minikube) so that the deployment can take place. +Image `seatunnel:2.3.10-flink-1.13` needs to be present in the host (minikube) so that the deployment can take place. Load image to minikube via: ```bash -minikube image load seatunnel:2.3.9-flink-1.13 +minikube image load seatunnel:2.3.10-flink-1.13 ``` @@ -72,7 +72,7 @@ minikube image load seatunnel:2.3.9-flink-1.13 ```Dockerfile FROM openjdk:8 -ENV SEATUNNEL_VERSION="2.3.9" +ENV SEATUNNEL_VERSION="2.3.10" ENV SEATUNNEL_HOME="/opt/seatunnel" RUN wget https://dlcdn.apache.org/seatunnel/${SEATUNNEL_VERSION}/apache-seatunnel-${SEATUNNEL_VERSION}-bin.tar.gz @@ -84,13 +84,13 @@ RUN cd ${SEATUNNEL_HOME} && sh bin/install-plugin.sh ${SEATUNNEL_VERSION} Then run the following commands to build the image: ```bash -docker build -t seatunnel:2.3.9 -f Dockerfile . +docker build -t seatunnel:2.3.10 -f Dockerfile . ``` -Image `seatunnel:2.3.9` need to be present in the host (minikube) so that the deployment can take place. +Image `seatunnel:2.3.10` need to be present in the host (minikube) so that the deployment can take place. Load image to minikube via: ```bash -minikube image load seatunnel:2.3.9 +minikube image load seatunnel:2.3.10 ``` @@ -100,7 +100,7 @@ minikube image load seatunnel:2.3.9 ```Dockerfile FROM openjdk:8 -ENV SEATUNNEL_VERSION="2.3.9" +ENV SEATUNNEL_VERSION="2.3.10" ENV SEATUNNEL_HOME="/opt/seatunnel" RUN wget https://dlcdn.apache.org/seatunnel/${SEATUNNEL_VERSION}/apache-seatunnel-${SEATUNNEL_VERSION}-bin.tar.gz @@ -112,13 +112,13 @@ RUN cd ${SEATUNNEL_HOME} && sh bin/install-plugin.sh ${SEATUNNEL_VERSION} Then run the following commands to build the image: ```bash -docker build -t seatunnel:2.3.9 -f Dockerfile . +docker build -t seatunnel:2.3.10 -f Dockerfile . ``` -Image `seatunnel:2.3.9` needs to be present in the host (minikube) so that the deployment can take place. +Image `seatunnel:2.3.10` needs to be present in the host (minikube) so that the deployment can take place. Load image to minikube via: ```bash -minikube image load seatunnel:2.3.9 +minikube image load seatunnel:2.3.10 ``` @@ -191,7 +191,7 @@ none ]}> -In this guide we will use [seatunnel.streaming.conf](https://github.com/apache/seatunnel/blob/2.3.9-release/config/v2.streaming.conf.template): +In this guide we will use [seatunnel.streaming.conf](https://github.com/apache/seatunnel/blob/2.3.10-release/config/v2.streaming.conf.template): ```conf env { @@ -245,7 +245,7 @@ kind: FlinkDeployment metadata: name: seatunnel-flink-streaming-example spec: - image: seatunnel:2.3.9-flink-1.13 + image: seatunnel:2.3.10-flink-1.13 flinkVersion: v1_13 flinkConfiguration: taskmanager.numberOfTaskSlots: "2" @@ -291,7 +291,7 @@ kubectl apply -f seatunnel-flink.yaml -In this guide we will use [seatunnel.streaming.conf](https://github.com/apache/seatunnel/blob/2.3.9-release/config/v2.streaming.conf.template): +In this guide we will use [seatunnel.streaming.conf](https://github.com/apache/seatunnel/blob/2.3.10-release/config/v2.streaming.conf.template): ```conf env { @@ -334,7 +334,7 @@ metadata: spec: containers: - name: seatunnel - image: seatunnel:2.3.9 + image: seatunnel:2.3.10 command: ["/bin/sh","-c","/opt/seatunnel/bin/seatunnel.sh --config /data/seatunnel.streaming.conf -e local"] resources: limits: @@ -366,7 +366,7 @@ kubectl apply -f seatunnel.yaml -In this guide we will use [seatunnel.streaming.conf](https://github.com/apache/seatunnel/blob/2.3.9-release/config/v2.streaming.conf.template): +In this guide we will use [seatunnel.streaming.conf](https://github.com/apache/seatunnel/blob/2.3.10-release/config/v2.streaming.conf.template): ```conf env { @@ -524,7 +524,7 @@ spec: spec: containers: - name: seatunnel - image: seatunnel:2.3.9 + image: seatunnel:2.3.10 imagePullPolicy: IfNotPresent ports: - containerPort: 5801 diff --git a/docs/en/start-v2/locally/deployment.md b/docs/en/start-v2/locally/deployment.md index 4684871acb0..a7043a51e16 100644 --- a/docs/en/start-v2/locally/deployment.md +++ b/docs/en/start-v2/locally/deployment.md @@ -22,7 +22,7 @@ Visit the [SeaTunnel Download Page](https://seatunnel.apache.org/download) to do Or you can also download it through the terminal: ```shell -export version="2.3.9" +export version="2.3.10" wget "https://archive.apache.org/dist/seatunnel/${version}/apache-seatunnel-${version}-bin.tar.gz" tar -xzvf "apache-seatunnel-${version}-bin.tar.gz" ``` @@ -35,10 +35,10 @@ Starting from version 2.2.0-beta, the binary package no longer provides connecto sh bin/install-plugin.sh ``` -If you need a specific connector version, taking 2.3.9 as an example, you need to execute the following command: +If you need a specific connector version, taking 2.3.10 as an example, you need to execute the following command: ```bash -sh bin/install-plugin.sh 2.3.9 +sh bin/install-plugin.sh 2.3.10 ``` Typically, you do not need all the connector plugins. You can specify the required plugins by configuring `config/plugin_config`. For example, if you want the sample application to work properly, you will need the `connector-console` and `connector-fake` plugins. You can modify the `plugin_config` configuration file as follows: @@ -71,7 +71,7 @@ You can download the source code from the [download page](https://seatunnel.apac cd seatunnel sh ./mvnw clean install -DskipTests -Dskip.spotless=true # get the binary package -cp seatunnel-dist/target/apache-seatunnel-2.3.9-bin.tar.gz /The-Path-You-Want-To-Copy +cp seatunnel-dist/target/apache-seatunnel-2.3.10-bin.tar.gz /The-Path-You-Want-To-Copy cd /The-Path-You-Want-To-Copy tar -xzvf "apache-seatunnel-${version}-bin.tar.gz" diff --git a/docs/zh/seatunnel-engine/download-seatunnel.md b/docs/zh/seatunnel-engine/download-seatunnel.md index 8d06a2e4f78..b6e6f8e0cdf 100644 --- a/docs/zh/seatunnel-engine/download-seatunnel.md +++ b/docs/zh/seatunnel-engine/download-seatunnel.md @@ -20,7 +20,7 @@ import TabItem from '@theme/TabItem'; 或者您也可以通过终端下载 ```shell -export version="2.3.9" +export version="2.3.10" wget "https://archive.apache.org/dist/seatunnel/${version}/apache-seatunnel-${version}-bin.tar.gz" tar -xzvf "apache-seatunnel-${version}-bin.tar.gz" ``` @@ -30,13 +30,13 @@ tar -xzvf "apache-seatunnel-${version}-bin.tar.gz" 从2.2.0-beta版本开始，二进制包不再默认提供连接器依赖，因此在第一次使用它时，您需要执行以下命令来安装连接器：(当然，您也可以从 [Apache Maven Repository](https://repo.maven.apache.org/maven2/org/apache/seatunnel/) 手动下载连接器，然后将其移动至`connectors/seatunnel`目录下)。 ```bash -sh bin/install-plugin.sh 2.3.9 +sh bin/install-plugin.sh 2.3.10 ``` -如果您需要指定的连接器版本，以2.3.9为例，您需要执行如下命令 +如果您需要指定的连接器版本，以2.3.10为例，您需要执行如下命令 ```bash -sh bin/install-plugin.sh 2.3.9 +sh bin/install-plugin.sh 2.3.10 ``` 通常您并不需要所有的连接器插件，所以您可以通过配置`config/plugin_config`来指定您所需要的插件，例如，您只需要`connector-console`插件，那么您可以修改plugin.properties配置文件如下 diff --git a/docs/zh/seatunnel-engine/rest-api-v1.md b/docs/zh/seatunnel-engine/rest-api-v1.md index 15b0cf0545d..f23021cdb94 100644 --- a/docs/zh/seatunnel-engine/rest-api-v1.md +++ b/docs/zh/seatunnel-engine/rest-api-v1.md @@ -54,7 +54,7 @@ network: ```json { - "projectVersion":"2.3.5-SNAPSHOT", + "projectVersion":"2.3.10-SNAPSHOT", "gitCommitAbbrev":"DeadD0d0", "totalSlot":"0", "unassignedSlot":"0", diff --git a/docs/zh/seatunnel-engine/rest-api-v2.md b/docs/zh/seatunnel-engine/rest-api-v2.md index bf118fe1c15..ed3c6803c7a 100644 --- a/docs/zh/seatunnel-engine/rest-api-v2.md +++ b/docs/zh/seatunnel-engine/rest-api-v2.md @@ -52,7 +52,7 @@ seatunnel: ```json { - "projectVersion":"2.3.5-SNAPSHOT", + "projectVersion":"2.3.10-SNAPSHOT", "gitCommitAbbrev":"DeadD0d0", "totalSlot":"0", "unassignedSlot":"0", diff --git a/docs/zh/start-v2/docker/docker.md b/docs/zh/start-v2/docker/docker.md index 84e93a22167..87aabebeace 100644 --- a/docs/zh/start-v2/docker/docker.md +++ b/docs/zh/start-v2/docker/docker.md @@ -40,7 +40,7 @@ docker run --rm -it -v /tmp/job/:/config apache/seatunnel: ./bin/se ```shell cd seatunnel # Use already sett maven profile -mvn -B clean install -Dmaven.test.skip=true -Dmaven.javadoc.skip=true -Dlicense.skipAddThirdParty=true -D"docker.build.skip"=false -D"docker.verify.skip"=false -D"docker.push.skip"=true -D"docker.tag"=2.3.9 -Dmaven.deploy.skip -D"skip.spotless"=true --no-snapshot-updates -Pdocker,seatunnel +mvn -B clean install -Dmaven.test.skip=true -Dmaven.javadoc.skip=true -Dlicense.skipAddThirdParty=true -D"docker.build.skip"=false -D"docker.verify.skip"=false -D"docker.push.skip"=true -D"docker.tag"=2.3.10 -Dmaven.deploy.skip -D"skip.spotless"=true --no-snapshot-updates -Pdocker,seatunnel # Check the docker image docker images | grep apache/seatunnel @@ -53,10 +53,10 @@ mvn clean package -DskipTests -Dskip.spotless=true # Build docker image cd seatunnel-dist -docker build -f src/main/docker/Dockerfile --build-arg VERSION=2.3.9 -t apache/seatunnel:2.3.9 . +docker build -f src/main/docker/Dockerfile --build-arg VERSION=2.3.10 -t apache/seatunnel:2.3.10 . # If you build from dev branch, you should add SNAPSHOT suffix to the version -docker build -f src/main/docker/Dockerfile --build-arg VERSION=2.3.9-SNAPSHOT -t apache/seatunnel:2.3.9-SNAPSHOT . +docker build -f src/main/docker/Dockerfile --build-arg VERSION=2.3.10-SNAPSHOT -t apache/seatunnel:2.3.10-SNAPSHOT . # Check the docker image docker images | grep apache/seatunnel @@ -72,7 +72,7 @@ COPY ./target/apache-seatunnel-${VERSION}-bin.tar.gz /opt/ # Download From Internet # Please Note this file only include fake/console connector, You'll need to download the other connectors manually -# wget -P /opt https://dlcdn.apache.org/seatunnel/2.3.6/apache-seatunnel-${VERSION}-bin.tar.gz +# wget -P /opt https://dlcdn.apache.org/seatunnel/${VERSION}/apache-seatunnel-${VERSION}-bin.tar.gz RUN cd /opt && \ tar -zxvf apache-seatunnel-${VERSION}-bin.tar.gz && \ diff --git a/docs/zh/start-v2/kubernetes/helm.md b/docs/zh/start-v2/kubernetes/helm.md index e0d6c60aca9..05e8733d6c7 100644 --- a/docs/zh/start-v2/kubernetes/helm.md +++ b/docs/zh/start-v2/kubernetes/helm.md @@ -30,7 +30,7 @@ minikube start --kubernetes-version=v1.23.3 使用默认配置安装 ```bash # Choose the corresponding version yourself -export VERSION=2.3.9 +export VERSION=2.3.10 helm pull oci://registry-1.docker.io/apache/seatunnel-helm --version ${VERSION} tar -xvf seatunnel-helm-${VERSION}.tgz cd seatunnel-helm diff --git a/docs/zh/start-v2/locally/deployment.md b/docs/zh/start-v2/locally/deployment.md index 927f5476ece..bf259966e5b 100644 --- a/docs/zh/start-v2/locally/deployment.md +++ b/docs/zh/start-v2/locally/deployment.md @@ -22,7 +22,7 @@ import TabItem from '@theme/TabItem'; 或者您也可以通过终端下载： ```shell -export version="2.3.9" +export version="2.3.10" wget "https://archive.apache.org/dist/seatunnel/${version}/apache-seatunnel-${version}-bin.tar.gz" tar -xzvf "apache-seatunnel-${version}-bin.tar.gz" ``` @@ -35,10 +35,10 @@ tar -xzvf "apache-seatunnel-${version}-bin.tar.gz" sh bin/install-plugin.sh ``` -如果您需要指定的连接器版本，以2.3.9为例，您需要执行如下命令： +如果您需要指定的连接器版本，以2.3.10为例，您需要执行如下命令： ```bash -sh bin/install-plugin.sh 2.3.9 +sh bin/install-plugin.sh 2.3.10 ``` 通常情况下，你不需要所有的连接器插件。你可以通过配置`config/plugin_config`来指定所需的插件。例如，如果你想让示例应用程序正常工作，你将需要`connector-console`和`connector-fake`插件。你可以修改`plugin_config`配置文件，如下所示： @@ -71,7 +71,7 @@ connector-console cd seatunnel sh ./mvnw clean install -DskipTests -Dskip.spotless=true # 获取构建好的二进制包 -cp seatunnel-dist/target/apache-seatunnel-2.3.9-bin.tar.gz /The-Path-You-Want-To-Copy +cp seatunnel-dist/target/apache-seatunnel-2.3.10-bin.tar.gz /The-Path-You-Want-To-Copy cd /The-Path-You-Want-To-Copy tar -xzvf "apache-seatunnel-${version}-bin.tar.gz" diff --git a/pom.xml b/pom.xml index 44c7b35026d..e4a6f6c9eb7 100644 --- a/pom.xml +++ b/pom.xml @@ -57,7 +57,7 @@ - 2.3.9-SNAPSHOT + 2.3.10-SNAPSHOT 2.1.1 UTF-8 1.8 diff --git a/seatunnel-connectors-v2/connector-clickhouse/src/main/java/org/apache/seatunnel/connectors/seatunnel/clickhouse/util/ClickhouseCatalogUtil.java b/seatunnel-connectors-v2/connector-clickhouse/src/main/java/org/apache/seatunnel/connectors/seatunnel/clickhouse/util/ClickhouseCatalogUtil.java index bf4e02c3fb1..73dfb766dd8 100644 --- a/seatunnel-connectors-v2/connector-clickhouse/src/main/java/org/apache/seatunnel/connectors/seatunnel/clickhouse/util/ClickhouseCatalogUtil.java +++ b/seatunnel-connectors-v2/connector-clickhouse/src/main/java/org/apache/seatunnel/connectors/seatunnel/clickhouse/util/ClickhouseCatalogUtil.java @@ -38,7 +38,9 @@ public String columnToConnectorType(Column column) { ClickhouseTypeConverter.INSTANCE.reconvert(column).getColumnType(), StringUtils.isEmpty(column.getComment()) ? "" - : "COMMENT '" + column.getComment() + "'"); + : "COMMENT '" + + column.getComment().replace("'", "''").replace("\\", "\\\\") + + "'"); } public String getDropTableSql(TablePath tablePath, boolean ignoreIfNotExists) { diff --git a/seatunnel-connectors-v2/connector-clickhouse/src/test/java/org/apache/seatunnel/connectors/seatunnel/clickhouse/ClickhouseCreateTableTest.java b/seatunnel-connectors-v2/connector-clickhouse/src/test/java/org/apache/seatunnel/connectors/seatunnel/clickhouse/ClickhouseCreateTableTest.java index d270be52a87..d83f490dd8d 100644 --- a/seatunnel-connectors-v2/connector-clickhouse/src/test/java/org/apache/seatunnel/connectors/seatunnel/clickhouse/ClickhouseCreateTableTest.java +++ b/seatunnel-connectors-v2/connector-clickhouse/src/test/java/org/apache/seatunnel/connectors/seatunnel/clickhouse/ClickhouseCreateTableTest.java @@ -53,7 +53,8 @@ public void test() { columns.add( PhysicalColumn.of( "age", BasicType.INT_TYPE, (Long) null, true, null, "test comment")); - columns.add(PhysicalColumn.of("score", BasicType.INT_TYPE, (Long) null, true, null, "")); + columns.add( + PhysicalColumn.of("score", BasicType.INT_TYPE, (Long) null, true, null, "'N'-N")); columns.add(PhysicalColumn.of("gender", BasicType.BYTE_TYPE, (Long) null, true, null, "")); columns.add( PhysicalColumn.of("create_time", BasicType.LONG_TYPE, (Long) null, true, null, "")); @@ -103,7 +104,7 @@ public void test() { "CREATE TABLE IF NOT EXISTS `test1`.`test2` (\n" + " `id` Int64 ,`age` Int32 COMMENT 'test comment',\n" + " `name` String ,\n" - + "`score` Int32 ,\n" + + "`score` Int32 COMMENT '''N''-N',\n" + "`gender` Int8 ,\n" + "`create_time` Int64 \n" + ") ENGINE = MergeTree()\n" diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/util/CatalogUtil.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/util/CatalogUtil.java index 1fdc59579a3..89764997044 100644 --- a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/util/CatalogUtil.java +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/util/CatalogUtil.java @@ -106,7 +106,9 @@ public String getCreateTableSql( SaveModePlaceHolder.ROWTYPE_FIELDS.getReplacePlaceHolder(), rowTypeFields) .replaceAll( SaveModePlaceHolder.COMMENT.getReplacePlaceHolder(), - Objects.isNull(comment) ? "" : comment); + Objects.isNull(comment) + ? "" + : comment.replace("'", "''").replace("\\", "\\\\")); } private String mergeColumnInTemplate( diff --git a/seatunnel-connectors-v2/connector-doris/src/main/java/org/apache/seatunnel/connectors/doris/util/DorisCatalogUtil.java b/seatunnel-connectors-v2/connector-doris/src/main/java/org/apache/seatunnel/connectors/doris/util/DorisCatalogUtil.java index 11018d5c5cc..24fd7280110 100644 --- a/seatunnel-connectors-v2/connector-doris/src/main/java/org/apache/seatunnel/connectors/doris/util/DorisCatalogUtil.java +++ b/seatunnel-connectors-v2/connector-doris/src/main/java/org/apache/seatunnel/connectors/doris/util/DorisCatalogUtil.java @@ -209,7 +209,12 @@ public static String getCreateTableStatement( SaveModePlaceHolder.ROWTYPE_FIELDS.getReplacePlaceHolder(), rowTypeFields) .replaceAll( SaveModePlaceHolder.COMMENT.getReplacePlaceHolder(), - Objects.isNull(catalogTable.getComment()) ? "" : catalogTable.getComment()); + Objects.isNull(catalogTable.getComment()) + ? "" + : catalogTable + .getComment() + .replace("'", "''") + .replace("\\", "\\\\")); } private static String mergeColumnInTemplate( @@ -263,6 +268,8 @@ private static String columnToDorisType( column.isNullable() ? "NULL" : "NOT NULL", StringUtils.isEmpty(column.getComment()) ? "" - : "COMMENT '" + column.getComment() + "'"); + : "COMMENT '" + + column.getComment().replace("'", "''").replace("\\", "\\\\") + + "'"); } } diff --git a/seatunnel-connectors-v2/connector-doris/src/test/java/org/apache/seatunnel/connectors/doris/catalog/DorisCreateTableTest.java b/seatunnel-connectors-v2/connector-doris/src/test/java/org/apache/seatunnel/connectors/doris/catalog/DorisCreateTableTest.java index 2142c0e2e2c..8dacabfa195 100644 --- a/seatunnel-connectors-v2/connector-doris/src/test/java/org/apache/seatunnel/connectors/doris/catalog/DorisCreateTableTest.java +++ b/seatunnel-connectors-v2/connector-doris/src/test/java/org/apache/seatunnel/connectors/doris/catalog/DorisCreateTableTest.java @@ -60,7 +60,8 @@ public void test() { columns.add( PhysicalColumn.of( "age", BasicType.INT_TYPE, (Long) null, true, null, "test comment")); - columns.add(PhysicalColumn.of("score", BasicType.INT_TYPE, (Long) null, true, null, "")); + columns.add( + PhysicalColumn.of("score", BasicType.INT_TYPE, (Long) null, true, null, "'N'-N")); columns.add(PhysicalColumn.of("gender", BasicType.BYTE_TYPE, (Long) null, true, null, "")); columns.add( PhysicalColumn.of("create_time", BasicType.LONG_TYPE, (Long) null, true, null, "")); @@ -125,7 +126,7 @@ public void test() { result, "CREATE TABLE IF NOT EXISTS `test1`.`test2` ( \n" + "`id` BIGINT NULL ,`age` INT NULL COMMENT 'test comment' , \n" - + "`name` STRING NULL ,`score` INT NULL , \n" + + "`name` STRING NULL ,`score` INT NULL COMMENT '''N''-N' , \n" + "`create_time` DATETIME NOT NULL , \n" + "`gender` TINYINT NULL \n" + ") ENGINE=OLAP \n" diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/oracle/OracleCatalog.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/oracle/OracleCatalog.java index ccbbfb509dd..3a90c61c6dd 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/oracle/OracleCatalog.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/oracle/OracleCatalog.java @@ -21,6 +21,7 @@ import org.apache.seatunnel.api.table.catalog.Column; import org.apache.seatunnel.api.table.catalog.ConstraintKey; import org.apache.seatunnel.api.table.catalog.TablePath; +import org.apache.seatunnel.api.table.catalog.exception.CatalogException; import org.apache.seatunnel.api.table.converter.BasicTypeDefine; import org.apache.seatunnel.common.utils.JdbcUrlUtil; import org.apache.seatunnel.connectors.seatunnel.jdbc.catalog.AbstractJdbcCatalog; @@ -36,6 +37,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; @Slf4j @@ -100,11 +102,6 @@ public OracleCatalog( this.decimalTypeNarrowing = decimalTypeNarrowing; } - @Override - protected String getDatabaseWithConditionSql(String databaseName) { - return String.format(getListDatabaseSql() + " where name = '%s'", databaseName); - } - @Override protected String getTableWithConditionSql(TablePath tablePath) { return getListTableSql(tablePath.getDatabaseName()) @@ -116,8 +113,13 @@ protected String getTableWithConditionSql(TablePath tablePath) { } @Override - protected String getListDatabaseSql() { - return "SELECT name FROM v$database"; + public boolean databaseExists(String databaseName) throws CatalogException { + return true; + } + + @Override + public List listDatabases() throws CatalogException { + return new ArrayList<>(Collections.singletonList("default")); } @Override @@ -195,11 +197,6 @@ protected String getOptionTableName(TablePath tablePath) { return tablePath.getSchemaAndTableName(); } - private List listTables() { - List databases = listDatabases(); - return listTables(databases.get(0)); - } - @Override public CatalogTable getTable(String sqlQuery) throws SQLException { Connection defaultConnection = getConnection(defaultUrl); diff --git a/seatunnel-connectors-v2/connector-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/mongodb/source/reader/MongodbReader.java b/seatunnel-connectors-v2/connector-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/mongodb/source/reader/MongodbReader.java index e55fa5c8761..84f7575ad54 100644 --- a/seatunnel-connectors-v2/connector-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/mongodb/source/reader/MongodbReader.java +++ b/seatunnel-connectors-v2/connector-mongodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/mongodb/source/reader/MongodbReader.java @@ -82,6 +82,9 @@ public void close() { if (cursor != null) { cursor.close(); } + if (clientProvider != null) { + clientProvider.close(); + } } @Override diff --git a/seatunnel-connectors-v2/connector-starrocks/src/main/java/org/apache/seatunnel/connectors/seatunnel/starrocks/sink/StarRocksSaveModeUtil.java b/seatunnel-connectors-v2/connector-starrocks/src/main/java/org/apache/seatunnel/connectors/seatunnel/starrocks/sink/StarRocksSaveModeUtil.java index 02d3118e071..223695978fd 100644 --- a/seatunnel-connectors-v2/connector-starrocks/src/main/java/org/apache/seatunnel/connectors/seatunnel/starrocks/sink/StarRocksSaveModeUtil.java +++ b/seatunnel-connectors-v2/connector-starrocks/src/main/java/org/apache/seatunnel/connectors/seatunnel/starrocks/sink/StarRocksSaveModeUtil.java @@ -45,7 +45,9 @@ public String columnToConnectorType(Column column) { column.isNullable() ? "NULL" : "NOT NULL", StringUtils.isEmpty(column.getComment()) ? "" - : "COMMENT '" + column.getComment() + "'"); + : "COMMENT '" + + column.getComment().replace("'", "''").replace("\\", "\\\\") + + "'"); } private static String dataTypeToStarrocksType(SeaTunnelDataType dataType, long length) { diff --git a/seatunnel-connectors-v2/connector-starrocks/src/test/java/org/apache/seatunnel/connectors/seatunnel/starrocks/catalog/StarRocksCreateTableTest.java b/seatunnel-connectors-v2/connector-starrocks/src/test/java/org/apache/seatunnel/connectors/seatunnel/starrocks/catalog/StarRocksCreateTableTest.java index 25b06f2806c..ca466745ed2 100644 --- a/seatunnel-connectors-v2/connector-starrocks/src/test/java/org/apache/seatunnel/connectors/seatunnel/starrocks/catalog/StarRocksCreateTableTest.java +++ b/seatunnel-connectors-v2/connector-starrocks/src/test/java/org/apache/seatunnel/connectors/seatunnel/starrocks/catalog/StarRocksCreateTableTest.java @@ -58,7 +58,7 @@ public void test() { columns.add( PhysicalColumn.of( "name", BasicType.STRING_TYPE, (Long) null, true, null, "test comment")); - columns.add(PhysicalColumn.of("age", BasicType.INT_TYPE, (Long) null, true, null, "")); + columns.add(PhysicalColumn.of("age", BasicType.INT_TYPE, (Long) null, true, null, "'N'-N")); columns.add(PhysicalColumn.of("score", BasicType.INT_TYPE, (Long) null, true, null, "")); columns.add(PhysicalColumn.of("gender", BasicType.BYTE_TYPE, (Long) null, true, null, "")); columns.add( @@ -115,7 +115,7 @@ public void test() { StarRocksSinkOptions.SAVE_MODE_CREATE_TEMPLATE.key()); Assertions.assertEquals( "CREATE TABLE IF NOT EXISTS `test1`.`test2` ( \n" - + "`id` BIGINT NULL ,`age` INT NULL , \n" + + "`id` BIGINT NULL ,`age` INT NULL COMMENT '''N''-N' , \n" + "`name` STRING NULL COMMENT 'test comment',`score` INT NULL , \n" + "`create_time` DATETIME NOT NULL , \n" + "`gender` TINYINT NULL \n" diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/clickhouse/ClickhouseIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/clickhouse/ClickhouseIT.java index b830a113893..8e8a7c825d9 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/clickhouse/ClickhouseIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/clickhouse/ClickhouseIT.java @@ -107,6 +107,14 @@ public void testSourceParallelism(TestContainer container) throws Exception { Assertions.assertEquals(0, execResult.getExitCode()); } + @TestTemplate + public void testClickhouseWithCreateSchemaWhenComment(TestContainer container) + throws Exception { + Container.ExecResult execResult = + container.executeJob("/clickhouse_with_create_schema_when_comment.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + } + @TestTemplate public void clickhouseWithCreateSchemaWhenNotExist(TestContainer container) throws Exception { String tableName = "default.sink_table_for_schema"; diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/resources/clickhouse_with_create_schema_when_comment.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/resources/clickhouse_with_create_schema_when_comment.conf new file mode 100644 index 00000000000..5d21b41a100 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/resources/clickhouse_with_create_schema_when_comment.conf @@ -0,0 +1,62 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + parallelism = 1 + job.mode = "BATCH" + checkpoint.interval = 10 +} + +source { + # This is a example source plugin **only for test and demonstrate the feature source plugin** + Clickhouse { + host = "clickhouse:8123" + database = "default" + sql = "select * from source_table" + username = "default" + password = "" + plugin_output = "source_table" + } + # If you would like to get more information about how to configure seatunnel and see full list of source plugins, + # please go to https://seatunnel.apache.org/docs/connector-v2/source/ClickhouseSource +} + +sink { + Clickhouse { + host = "clickhouse:8123" + database = "default" + table = "clickhouse_with_create_schema_when_comment" + username = "default" + password = "" + "schema_save_mode"="CREATE_SCHEMA_WHEN_NOT_EXIST" + "data_save_mode"="APPEND_DATA" + "save_mode_create_template" = """ + CREATE TABLE IF NOT EXISTS `${database}`.`${table}` ( + ${rowtype_fields} + ) ENGINE =Memory + COMMENT '${comment}'; + """ + support_upsert = true + allow_experimental_lightweight_delete = true + } + + # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, + # please go to https://seatunnel.apache.org/docs/connector-v2/sink +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/resources/init/clickhouse_init.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/resources/init/clickhouse_init.conf index 78f2daa1d79..05114016128 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/resources/init/clickhouse_init.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/resources/init/clickhouse_init.conf @@ -19,8 +19,8 @@ source_table = """ set allow_experimental_geo_types = 1; create table if not exists `default`.source_table( `id` Int64, - `c_map` Map(String, Int32), - `c_array_string` Array(String), + `c_map` Map(String, Int32) COMMENT '''N''-N', + `c_array_string` Array(String) COMMENT '\\N\\-N', `c_array_short` Array(Int16), `c_array_int` Array(Int32), `c_array_long` Array(Int64), @@ -51,14 +51,15 @@ create table if not exists `default`.source_table( `c_uint256` UInt256, `c_point` Point, `c_ring` Ring -)engine=Memory; +)engine=Memory +comment '''N''-N'; """ sink_table = """ create table if not exists `default`.sink_table( `id` Int64, - `c_map` Map(String, Int32), - `c_array_string` Array(String), + `c_map` Map(String, Int32) COMMENT '''N''-N', + `c_array_string` Array(String) COMMENT '\\N\\-N', `c_array_short` Array(Int16), `c_array_int` Array(Int32), `c_array_long` Array(Int64), @@ -89,7 +90,8 @@ create table if not exists `default`.sink_table( `c_uint256` UInt256, `c_point` Point, `c_ring` Ring -)engine=Memory; +)engine=Memory +comment '''N''-N'; """ insert_sql = """ diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/src/test/java/org/apache/seatunnel/e2e/connector/doris/DorisCDCSinkIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/src/test/java/org/apache/seatunnel/e2e/connector/doris/DorisCDCSinkIT.java index 7fa699d9981..d13ff4a501e 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/src/test/java/org/apache/seatunnel/e2e/connector/doris/DorisCDCSinkIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/src/test/java/org/apache/seatunnel/e2e/connector/doris/DorisCDCSinkIT.java @@ -26,6 +26,7 @@ import org.apache.seatunnel.e2e.common.junit.DisabledOnContainer; import org.apache.seatunnel.e2e.common.junit.TestContainerExtension; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.TestTemplate; @@ -129,6 +130,13 @@ public void init() { initializeJdbcTable(); } + @AfterAll + public void close() { + if (MYSQL_CONTAINER != null) { + MYSQL_CONTAINER.close(); + } + } + @TestTemplate public void testDorisCDCSink(TestContainer container) throws Exception { diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/src/test/java/org/apache/seatunnel/e2e/connector/doris/DorisCatalogIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/src/test/java/org/apache/seatunnel/e2e/connector/doris/DorisCatalogIT.java index 3fcca216a09..2a55cc350c2 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/src/test/java/org/apache/seatunnel/e2e/connector/doris/DorisCatalogIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/src/test/java/org/apache/seatunnel/e2e/connector/doris/DorisCatalogIT.java @@ -66,7 +66,7 @@ public class DorisCatalogIT extends AbstractDorisIT { TableSchema.Builder builder = TableSchema.builder(); builder.column(PhysicalColumn.of("k1", BasicType.INT_TYPE, 10, false, 0, "k1")); builder.column(PhysicalColumn.of("k2", BasicType.STRING_TYPE, 64, false, "", "k2")); - builder.column(PhysicalColumn.of("v1", BasicType.DOUBLE_TYPE, 10, true, null, "v1")); + builder.column(PhysicalColumn.of("v1", BasicType.DOUBLE_TYPE, 10, true, null, "v1-'v1'")); builder.column(PhysicalColumn.of("v2", new DecimalType(10, 2), 0, false, 0.1, "v2")); builder.primaryKey(PrimaryKey.of("pk", Arrays.asList("k1", "k2"))); catalogTable = @@ -75,7 +75,7 @@ public class DorisCatalogIT extends AbstractDorisIT { builder.build(), Collections.emptyMap(), Collections.emptyList(), - "test"); + "test - \\ 'test'"); } private DorisCatalogFactory factory; diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/src/test/java/org/apache/seatunnel/e2e/connector/doris/DorisSchemaChangeIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/src/test/java/org/apache/seatunnel/e2e/connector/doris/DorisSchemaChangeIT.java index 4f90f53aa31..0eeb2cdbea0 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/src/test/java/org/apache/seatunnel/e2e/connector/doris/DorisSchemaChangeIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/src/test/java/org/apache/seatunnel/e2e/connector/doris/DorisSchemaChangeIT.java @@ -17,8 +17,6 @@ package org.apache.seatunnel.e2e.connector.doris; -import org.apache.seatunnel.shade.com.google.common.collect.Lists; - import org.apache.seatunnel.connectors.seatunnel.cdc.mysql.testutils.MySqlContainer; import org.apache.seatunnel.connectors.seatunnel.cdc.mysql.testutils.MySqlVersion; import org.apache.seatunnel.connectors.seatunnel.cdc.mysql.testutils.UniqueDatabase; @@ -80,7 +78,9 @@ public class DorisSchemaChangeIT extends AbstractDorisIT { private static final String PROJECTION_QUERY = "select id,name,description,weight,add_column1,add_column2,add_column3 from %s.%s order by id;"; private static final MySqlContainer MYSQL_CONTAINER = createMySqlContainer(MySqlVersion.V8_0); - private final UniqueDatabase shopDatabase = new UniqueDatabase(MYSQL_CONTAINER, DATABASE); + private final UniqueDatabase shopDatabase = + new UniqueDatabase( + MYSQL_CONTAINER, DATABASE, MYSQL_USER_NAME, MYSQL_USER_PASSWORD, DATABASE); @TestContainerExtension private final ContainerExtendedFactory extendedFactory = @@ -107,7 +107,6 @@ private static MySqlContainer createMySqlContainer(MySqlVersion version) { .withLogConsumer( new Slf4jLogConsumer( DockerLoggerFactory.getLogger("mysql-docker-image"))); - mySqlContainer.setPortBindings(Lists.newArrayList(String.format("%s:%s", 3306, 3306))); return mySqlContainer; } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-starrocks-e2e/src/test/java/org/apache/seatunnel/e2e/connector/starrocks/StarRocksIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-starrocks-e2e/src/test/java/org/apache/seatunnel/e2e/connector/starrocks/StarRocksIT.java index 1e984205ae9..c49b1bfa418 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-starrocks-e2e/src/test/java/org/apache/seatunnel/e2e/connector/starrocks/StarRocksIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-starrocks-e2e/src/test/java/org/apache/seatunnel/e2e/connector/starrocks/StarRocksIT.java @@ -89,8 +89,9 @@ public class StarRocksIT extends TestSuiteBase implements TestResource { + SOURCE_TABLE + " (\n" + " BIGINT_COL BIGINT,\n" - + " LARGEINT_COL LARGEINT,\n" - + " SMALLINT_COL SMALLINT,\n" + // add comment for test + + " LARGEINT_COL LARGEINT COMMENT '''N''-N',\n" + + " SMALLINT_COL SMALLINT COMMENT '\\N\\-N',\n" + " TINYINT_COL TINYINT,\n" + " BOOLEAN_COL BOOLEAN,\n" + " DECIMAL_COL Decimal(12, 1),\n" @@ -365,6 +366,13 @@ public void testCatalog() { "CREATE TABLE IF NOT EXISTS `${database}`.`${table}` (\n ${rowtype_fields}\n ) ENGINE=OLAP \n DUPLICATE KEY(`BIGINT_COL`) \n COMMENT '${comment}' \n DISTRIBUTED BY HASH (BIGINT_COL) BUCKETS 1 \n PROPERTIES (\n \"replication_num\" = \"1\", \n \"in_memory\" = \"false\" , \n \"storage_format\" = \"DEFAULT\" \n )"); starRocksCatalog.open(); CatalogTable catalogTable = starRocksCatalog.getTable(tablePathStarRocksSource); + catalogTable = + CatalogTable.of( + catalogTable.getTableId(), + catalogTable.getTableSchema(), + catalogTable.getOptions(), + catalogTable.getPartitionKeys(), + "test'1'"); // sink tableExists ? starRocksCatalog.dropTable(tablePathStarRocksSink, true); boolean tableExistsBefore = starRocksCatalog.tableExists(tablePathStarRocksSink); diff --git a/seatunnel-examples/seatunnel-spark-connector-v2-example/pom.xml b/seatunnel-examples/seatunnel-spark-connector-v2-example/pom.xml index 16f974585e9..1602286cf78 100644 --- a/seatunnel-examples/seatunnel-spark-connector-v2-example/pom.xml +++ b/seatunnel-examples/seatunnel-spark-connector-v2-example/pom.xml @@ -85,8 +85,21 @@ com.thoughtworks.paranamer paranamer + + io.netty + netty + + + io.netty + netty-all + + + io.netty + netty-all + 4.1.104.Final + com.thoughtworks.paranamer paranamer diff --git a/tools/dependencies/known-dependencies.txt b/tools/dependencies/known-dependencies.txt index ce7d25b445f..c9513149531 100755 --- a/tools/dependencies/known-dependencies.txt +++ b/tools/dependencies/known-dependencies.txt @@ -26,9 +26,9 @@ protostuff-collectionschema-1.8.0.jar protostuff-core-1.8.0.jar protostuff-runtime-1.8.0.jar scala-library-2.12.15.jar -seatunnel-jackson-2.3.9-SNAPSHOT-optional.jar -seatunnel-guava-2.3.9-SNAPSHOT-optional.jar -seatunnel-hazelcast-shade-2.3.9-SNAPSHOT-optional.jar +seatunnel-jackson-2.3.10-SNAPSHOT-optional.jar +seatunnel-guava-2.3.10-SNAPSHOT-optional.jar +seatunnel-hazelcast-shade-2.3.10-SNAPSHOT-optional.jar slf4j-api-1.7.25.jar jsqlparser-4.9.jar animal-sniffer-annotations-1.17.jar @@ -46,7 +46,7 @@ accessors-smart-2.4.7.jar asm-9.1.jar avro-1.11.1.jar groovy-4.0.16.jar -seatunnel-janino-2.3.9-SNAPSHOT-optional.jar +seatunnel-janino-2.3.10-SNAPSHOT-optional.jar protobuf-java-util-3.25.3.jar protobuf-java-3.25.3.jar protoc-jar-3.11.4.jar @@ -69,7 +69,7 @@ jetty-util-9.4.20.v20190813.jar jetty-util-9.4.56.v20240826.jar jetty-util-ajax-9.4.56.v20240826.jar javax.servlet-api-3.1.0.jar -seatunnel-jetty9-9.4.56-2.3.9-SNAPSHOT-optional.jar +seatunnel-jetty9-9.4.56-2.3.10-SNAPSHOT-optional.jar arrow-format-15.0.1.jar arrow-memory-core-15.0.1.jar arrow-memory-netty-15.0.1.jar @@ -79,4 +79,4 @@ eclipse-collections-api-11.1.0.jar flatbuffers-java-23.5.26.jar netty-buffer-4.1.104.Final.jar netty-common-4.1.104.Final.jar -seatunnel-arrow-2.3.9-SNAPSHOT-optional.jar +seatunnel-arrow-2.3.10-SNAPSHOT-optional.jar