Skip to content

Commit 994d17a

Browse files
author
linjc13
committed
[FLINK-36794] [cdc-composer/cli] pipeline cdc connector support multiple data sources
1 parent 92081df commit 994d17a

File tree

13 files changed

+451
-102
lines changed

13 files changed

+451
-102
lines changed

docs/content.zh/docs/connectors/pipeline-connectors/mysql.md

Lines changed: 40 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,9 +50,9 @@ MySQL CDC Pipeline 连接器允许从 MySQL 数据库读取快照数据和增量
5050
</table>
5151
</div>
5252

53-
## 示例
53+
## 单数据源示例
5454

55-
MySQL 读取数据同步到 Doris 的 Pipeline 可以定义如下:
55+
单数据源,从单个 MySQL 读取数据同步到 Doris 的 Pipeline 可以定义如下:
5656

5757
```yaml
5858
source:
@@ -77,6 +77,44 @@ pipeline:
7777
parallelism: 4
7878
```
7979
80+
## 多数据源示例
81+
82+
多数据源,从多个mysql数据源读取数据同步到 Doris 的 Pipeline 可以定义如下:
83+
84+
```yaml
85+
sources:
86+
- type: mysql
87+
name: MySQL multiple Source1
88+
hostname: 127.0.0.1
89+
port: 3306
90+
username: admin
91+
password: pass
92+
tables: adb.\.*, bdb.user_table_[0-9]+, [app|web].order_\.*
93+
server-id: 5400-5404
94+
server-time-zone: Asia/Shanghai
95+
96+
- type: mysql
97+
name: MySQL multiple Source2
98+
hostname: 127.0.0.2
99+
port: 3307
100+
username: admin
101+
password: pass
102+
tables: adb.\.*, bdb.user_table_[0-9]+, [app|web].order_\.*
103+
server-id: 5405-5409
104+
server-time-zone: Asia/Shanghai
105+
106+
sink:
107+
type: doris
108+
name: Doris Sink
109+
fenodes: 127.0.0.1:8030
110+
username: root
111+
password: pass
112+
113+
pipeline:
114+
name: MySQL to Doris Pipeline
115+
parallelism: 4
116+
```
117+
80118
## 连接器配置项
81119
82120
<div class="highlight">

docs/content/docs/connectors/pipeline-connectors/mysql.md

Lines changed: 40 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -51,9 +51,9 @@ You may need to configure the following dependencies manually, and pass it with
5151
</table>
5252
</div>
5353

54-
## Example
54+
## single data source Example
5555

56-
An example of the pipeline for reading data from MySQL and sink to Doris can be defined as follows:
56+
An example of the pipeline for reading data from single MySQL and sink to Doris can be defined as follows:
5757

5858
```yaml
5959
source:
@@ -78,6 +78,44 @@ pipeline:
7878
parallelism: 4
7979
```
8080
81+
## multiple data source Example
82+
83+
An example of the pipeline for reading data from multiple MySQL datasource and sink to Doris can be defined as follows:
84+
85+
```yaml
86+
sources:
87+
- type: mysql
88+
name: MySQL multiple Source1
89+
hostname: 127.0.0.1
90+
port: 3306
91+
username: admin
92+
password: pass
93+
tables: adb.\.*, bdb.user_table_[0-9]+, [app|web].order_\.*
94+
server-id: 5400-5404
95+
server-time-zone: Asia/Shanghai
96+
97+
- type: mysql
98+
name: MySQL multiple Source2
99+
hostname: 127.0.0.2
100+
port: 3307
101+
username: admin
102+
password: pass
103+
tables: adb.\.*, bdb.user_table_[0-9]+, [app|web].order_\.*
104+
server-id: 5405-5409
105+
server-time-zone: Asia/Shanghai
106+
107+
sink:
108+
type: doris
109+
name: Doris Sink
110+
fenodes: 127.0.0.1:8030
111+
username: root
112+
password: pass
113+
114+
pipeline:
115+
name: MySQL to Doris Pipeline
116+
parallelism: 4
117+
```
118+
81119
## Connector Options
82120
83121
<div class="highlight">

flink-cdc-cli/src/main/java/org/apache/flink/cdc/cli/parser/YamlPipelineDefinitionParser.java

Lines changed: 36 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
import java.nio.file.Path;
4141
import java.util.ArrayList;
4242
import java.util.Arrays;
43+
import java.util.Iterator;
4344
import java.util.List;
4445
import java.util.Map;
4546
import java.util.Optional;
@@ -55,6 +56,7 @@ public class YamlPipelineDefinitionParser implements PipelineDefinitionParser {
5556

5657
// Parent node keys
5758
private static final String SOURCE_KEY = "source";
59+
private static final String MULTIPLE_SOURCE_KEY = "sources";
5860
private static final String SINK_KEY = "sink";
5961
private static final String ROUTE_KEY = "route";
6062
private static final String TRANSFORM_KEY = "transform";
@@ -63,6 +65,7 @@ public class YamlPipelineDefinitionParser implements PipelineDefinitionParser {
6365

6466
// Source / sink keys
6567
private static final String TYPE_KEY = "type";
68+
private static final String SOURCES = "sources";
6669
private static final String NAME_KEY = "name";
6770
private static final String INCLUDE_SCHEMA_EVOLUTION_TYPES = "include.schema.changes";
6871
private static final String EXCLUDE_SCHEMA_EVOLUTION_TYPES = "exclude.schema.changes";
@@ -135,13 +138,20 @@ private PipelineDef parse(JsonNode pipelineDefJsonNode, Configuration globalPipe
135138
SchemaChangeBehavior schemaChangeBehavior =
136139
userPipelineConfig.get(PIPELINE_SCHEMA_CHANGE_BEHAVIOR);
137140

138-
// Source is required
139-
SourceDef sourceDef =
140-
toSourceDef(
141-
checkNotNull(
142-
pipelineDefJsonNode.get(SOURCE_KEY),
143-
"Missing required field \"%s\" in pipeline definition",
144-
SOURCE_KEY));
141+
JsonNode multipleSourceNode = pipelineDefJsonNode.get(MULTIPLE_SOURCE_KEY);
142+
List<SourceDef> sourceDefs = new ArrayList<>();
143+
SourceDef sourceDef = null;
144+
if (multipleSourceNode != null) {
145+
Iterator<JsonNode> it = multipleSourceNode.elements();
146+
while (it.hasNext()) {
147+
JsonNode sourceNode = it.next();
148+
getSourceDefs(sourceNode, sourceDefs);
149+
}
150+
} else {
151+
JsonNode sourceNode = pipelineDefJsonNode.get(SOURCE_KEY);
152+
// Source is required
153+
sourceDef = getSourceDefs(sourceNode, sourceDefs);
154+
}
145155

146156
// Sink is required
147157
SinkDef sinkDef =
@@ -171,7 +181,25 @@ private PipelineDef parse(JsonNode pipelineDefJsonNode, Configuration globalPipe
171181
pipelineConfig.addAll(userPipelineConfig);
172182

173183
return new PipelineDef(
174-
sourceDef, sinkDef, routeDefs, transformDefs, udfDefs, modelDefs, pipelineConfig);
184+
sourceDefs,
185+
sourceDef,
186+
sinkDef,
187+
routeDefs,
188+
transformDefs,
189+
udfDefs,
190+
modelDefs,
191+
pipelineConfig);
192+
}
193+
194+
private SourceDef getSourceDefs(JsonNode root, List<SourceDef> sourceDefs) {
195+
SourceDef sourceDef =
196+
toSourceDef(
197+
checkNotNull(
198+
root,
199+
"Missing required field \"%s\" in pipeline definition",
200+
SOURCE_KEY));
201+
sourceDefs.add(sourceDef);
202+
return sourceDef;
175203
}
176204

177205
private SourceDef toSourceDef(JsonNode sourceNode) {

0 commit comments

Comments
 (0)