Skip to content

Commit 5c176c8

Browse files
author
dapeng
committed
Merge branch '1.8_release_3.10.x' into 1.8_release-github
# Conflicts: # core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java # launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java # launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java # rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java
2 parents 6f8c9aa + db38a64 commit 5c176c8

File tree

314 files changed

+8106
-4020
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

314 files changed

+8106
-4020
lines changed

‎.gitlab-ci.yml

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
build:
2+
stage: test
3+
script:
4+
- mvn clean org.jacoco:jacoco-maven-plugin:0.7.8:prepare-agent package -Dmaven.test.failure.ignore=true -q
5+
- mvn sonar:sonar -Dsonar.projectKey="dt-insight-engine/flinkStreamSQL" -Dsonar.login=11974c5e9a29625efa09fdc3c3fdc031efb1aab1 -Dsonar.host.url=http://172.16.100.198:9000 -Dsonar.jdbc.url=jdbc:postgresql://172.16.100.198:5432/sonar -Dsonar.java.binaries=target/sonar
6+
- sh ci/sonar_notify.sh
7+
only:
8+
- v1.8.0_dev
9+
tags:
10+
- dt-insight-engine

‎README.md

+2-2
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,8 @@ FlinkStreamSQL
1515
* 自定义create view 语法
1616
* 自定义create function 语法
1717
* 实现了流与维表的join
18-
* 支持原生FLinkSQL所有的语法
19-
* 扩展了输入和输出的性能指标到promethus
18+
* 支持原生FlinkSQL所有的语法
19+
* 扩展了输入和输出的性能指标到Task metrics
2020

2121
## 目录
2222

‎cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java

+5-7
Original file line numberDiff line numberDiff line change
@@ -34,10 +34,10 @@
3434
import com.datastax.driver.core.SocketOptions;
3535
import com.datastax.driver.core.policies.DowngradingConsistencyRetryPolicy;
3636
import com.datastax.driver.core.policies.RetryPolicy;
37-
import com.dtstack.flink.sql.side.AllReqRow;
37+
import com.dtstack.flink.sql.side.BaseAllReqRow;
3838
import com.dtstack.flink.sql.side.FieldInfo;
3939
import com.dtstack.flink.sql.side.JoinInfo;
40-
import com.dtstack.flink.sql.side.SideTableInfo;
40+
import com.dtstack.flink.sql.side.AbstractSideTableInfo;
4141
import com.dtstack.flink.sql.side.cassandra.table.CassandraSideTableInfo;
4242
import com.google.common.collect.Lists;
4343
import com.google.common.collect.Maps;
@@ -62,14 +62,12 @@
6262
*
6363
* @author xuqianjin
6464
*/
65-
public class CassandraAllReqRow extends AllReqRow {
65+
public class CassandraAllReqRow extends BaseAllReqRow {
6666

6767
private static final long serialVersionUID = 54015343561288219L;
6868

6969
private static final Logger LOG = LoggerFactory.getLogger(CassandraAllReqRow.class);
7070

71-
private static final String cassandra_DRIVER = "com.cassandra.jdbc.Driver";
72-
7371
private static final int CONN_RETRY_NUM = 3;
7472

7573
private static final int FETCH_SIZE = 1000;
@@ -79,7 +77,7 @@ public class CassandraAllReqRow extends AllReqRow {
7977

8078
private AtomicReference<Map<String, List<Map<String, Object>>>> cacheRef = new AtomicReference<>();
8179

82-
public CassandraAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List<FieldInfo> outFieldInfoList, SideTableInfo sideTableInfo) {
80+
public CassandraAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List<FieldInfo> outFieldInfoList, AbstractSideTableInfo sideTableInfo) {
8381
super(new com.dtstack.flink.sql.side.cassandra.CassandraAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo));
8482
}
8583

@@ -269,7 +267,7 @@ private void loadData(Map<String, List<Map<String, Object>>> tmpCache) throws SQ
269267
String connInfo = "address:" + tableInfo.getAddress() + ";userName:" + tableInfo.getUserName()
270268
+ ",pwd:" + tableInfo.getPassword();
271269
LOG.warn("get conn fail, wait for 5 sec and try again, connInfo:" + connInfo);
272-
Thread.sleep(5 * 1000);
270+
Thread.sleep(LOAD_DATA_ERROR_SLEEP_TIME);
273271
} catch (InterruptedException e1) {
274272
LOG.error("", e1);
275273
}

‎cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllSideInfo.java

+5-5
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,8 @@
2020

2121
import com.dtstack.flink.sql.side.FieldInfo;
2222
import com.dtstack.flink.sql.side.JoinInfo;
23-
import com.dtstack.flink.sql.side.SideInfo;
24-
import com.dtstack.flink.sql.side.SideTableInfo;
23+
import com.dtstack.flink.sql.side.BaseSideInfo;
24+
import com.dtstack.flink.sql.side.AbstractSideTableInfo;
2525
import com.dtstack.flink.sql.side.cassandra.table.CassandraSideTableInfo;
2626
import com.dtstack.flink.sql.util.ParseUtils;
2727
import org.apache.calcite.sql.SqlNode;
@@ -37,16 +37,16 @@
3737
*
3838
* @author xuqianjin
3939
*/
40-
public class CassandraAllSideInfo extends SideInfo {
40+
public class CassandraAllSideInfo extends BaseSideInfo {
4141

4242
private static final long serialVersionUID = -8690814317653033557L;
4343

44-
public CassandraAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List<FieldInfo> outFieldInfoList, SideTableInfo sideTableInfo) {
44+
public CassandraAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List<FieldInfo> outFieldInfoList, AbstractSideTableInfo sideTableInfo) {
4545
super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo);
4646
}
4747

4848
@Override
49-
public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) {
49+
public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) {
5050
CassandraSideTableInfo cassandraSideTableInfo = (CassandraSideTableInfo) sideTableInfo;
5151

5252
sqlCondition = "select ${selectField} from ${tableName} ";

‎cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java

+28-68
Original file line numberDiff line numberDiff line change
@@ -37,11 +37,11 @@
3737
import com.datastax.driver.core.policies.DowngradingConsistencyRetryPolicy;
3838
import com.datastax.driver.core.policies.RetryPolicy;
3939
import com.dtstack.flink.sql.enums.ECacheContentType;
40-
import com.dtstack.flink.sql.side.AsyncReqRow;
40+
import com.dtstack.flink.sql.side.BaseAsyncReqRow;
4141
import com.dtstack.flink.sql.side.CacheMissVal;
4242
import com.dtstack.flink.sql.side.FieldInfo;
4343
import com.dtstack.flink.sql.side.JoinInfo;
44-
import com.dtstack.flink.sql.side.SideTableInfo;
44+
import com.dtstack.flink.sql.side.AbstractSideTableInfo;
4545
import com.dtstack.flink.sql.side.cache.CacheObj;
4646
import com.dtstack.flink.sql.side.cassandra.table.CassandraSideTableInfo;
4747
import com.google.common.base.Function;
@@ -67,7 +67,7 @@
6767
*
6868
* @author xuqianjin
6969
*/
70-
public class CassandraAsyncReqRow extends AsyncReqRow {
70+
public class CassandraAsyncReqRow extends BaseAsyncReqRow {
7171

7272
private static final long serialVersionUID = 6631584128079864735L;
7373

@@ -83,7 +83,7 @@ public class CassandraAsyncReqRow extends AsyncReqRow {
8383
private transient ListenableFuture session;
8484
private transient CassandraSideTableInfo cassandraSideTableInfo;
8585

86-
public CassandraAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List<FieldInfo> outFieldInfoList, SideTableInfo sideTableInfo) {
86+
public CassandraAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List<FieldInfo> outFieldInfoList, AbstractSideTableInfo sideTableInfo) {
8787
super(new com.dtstack.flink.sql.side.cassandra.CassandraAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo));
8888
}
8989

@@ -162,61 +162,14 @@ private void connCassandraDB(CassandraSideTableInfo tableInfo) {
162162
}
163163

164164
@Override
165-
public void asyncInvoke(CRow input, ResultFuture<CRow> resultFuture) throws Exception {
166-
CRow inputCopy = new CRow(input.row(), input.change());
167-
JsonArray inputParams = new JsonArray();
168-
StringBuffer stringBuffer = new StringBuffer();
169-
String sqlWhere = " where ";
170-
171-
for (int i = 0; i < sideInfo.getEqualFieldList().size(); i++) {
172-
Integer conValIndex = sideInfo.getEqualValIndex().get(i);
173-
Object equalObj = inputCopy.row().getField(conValIndex);
174-
if (equalObj == null) {
175-
dealMissKey(inputCopy, resultFuture);
176-
return;
177-
}
178-
inputParams.add(equalObj);
179-
StringBuffer sqlTemp = stringBuffer.append(sideInfo.getEqualFieldList().get(i))
180-
.append(" = ");
181-
if (equalObj instanceof String) {
182-
sqlTemp.append("'" + equalObj + "'")
183-
.append(" and ");
184-
} else {
185-
sqlTemp.append(equalObj)
186-
.append(" and ");
187-
}
188-
189-
}
165+
public void handleAsyncInvoke(Map<String, Object> inputParams, CRow input, ResultFuture<CRow> resultFuture) throws Exception {
190166

191167
String key = buildCacheKey(inputParams);
192-
sqlWhere = sqlWhere + stringBuffer.toString().substring(0, stringBuffer.lastIndexOf(" and "));
193-
194-
if (openCache()) {
195-
CacheObj val = getFromCache(key);
196-
if (val != null) {
197-
198-
if (ECacheContentType.MissVal == val.getType()) {
199-
dealMissKey(inputCopy, resultFuture);
200-
return;
201-
} else if (ECacheContentType.MultiLine == val.getType()) {
202-
List<CRow> rowList = Lists.newArrayList();
203-
for (Object jsonArray : (List) val.getContent()) {
204-
Row row = fillData(inputCopy.row(), jsonArray);
205-
rowList.add(new CRow(row, inputCopy.change()));
206-
}
207-
resultFuture.complete(rowList);
208-
} else {
209-
throw new RuntimeException("not support cache obj type " + val.getType());
210-
}
211-
return;
212-
}
213-
}
214-
215168
//connect Cassandra
216169
connCassandraDB(cassandraSideTableInfo);
217170

218-
String sqlCondition = sideInfo.getSqlCondition() + " " + sqlWhere + " ALLOW FILTERING ";
219-
System.out.println("sqlCondition:" + sqlCondition);
171+
String sqlCondition = sideInfo.getSqlCondition() + " " + buildWhereCondition(inputParams) + " ALLOW FILTERING ";
172+
LOG.info("sqlCondition:{}" + sqlCondition);
220173

221174
ListenableFuture<ResultSet> resultSet = Futures.transformAsync(session,
222175
new AsyncFunction<Session, ResultSet>() {
@@ -242,18 +195,18 @@ public void onSuccess(List<com.datastax.driver.core.Row> rows) {
242195
List<com.datastax.driver.core.Row> cacheContent = Lists.newArrayList();
243196
List<CRow> rowList = Lists.newArrayList();
244197
for (com.datastax.driver.core.Row line : rows) {
245-
Row row = fillData(inputCopy.row(), line);
198+
Row row = fillData(input.row(), line);
246199
if (openCache()) {
247200
cacheContent.add(line);
248201
}
249-
rowList.add(new CRow(row,inputCopy.change()));
202+
rowList.add(new CRow(row, input.change()));
250203
}
251204
resultFuture.complete(rowList);
252205
if (openCache()) {
253206
putCache(key, CacheObj.buildCacheObj(ECacheContentType.MultiLine, cacheContent));
254207
}
255208
} else {
256-
dealMissKey(inputCopy, resultFuture);
209+
dealMissKey(input, resultFuture);
257210
if (openCache()) {
258211
putCache(key, CacheMissVal.getMissKeyObj());
259212
}
@@ -265,13 +218,30 @@ public void onSuccess(List<com.datastax.driver.core.Row> rows) {
265218
public void onFailure(Throwable t) {
266219
LOG.error("Failed to retrieve the data: %s%n",
267220
t.getMessage());
268-
System.out.println("Failed to retrieve the data: " + t.getMessage());
269221
cluster.closeAsync();
270222
resultFuture.completeExceptionally(t);
271223
}
272224
});
273225
}
274226

227+
@Override
228+
public String buildCacheKey(Map<String, Object> inputParams) {
229+
StringBuilder sb = new StringBuilder();
230+
for (Object ele : inputParams.values()) {
231+
sb.append(ele.toString()).append("_");
232+
}
233+
return sb.toString();
234+
}
235+
236+
private String buildWhereCondition(Map<String, Object> inputParams){
237+
StringBuilder sb = new StringBuilder(" where ");
238+
for(Map.Entry<String, Object> entry : inputParams.entrySet()){
239+
Object value = entry.getValue() instanceof String ? "'" + entry.getValue() + "'" : entry.getValue();
240+
sb.append(String.format("%s = %s", entry.getKey(), value));
241+
}
242+
return sb.toString();
243+
}
244+
275245
@Override
276246
public Row fillData(Row input, Object line) {
277247
com.datastax.driver.core.Row rowArray = (com.datastax.driver.core.Row) line;
@@ -306,14 +276,4 @@ public void close() throws Exception {
306276
cluster = null;
307277
}
308278
}
309-
310-
public String buildCacheKey(JsonArray jsonArray) {
311-
StringBuilder sb = new StringBuilder();
312-
for (Object ele : jsonArray.getList()) {
313-
sb.append(ele.toString())
314-
.append("_");
315-
}
316-
317-
return sb.toString();
318-
}
319279
}

‎cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncSideInfo.java

+11-7
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,8 @@
2020

2121
import com.dtstack.flink.sql.side.FieldInfo;
2222
import com.dtstack.flink.sql.side.JoinInfo;
23-
import com.dtstack.flink.sql.side.SideInfo;
24-
import com.dtstack.flink.sql.side.SideTableInfo;
23+
import com.dtstack.flink.sql.side.BaseSideInfo;
24+
import com.dtstack.flink.sql.side.AbstractSideTableInfo;
2525
import com.dtstack.flink.sql.side.cassandra.table.CassandraSideTableInfo;
2626
import com.dtstack.flink.sql.util.ParseUtils;
2727
import org.apache.calcite.sql.SqlBasicCall;
@@ -30,6 +30,8 @@
3030
import org.apache.calcite.sql.SqlNode;
3131
import org.apache.flink.api.java.typeutils.RowTypeInfo;
3232
import com.google.common.collect.Lists;
33+
import org.slf4j.Logger;
34+
import org.slf4j.LoggerFactory;
3335

3436
import java.util.List;
3537

@@ -39,16 +41,18 @@
3941
*
4042
* @author xuqianjin
4143
*/
42-
public class CassandraAsyncSideInfo extends SideInfo {
44+
public class CassandraAsyncSideInfo extends BaseSideInfo {
4345

4446
private static final long serialVersionUID = -4403313049809013362L;
47+
private static final Logger LOG = LoggerFactory.getLogger(CassandraAsyncSideInfo.class.getSimpleName());
4548

46-
public CassandraAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List<FieldInfo> outFieldInfoList, SideTableInfo sideTableInfo) {
49+
50+
public CassandraAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List<FieldInfo> outFieldInfoList, AbstractSideTableInfo sideTableInfo) {
4751
super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo);
4852
}
4953

5054
@Override
51-
public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) {
55+
public void buildEqualInfo(JoinInfo joinInfo, AbstractSideTableInfo sideTableInfo) {
5256
CassandraSideTableInfo cassandraSideTableInfo = (CassandraSideTableInfo) sideTableInfo;
5357

5458
String sideTableName = joinInfo.getSideTableName();
@@ -63,9 +67,9 @@ public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) {
6367
}
6468

6569
sqlCondition = "select ${selectField} from ${tableName}";
66-
6770
sqlCondition = sqlCondition.replace("${tableName}", cassandraSideTableInfo.getDatabase()+"."+cassandraSideTableInfo.getTableName()).replace("${selectField}", sideSelectFields);
68-
System.out.println("---------side_exe_sql-----\n" + sqlCondition);
71+
72+
LOG.info("---------side_exe_sql-----\n{}" + sqlCondition);
6973
}
7074

7175

‎cassandra/cassandra-side/cassandra-side-core/src/main/java/com/dtstack/flink/sql/side/cassandra/table/CassandraSideParser.java

+9-8
Original file line numberDiff line numberDiff line change
@@ -19,26 +19,24 @@
1919

2020
package com.dtstack.flink.sql.side.cassandra.table;
2121

22-
import com.dtstack.flink.sql.table.AbsSideTableParser;
23-
import com.dtstack.flink.sql.table.TableInfo;
22+
import com.dtstack.flink.sql.table.AbstractSideTableParser;
23+
import com.dtstack.flink.sql.table.AbstractTableInfo;
2424
import com.dtstack.flink.sql.util.MathUtil;
2525

26-
import java.math.BigDecimal;
27-
import java.sql.Date;
2826
import java.sql.Timestamp;
2927
import java.util.Map;
3028
import java.util.regex.Matcher;
3129
import java.util.regex.Pattern;
3230

33-
import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY;
31+
import static com.dtstack.flink.sql.table.AbstractTableInfo.PARALLELISM_KEY;
3432

3533
/**
3634
* Reason:
3735
* Date: 2018/11/22
3836
*
3937
* @author xuqianjin
4038
*/
41-
public class CassandraSideParser extends AbsSideTableParser {
39+
public class CassandraSideParser extends AbstractSideTableParser {
4240

4341
private final static String SIDE_SIGN_KEY = "sideSignKey";
4442

@@ -73,7 +71,7 @@ public CassandraSideParser() {
7371
}
7472

7573
@Override
76-
public TableInfo getTableInfo(String tableName, String fieldsInfo, Map<String, Object> props) {
74+
public AbstractTableInfo getTableInfo(String tableName, String fieldsInfo, Map<String, Object> props) {
7775
com.dtstack.flink.sql.side.cassandra.table.CassandraSideTableInfo cassandraSideTableInfo = new com.dtstack.flink.sql.side.cassandra.table.CassandraSideTableInfo();
7876
cassandraSideTableInfo.setName(tableName);
7977
parseFieldsInfo(fieldsInfo, cassandraSideTableInfo);
@@ -96,9 +94,10 @@ public TableInfo getTableInfo(String tableName, String fieldsInfo, Map<String, O
9694
return cassandraSideTableInfo;
9795
}
9896

99-
private void dealSideSign(Matcher matcher, TableInfo tableInfo) {
97+
private void dealSideSign(Matcher matcher, AbstractTableInfo tableInfo) {
10098
}
10199

100+
@Override
102101
public Class dbTypeConvertToJavaType(String fieldType) {
103102
switch (fieldType.toLowerCase()) {
104103
case "bigint":
@@ -121,6 +120,8 @@ public Class dbTypeConvertToJavaType(String fieldType) {
121120
return Double.class;
122121
case "timestamp":
123122
return Timestamp.class;
123+
default:
124+
break;
124125
}
125126

126127
throw new RuntimeException("不支持 " + fieldType + " 类型");

‎cassandra/cassandra-side/cassandra-side-core/src/main/java/com/dtstack/flink/sql/side/cassandra/table/CassandraSideTableInfo.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919

2020
package com.dtstack.flink.sql.side.cassandra.table;
2121

22-
import com.dtstack.flink.sql.side.SideTableInfo;
22+
import com.dtstack.flink.sql.side.AbstractSideTableInfo;
2323
import com.google.common.base.Preconditions;
2424

2525
/**
@@ -28,7 +28,7 @@
2828
*
2929
* @author xuqianjin
3030
*/
31-
public class CassandraSideTableInfo extends SideTableInfo {
31+
public class CassandraSideTableInfo extends AbstractSideTableInfo {
3232

3333
private static final long serialVersionUID = -5556431094535478915L;
3434

0 commit comments

Comments
 (0)