Skip to content

Commit adf888e

Browse files
Hisoka-XHisoka-X
authored
[Improve][Connector-V2] Remove system table limit (#7391)
1 parent 6937d10 commit adf888e

File tree

13 files changed

+3
-185
lines changed

13 files changed

+3
-185
lines changed

‎seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/AbstractJdbcCatalog.java

Lines changed: 2 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -55,12 +55,10 @@
5555
import java.util.ArrayList;
5656
import java.util.Collections;
5757
import java.util.HashMap;
58-
import java.util.HashSet;
5958
import java.util.LinkedHashMap;
6059
import java.util.List;
6160
import java.util.Map;
6261
import java.util.Optional;
63-
import java.util.Set;
6462
import java.util.concurrent.ConcurrentHashMap;
6563

6664
import static com.google.common.base.Preconditions.checkArgument;
@@ -71,9 +69,6 @@
7169
public abstract class AbstractJdbcCatalog implements Catalog {
7270
private static final Logger LOG = LoggerFactory.getLogger(AbstractJdbcCatalog.class);
7371

74-
protected static final Set<String> SYS_DATABASES = new HashSet<>();
75-
protected static final Set<String> EXCLUDED_SCHEMAS = new HashSet<>();
76-
7772
protected final String catalogName;
7873
protected final String defaultDatabase;
7974
protected final String username;
@@ -271,13 +266,7 @@ protected String getDatabaseWithConditionSql(String databaseName) {
271266
@Override
272267
public List<String> listDatabases() throws CatalogException {
273268
try {
274-
return queryString(
275-
defaultUrl,
276-
getListDatabaseSql(),
277-
rs -> {
278-
String s = rs.getString(1);
279-
return SYS_DATABASES.contains(s) ? null : s;
280-
});
269+
return queryString(defaultUrl, getListDatabaseSql(), rs -> rs.getString(1));
281270
} catch (Exception e) {
282271
throw new CatalogException(
283272
String.format("Failed listing database in catalog %s", this.catalogName), e);
@@ -289,9 +278,6 @@ public boolean databaseExists(String databaseName) throws CatalogException {
289278
if (StringUtils.isBlank(databaseName)) {
290279
return false;
291280
}
292-
if (SYS_DATABASES.contains(databaseName)) {
293-
return false;
294-
}
295281
try {
296282
return querySQLResultExists(
297283
getUrlFromDatabaseName(databaseName),
@@ -318,7 +304,7 @@ protected String getTableWithConditionSql(TablePath tablePath) {
318304
protected String getTableName(ResultSet rs) throws SQLException {
319305
String schemaName = rs.getString(1);
320306
String tableName = rs.getString(2);
321-
if (StringUtils.isNotBlank(schemaName) && !SYS_DATABASES.contains(schemaName)) {
307+
if (StringUtils.isNotBlank(schemaName)) {
322308
return schemaName + "." + tableName;
323309
}
324310
return null;
@@ -347,9 +333,6 @@ public List<String> listTables(String databaseName)
347333
@Override
348334
public boolean tableExists(TablePath tablePath) throws CatalogException {
349335
String databaseName = tablePath.getDatabaseName();
350-
if (EXCLUDED_SCHEMAS.contains(tablePath.getSchemaName())) {
351-
return false;
352-
}
353336
try {
354337
return querySQLResultExists(
355338
this.getUrlFromDatabaseName(databaseName), getTableWithConditionSql(tablePath));

‎seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/dm/DamengCatalog.java

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -37,15 +37,10 @@
3737
import java.sql.ResultSet;
3838
import java.sql.SQLException;
3939
import java.util.ArrayList;
40-
import java.util.Arrays;
41-
import java.util.Collections;
4240
import java.util.List;
4341

4442
@Slf4j
4543
public class DamengCatalog extends AbstractJdbcCatalog {
46-
private static final List<String> EXCLUDED_SCHEMAS =
47-
Collections.unmodifiableList(
48-
Arrays.asList("SYS", "SYSDBA", "SYSSSO", "SYSAUDITOR", "CTISYS"));
4944

5045
private static final String SELECT_COLUMNS_SQL =
5146
"SELECT COLUMNS.COLUMN_NAME, COLUMNS.DATA_TYPE, COLUMNS.DATA_LENGTH, COLUMNS.DATA_PRECISION, COLUMNS.DATA_SCALE "
@@ -110,9 +105,6 @@ protected String getListTableSql(String databaseName) {
110105

111106
@Override
112107
protected String getTableName(ResultSet rs) throws SQLException {
113-
if (EXCLUDED_SCHEMAS.contains(rs.getString(1))) {
114-
return null;
115-
}
116108
return rs.getString(1) + "." + rs.getString(2);
117109
}
118110

@@ -177,9 +169,6 @@ public List<String> listTables(String databaseName)
177169

178170
List<String> tables = new ArrayList<>();
179171
while (rs.next()) {
180-
if (EXCLUDED_SCHEMAS.contains(rs.getString(1))) {
181-
continue;
182-
}
183172
tables.add(rs.getString(1) + "." + rs.getString(2));
184173
}
185174

‎seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/iris/IrisCatalog.java

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,6 @@ public class IrisCatalog extends AbstractJdbcCatalog {
6262
public IrisCatalog(
6363
String catalogName, String username, String password, JdbcUrlUtil.UrlInfo urlInfo) {
6464
super(catalogName, username, password, urlInfo, null);
65-
SYS_DATABASES.add("%SYS");
6665
}
6766

6867
@Override
@@ -138,9 +137,6 @@ public boolean databaseExists(String databaseName) throws CatalogException {
138137

139138
@Override
140139
public boolean tableExists(TablePath tablePath) throws CatalogException {
141-
if (EXCLUDED_SCHEMAS.contains(tablePath.getSchemaName())) {
142-
return false;
143-
}
144140
return querySQLResultExists(
145141
this.getUrlFromDatabaseName(tablePath.getDatabaseName()),
146142
getTableWithConditionSql(tablePath));

‎seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/mysql/MySqlCatalog.java

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -57,13 +57,6 @@ public class MySqlCatalog extends AbstractJdbcCatalog {
5757
private static final String SELECT_TABLE_EXISTS =
5858
"SELECT TABLE_SCHEMA,TABLE_NAME FROM information_schema.tables WHERE table_schema = '%s' AND table_name = '%s'";
5959

60-
static {
61-
SYS_DATABASES.add("information_schema");
62-
SYS_DATABASES.add("mysql");
63-
SYS_DATABASES.add("performance_schema");
64-
SYS_DATABASES.add("sys");
65-
}
66-
6760
private MySqlVersion version;
6861
private MySqlTypeConverter typeConverter;
6962

‎seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/oceanbase/OceanBaseMySqlCatalog.java

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -56,16 +56,6 @@ public class OceanBaseMySqlCatalog extends AbstractJdbcCatalog {
5656
private static final String SELECT_TABLE_EXISTS =
5757
"SELECT TABLE_SCHEMA,TABLE_NAME FROM information_schema.tables WHERE table_schema = '%s' AND table_name = '%s'";
5858

59-
static {
60-
SYS_DATABASES.clear();
61-
SYS_DATABASES.add("information_schema");
62-
SYS_DATABASES.add("mysql");
63-
SYS_DATABASES.add("oceanbase");
64-
SYS_DATABASES.add("LBACSYS");
65-
SYS_DATABASES.add("ORAAUDITOR");
66-
SYS_DATABASES.add("SYS");
67-
}
68-
6959
private OceanBaseMySqlTypeConverter typeConverter;
7060

7161
public OceanBaseMySqlCatalog(

‎seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/oceanbase/OceanBaseOracleCatalog.java

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -31,13 +31,6 @@
3131

3232
public class OceanBaseOracleCatalog extends OracleCatalog {
3333

34-
static {
35-
EXCLUDED_SCHEMAS.add("oceanbase");
36-
EXCLUDED_SCHEMAS.add("LBACSYS");
37-
EXCLUDED_SCHEMAS.add("ORAAUDITOR");
38-
EXCLUDED_SCHEMAS.add("SYS");
39-
}
40-
4134
public OceanBaseOracleCatalog(
4235
String catalogName,
4336
String username,
@@ -59,9 +52,6 @@ protected String getDatabaseWithConditionSql(String databaseName) {
5952

6053
@Override
6154
public boolean tableExists(TablePath tablePath) throws CatalogException {
62-
if (EXCLUDED_SCHEMAS.contains(tablePath.getSchemaName())) {
63-
return false;
64-
}
6555
return querySQLResultExists(
6656
this.getUrlFromDatabaseName(tablePath.getDatabaseName()),
6757
getTableWithConditionSql(tablePath));

‎seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/oracle/OracleCatalog.java

Lines changed: 0 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -35,37 +35,11 @@
3535
import java.sql.ResultSet;
3636
import java.sql.SQLException;
3737
import java.util.ArrayList;
38-
import java.util.Arrays;
39-
import java.util.Collections;
4038
import java.util.List;
4139

4240
@Slf4j
4341
public class OracleCatalog extends AbstractJdbcCatalog {
4442

45-
protected static List<String> EXCLUDED_SCHEMAS_ALL =
46-
Collections.unmodifiableList(
47-
Arrays.asList(
48-
"APPQOSSYS",
49-
"AUDSYS",
50-
"CTXSYS",
51-
"DVSYS",
52-
"DBSFWUSER",
53-
"DBSNMP",
54-
"GSMADMIN_INTERNAL",
55-
"LBACSYS",
56-
"MDSYS",
57-
"OJVMSYS",
58-
"OLAPSYS",
59-
"ORDDATA",
60-
"ORDSYS",
61-
"OUTLN",
62-
"SYS",
63-
"SYSTEM",
64-
"WMSYS",
65-
"XDB",
66-
"EXFSYS",
67-
"SYSMAN"));
68-
6943
private static final String SELECT_COLUMNS_SQL_TEMPLATE =
7044
"SELECT\n"
7145
+ " cols.COLUMN_NAME,\n"
@@ -97,10 +71,6 @@ public class OracleCatalog extends AbstractJdbcCatalog {
9771
+ "ORDER BY \n"
9872
+ " cols.column_id \n";
9973

100-
static {
101-
EXCLUDED_SCHEMAS.addAll(EXCLUDED_SCHEMAS_ALL);
102-
}
103-
10474
public OracleCatalog(
10575
String catalogName,
10676
String username,
@@ -157,9 +127,6 @@ protected String getListTableSql(String databaseName) {
157127

158128
@Override
159129
protected String getTableName(ResultSet rs) throws SQLException {
160-
if (EXCLUDED_SCHEMAS.contains(rs.getString(1))) {
161-
return null;
162-
}
163130
return rs.getString(1) + "." + rs.getString(2);
164131
}
165132

‎seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/psql/PostgresCatalog.java

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -81,18 +81,6 @@ public class PostgresCatalog extends AbstractJdbcCatalog {
8181
+ "ORDER BY \n"
8282
+ " a.attnum;";
8383

84-
static {
85-
SYS_DATABASES.add("information_schema");
86-
SYS_DATABASES.add("pg_catalog");
87-
SYS_DATABASES.add("root");
88-
SYS_DATABASES.add("pg_toast");
89-
SYS_DATABASES.add("pg_temp_1");
90-
SYS_DATABASES.add("pg_toast_temp_1");
91-
SYS_DATABASES.add("postgres");
92-
SYS_DATABASES.add("template0");
93-
SYS_DATABASES.add("template1");
94-
}
95-
9684
public PostgresCatalog(
9785
String catalogName,
9886
String username,

‎seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/redshift/RedshiftCatalog.java

Lines changed: 0 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -44,23 +44,6 @@ public class RedshiftCatalog extends AbstractJdbcCatalog {
4444
private final String SELECT_COLUMNS =
4545
"SELECT * FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '%s' AND TABLE_NAME ='%s' ORDER BY ordinal_position ASC";
4646

47-
static {
48-
EXCLUDED_SCHEMAS.add("information_schema");
49-
EXCLUDED_SCHEMAS.add("catalog_history");
50-
EXCLUDED_SCHEMAS.add("pg_auto_copy");
51-
EXCLUDED_SCHEMAS.add("pg_automv");
52-
EXCLUDED_SCHEMAS.add("pg_catalog");
53-
EXCLUDED_SCHEMAS.add("pg_internal");
54-
EXCLUDED_SCHEMAS.add("pg_s3");
55-
}
56-
57-
static {
58-
SYS_DATABASES.add("template0");
59-
SYS_DATABASES.add("template1");
60-
SYS_DATABASES.add("awsdatacatalog");
61-
SYS_DATABASES.add("padb_harvest");
62-
}
63-
6447
protected final Map<String, Connection> connectionMap;
6548

6649
public RedshiftCatalog(

‎seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/saphana/SapHanaCatalog.java

Lines changed: 0 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -43,39 +43,6 @@
4343
@Slf4j
4444
public class SapHanaCatalog extends AbstractJdbcCatalog {
4545

46-
static {
47-
SYS_DATABASES.add("SYS");
48-
SYS_DATABASES.add("SYSTEM");
49-
SYS_DATABASES.add("SYS_DATABASES");
50-
SYS_DATABASES.add("_SYS_ADVISOR");
51-
SYS_DATABASES.add("_SYS_AFL");
52-
SYS_DATABASES.add("_SYS_BI");
53-
SYS_DATABASES.add("_SYS_BIC");
54-
SYS_DATABASES.add("_SYS_DATA_ANONYMIZATION");
55-
SYS_DATABASES.add("_SYS_DI");
56-
SYS_DATABASES.add("_SYS_EPM");
57-
SYS_DATABASES.add("_SYS_LDB");
58-
SYS_DATABASES.add("_SYS_PLAN_STABILITY");
59-
SYS_DATABASES.add("_SYS_REPO");
60-
SYS_DATABASES.add("_SYS_RT");
61-
SYS_DATABASES.add("_SYS_SECURITY");
62-
SYS_DATABASES.add("_SYS_SQL_ANALYZER");
63-
SYS_DATABASES.add("_SYS_STATISTICS");
64-
SYS_DATABASES.add("_SYS_TABLE_REPLICAS");
65-
SYS_DATABASES.add("_SYS_TASK");
66-
SYS_DATABASES.add("_SYS_TELEMETRY");
67-
SYS_DATABASES.add("_SYS_XS");
68-
SYS_DATABASES.add("_SYS_DI_CATALOG");
69-
SYS_DATABASES.add("_SYS_EPM_DATA");
70-
SYS_DATABASES.add("_SYS_DI_SU");
71-
SYS_DATABASES.add("_SYS_WORKLOAD_REPLAY");
72-
SYS_DATABASES.add("_SYS_AUDIT");
73-
SYS_DATABASES.add("_SYS_DI_BI_CATALOG");
74-
SYS_DATABASES.add("_SYS_DI_CDS_CATALOG");
75-
SYS_DATABASES.add("_SYS_DI_SEARCH_CATALOG");
76-
SYS_DATABASES.add("_SYS_DI_TO");
77-
}
78-
7946
private static final String SELECT_COLUMNS_SQL_TEMPLATE =
8047
"SELECT\n"
8148
+ " C.COLUMN_NAME,\n"

‎seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/tidb/TiDBCatalog.java

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -22,14 +22,6 @@
2222

2323
public class TiDBCatalog extends MySqlCatalog {
2424

25-
static {
26-
SYS_DATABASES.clear();
27-
SYS_DATABASES.add("information_schema");
28-
SYS_DATABASES.add("mysql");
29-
SYS_DATABASES.add("performance_schema");
30-
SYS_DATABASES.add("metrics_schema");
31-
}
32-
3325
public TiDBCatalog(
3426
String catalogName, String username, String pwd, JdbcUrlUtil.UrlInfo urlInfo) {
3527
super(catalogName, username, pwd, urlInfo);

‎seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/xugu/XuguCatalog.java

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -35,16 +35,11 @@
3535
import java.sql.ResultSet;
3636
import java.sql.SQLException;
3737
import java.util.ArrayList;
38-
import java.util.Arrays;
39-
import java.util.Collections;
4038
import java.util.List;
4139

4240
@Slf4j
4341
public class XuguCatalog extends AbstractJdbcCatalog {
4442

45-
protected static List<String> EXCLUDED_SCHEMAS =
46-
Collections.unmodifiableList(Arrays.asList("GUEST", "SYSAUDITOR", "SYSSSO"));
47-
4843
private static final String SELECT_COLUMNS_SQL_TEMPLATE =
4944
"SELECT\n"
5045
+ " dc.COLUMN_NAME,\n"
@@ -172,9 +167,6 @@ protected String getListTableSql(String databaseName) {
172167

173168
@Override
174169
protected String getTableName(ResultSet rs) throws SQLException {
175-
if (EXCLUDED_SCHEMAS.contains(rs.getString(1))) {
176-
return null;
177-
}
178170
return rs.getString(1) + "." + rs.getString(2);
179171
}
180172

‎seatunnel-connectors-v2/connector-starrocks/src/main/java/org/apache/seatunnel/connectors/seatunnel/starrocks/catalog/StarRocksCatalog.java

Lines changed: 1 addition & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -60,11 +60,9 @@
6060
import java.util.ArrayList;
6161
import java.util.Collections;
6262
import java.util.HashMap;
63-
import java.util.HashSet;
6463
import java.util.List;
6564
import java.util.Map;
6665
import java.util.Optional;
67-
import java.util.Set;
6866
import java.util.stream.IntStream;
6967

7068
import static org.apache.seatunnel.shade.com.google.common.base.Preconditions.checkArgument;
@@ -80,15 +78,8 @@ public class StarRocksCatalog implements Catalog {
8078
protected String defaultUrl;
8179
private final JdbcUrlUtil.UrlInfo urlInfo;
8280
private final String template;
83-
84-
private static final Set<String> SYS_DATABASES = new HashSet<>();
8581
private static final Logger LOG = LoggerFactory.getLogger(StarRocksCatalog.class);
8682

87-
static {
88-
SYS_DATABASES.add("information_schema");
89-
SYS_DATABASES.add("_statistics_");
90-
}
91-
9283
public StarRocksCatalog(
9384
String catalogName, String username, String pwd, String defaultUrl, String template) {
9485

@@ -114,10 +105,7 @@ public List<String> listDatabases() throws CatalogException {
114105
List<String> databases = new ArrayList<>();
115106

116107
while (rs.next()) {
117-
String databaseName = rs.getString(1);
118-
if (!SYS_DATABASES.contains(databaseName)) {
119-
databases.add(rs.getString(1));
120-
}
108+
databases.add(rs.getString(1));
121109
}
122110

123111
return databases;

0 commit comments

Comments
 (0)