Skip to content

Commit 8f8d942

Browse files
ruiqianggao1985ruiqianggao1985
committed
Merge branch 'dev' of https://github.com/sqlparser/gsp_demo_java into dev
2 parents 93f6c76 + 198cb05 commit 8f8d942

File tree

13 files changed

+299
-31
lines changed

13 files changed

+299
-31
lines changed

‎src/main/java/demos/dlineage/DataFlowAnalyzer.java‎

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -29,12 +29,13 @@ public class DataFlowAnalyzer {
2929
public static void main(String[] args) {
3030
if (args.length < 1) {
3131
System.out.println(
32-
"Usage: java DataFlowAnalyzer [/f <path_to_sql_file>] [/d <path_to_directory_includes_sql_files>] [/stat] [/s [/topselectlist] [/text] ] [/i] [/ic] [/lof] [/j] [/json] [/traceView] [/t <database type>] [/o <output file path>] [/version] [/env <path_to_metadata.json>] [/tableLineage [/csv [/delimeter <delimeter>]]] [/transform [/coor]] [/showConstant] [/treatArgumentsInCountFunctionAsDirectDataflow]");
32+
"Usage: java DataFlowAnalyzer [/f <path_to_sql_file>] [/d <path_to_directory_includes_sql_files>] [/stat] [/s [/topselectlist] [/text] [/withTemporaryTable]] [/i] [/ic] [/lof] [/j] [/json] [/traceView] [/t <database type>] [/o <output file path>] [/version] [/env <path_to_metadata.json>] [/tableLineage [/csv [/delimeter <delimeter>]]] [/transform [/coor]] [/showConstant] [/treatArgumentsInCountFunctionAsDirectDataflow]");
3333
System.out.println("/f: Optional, the full path to SQL file.");
3434
System.out.println("/d: Optional, the full path to the directory includes the SQL files.");
3535
System.out.println("/j: Optional, return the result including the join relation.");
3636
System.out.println("/s: Optional, simple output, ignore the intermediate results.");
3737
System.out.println("/topselectlist: Optional, simple output with top select results.");
38+
System.out.println("/withTemporaryTable: Optional, simple output with the temporary tables.");
3839
System.out.println(
3940
"/i: Optional, the same as /s option, but will keep the resultset generated by the SQL function, this parameter will have the same effect as /s /topselectlist + keep resultset generated by the sql function.");
4041
System.out.println(
@@ -130,6 +131,7 @@ else if (argList.indexOf("/fromdb") != -1 && argList.size() > argList.indexOf("/
130131
}
131132

132133
boolean simple = argList.indexOf("/s") != -1;
134+
boolean ignoreTemporaryTable = argList.indexOf("/withTemporaryTable") == -1;
133135
boolean ignoreResultSets = argList.indexOf("/i") != -1;
134136
boolean showJoin = argList.indexOf("/j") != -1;
135137
boolean transform = argList.indexOf("/transform") != -1;
@@ -198,12 +200,6 @@ else if (argList.indexOf("/fromdb") != -1 && argList.size() > argList.indexOf("/
198200
sqlFiles, vendor, simple);
199201

200202
if (sqlenv != null) {
201-
if (argList.indexOf("/defaultDatabase") != -1) {
202-
sqlenv.setDefaultCatalogName(args[argList.indexOf("/defaultDatabase") + 1]);
203-
}
204-
if (argList.indexOf("/defaultSchema") != -1) {
205-
sqlenv.setDefaultSchemaName(args[argList.indexOf("/defaultSchema") + 1]);
206-
}
207203
dlineage.setSqlEnv(sqlenv);
208204
}
209205

@@ -215,13 +211,21 @@ else if (argList.indexOf("/fromdb") != -1 && argList.size() > argList.indexOf("/
215211
dlineage.setIgnoreCoordinate(ignoreCoordinate);
216212
dlineage.setSimpleShowTopSelectResultSet(topselectlist);
217213
dlineage.setShowImplicitSchema(showImplicitSchema);
214+
dlineage.setIgnoreTemporaryTable(ignoreTemporaryTable);
218215
if(simple) {
219216
dlineage.setShowCallRelation(true);
220217
}
221218

222219
dlineage.setShowConstantTable(argList.indexOf("/showConstant")!=-1);
223220
dlineage.setShowCountTableColumn(argList.indexOf("/treatArgumentsInCountFunctionAsDirectDataflow")!=-1);
224-
221+
222+
if (argList.indexOf("/defaultDatabase") != -1) {
223+
dlineage.getOption().setDefaultDatabase(args[argList.indexOf("/defaultDatabase") + 1]);
224+
}
225+
if (argList.indexOf("/defaultSchema") != -1) {
226+
dlineage.getOption().setDefaultSchema(args[argList.indexOf("/defaultSchema") + 1]);
227+
}
228+
225229

226230
if (simple && !jsonFormat) {
227231
dlineage.setTextFormat(textFormat);

‎src/main/java/demos/visitors/xmlVisitor.java‎

Lines changed: 134 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
import gudusoft.gsqlparser.*;
55
import gudusoft.gsqlparser.nodes.*;
6+
import gudusoft.gsqlparser.nodes.bigquery.TReplaceExprAsIdentifier;
67
import gudusoft.gsqlparser.nodes.hive.THiveTablePartition;
78
import gudusoft.gsqlparser.nodes.mdx.EMdxExpSyntax;
89
import gudusoft.gsqlparser.nodes.mdx.IMdxIdentifierSegment;
@@ -28,6 +29,7 @@
2829
import gudusoft.gsqlparser.nodes.mssql.TForXMLClause;
2930
import gudusoft.gsqlparser.nodes.mssql.TXMLCommonDirective;
3031
import gudusoft.gsqlparser.nodes.oracle.TTableProperties;
32+
import gudusoft.gsqlparser.nodes.postgresql.TPartitionBoundSpecSqlNode;
3133
import gudusoft.gsqlparser.nodes.teradata.*;
3234
import gudusoft.gsqlparser.stmt.*;
3335
import gudusoft.gsqlparser.stmt.databricks.TCreateExternalLocationStmt;
@@ -803,6 +805,19 @@ public void postVisit(TResultColumnList node) {
803805
appendEndTag(node);
804806
}
805807

808+
809+
public void preVisit(TReplaceExprAsIdentifier node) {
810+
// appendStartTag(node);
811+
e_parent = (Element) elementStack.peek();
812+
Element e_expr_as_ident = xmldoc.createElement("expr_as_identifier");
813+
e_parent.appendChild(e_expr_as_ident);
814+
elementStack.push(e_expr_as_ident);
815+
node.getExpr().accept(this);
816+
node.getIdentifier().accept(this);
817+
elementStack.pop();
818+
819+
}
820+
806821
public void preVisit(TResultColumn node) {
807822
// appendStartTag(node);
808823
e_parent = (Element) elementStack.peek();
@@ -818,6 +833,16 @@ public void preVisit(TResultColumn node) {
818833
if (node.getExceptColumnList() != null){
819834
addElementOfNode("except_columns",node.getExceptColumnList());
820835
}
836+
837+
if (node.getReplaceExprAsIdentifiers() != null){
838+
Element e_replace_clause = xmldoc.createElement("replace_clause");
839+
e_result_column.appendChild(e_replace_clause);
840+
elementStack.push(e_replace_clause);
841+
for(int i=0;i<node.getReplaceExprAsIdentifiers().size();i++){
842+
node.getReplaceExprAsIdentifiers().get(i).accept(this);
843+
}
844+
elementStack.pop();
845+
}
821846
elementStack.pop();
822847
}
823848

@@ -1980,23 +2005,26 @@ public void preVisit(TObjectName node) {
19802005
e_object_name.appendChild(e_server);
19812006
e_server.setTextContent(node.getServerToken().toString());
19822007
}
1983-
if ((node.getDatabaseToken() != null) || (node.getImplictDatabaseString() != null)) {
2008+
2009+
Boolean showImplicitDBOrSchema = false;
2010+
2011+
if ((node.getDatabaseToken() != null) || (showImplicitDBOrSchema && (node.getImplictDatabaseString() != null))) {
19842012
Element e_database = xmldoc.createElement("database_name");
19852013
e_object_name.appendChild(e_database);
19862014
if (node.getDatabaseToken() != null) {
19872015
e_database.setTextContent(node.getDatabaseToken().toString());
19882016
} else {
1989-
e_database.setTextContent(node.getImplictDatabaseString());
2017+
e_database.setTextContent(node.getImplictDatabaseString());
19902018
}
19912019

19922020
}
1993-
if (((node.getSchemaToken() != null) && (!node.isImplicitSchema())) || (node.getImplictSchemaString() != null)) {
2021+
if (((node.getSchemaToken() != null) && (!node.isImplicitSchema())) || ( showImplicitDBOrSchema && (node.getImplictSchemaString() != null))) {
19942022
Element e_schema = xmldoc.createElement("schema_name");
19952023
e_object_name.appendChild(e_schema);
19962024
if ((node.getSchemaToken() != null) && (!node.isImplicitSchema())) {
19972025
e_schema.setTextContent(node.getSchemaToken().toString());
19982026
} else {
1999-
e_schema.setTextContent(node.getImplictSchemaString());
2027+
e_schema.setTextContent(node.getImplictSchemaString());
20002028
}
20012029
}
20022030
if (node.getObjectToken() != null) {
@@ -2579,6 +2607,25 @@ public void preVisit(TMssqlCreateFunction node) {
25792607

25802608
}
25812609

2610+
2611+
public void preVisit(TUnloadStmt stmt) {
2612+
e_parent = (Element) elementStack.peek();
2613+
Element e_unload_stmt = xmldoc.createElement("unload_stmt");
2614+
e_parent.appendChild(e_unload_stmt);
2615+
elementStack.push(e_unload_stmt);
2616+
if (stmt.getSelectSqlStatement() != null){
2617+
stmt.getSelectSqlStatement().accept(this);
2618+
}
2619+
2620+
if (stmt.getS3() != null){
2621+
addElementOfString("s3_path",stmt.getS3());
2622+
}
2623+
2624+
2625+
elementStack.pop();
2626+
2627+
}
2628+
25822629
public void preVisit(TCreateFunctionStmt node) {
25832630
e_parent = (Element) elementStack.peek();
25842631
Element e_function = xmldoc.createElement("create_function_statement");
@@ -2606,7 +2653,10 @@ public void preVisit(TCreateFunctionStmt node) {
26062653
node.getParameterDeclarations().accept(this);
26072654
}
26082655

2609-
if (node.getBodyStatements().size() > 0) {
2656+
if (node.getBlockBody() != null){
2657+
node.getBlockBody().accept(this);
2658+
}
2659+
else if (node.getBodyStatements().size() > 0) {
26102660
current_statement_list_tag = "body_statement_list";
26112661
node.getBodyStatements().accept(this);
26122662
}
@@ -2885,6 +2935,9 @@ public void preVisit(TAlterTableOption node) {
28852935
e_alter_table_option.appendChild(e_option);
28862936
elementStack.push(e_option);
28872937
node.getColumnName().accept(this);
2938+
if (node.getNewDataType() != null){
2939+
node.getNewDataType().accept(this);
2940+
}
28882941
elementStack.pop();
28892942
break;
28902943
case ChangeColumn:
@@ -2953,6 +3006,10 @@ public void preVisit(TAlterTableOption node) {
29533006
node.getPartitionSpecList().get(i).accept(this);
29543007
}
29553008
break;
3009+
case attachPartition:
3010+
node.getPartitionName().accept(this);
3011+
node.getPartitionBoundSpec().accept(this);
3012+
break;
29563013
default:
29573014
e_option = xmldoc.createElement("not_implemented_option");
29583015
e_alter_table_option.appendChild(e_option);
@@ -3544,10 +3601,45 @@ public void preVisit(TCreateTriggerStmt stmt) {
35443601
e_parent.appendChild(e_create_trigger);
35453602
elementStack.push(e_create_trigger);
35463603
addElementOfNode("trigger_name", stmt.getTriggerName());
3604+
stmt.getTriggeringClause().accept(this);
35473605
stmt.getBodyStatements().accept(this);
3606+
if (stmt.getFunctionCall() != null){
3607+
addElementOfNode("execute_function",stmt.getFunctionCall());
3608+
}
3609+
elementStack.pop();
3610+
}
3611+
3612+
public void preVisit(TDmlEventItem node) {
3613+
e_parent = (Element) elementStack.peek();
3614+
Element e_trigger_dml_event = xmldoc.createElement("dml_event");
3615+
e_parent.appendChild(e_trigger_dml_event);
3616+
elementStack.push(e_trigger_dml_event);
3617+
e_trigger_dml_event.setAttribute("event_type",node.getDmlType().toString());
3618+
if (node.getColumnList() != null){
3619+
node.getColumnList().accept(this);
3620+
}
35483621
elementStack.pop();
35493622
}
35503623

3624+
public void preVisit(TSimpleDmlTriggerClause node) {
3625+
e_parent = (Element) elementStack.peek();
3626+
Element e_create_trigger_clause = xmldoc.createElement("simple_dml_trigger_clause");
3627+
e_parent.appendChild(e_create_trigger_clause);
3628+
elementStack.push(e_create_trigger_clause);
3629+
e_create_trigger_clause.setAttribute("granularity",node.getGranularity().toString());
3630+
if (node.getEventClause() instanceof TDmlEventClause){
3631+
TDmlEventClause dmlEventClause = (TDmlEventClause)node.getEventClause();
3632+
e_create_trigger_clause.setAttribute("source_table",dmlEventClause.getTableName().toString());
3633+
for(int i=0;i<dmlEventClause.getEventItems().size();i++){
3634+
TDmlEventItem dmlEventItem = (TDmlEventItem)dmlEventClause.getEventItems().get(i);
3635+
dmlEventItem.accept(this);
3636+
}
3637+
}
3638+
elementStack.pop();
3639+
}
3640+
3641+
3642+
35513643
public void preVisit(TCreateVariableStmt stmt) {
35523644
e_parent = (Element) elementStack.peek();
35533645
Element e_create_variable = xmldoc.createElement("create_variable_statement");
@@ -3855,7 +3947,6 @@ public void preVisit(TOutputClause node) {
38553947
node.getSelectItemList().getElement(i).accept(this);
38563948
}
38573949
elementStack.pop();
3858-
38593950
}
38603951

38613952
public void preVisit(TCreateIndexSqlStatement stmt) {
@@ -3888,6 +3979,29 @@ public void preVisit(TCreateIndexSqlStatement stmt) {
38883979
elementStack.pop();
38893980
}
38903981

3982+
if (stmt.getWhereCondition() != null){
3983+
stmt.getWhereCondition().accept(this);
3984+
}
3985+
3986+
elementStack.pop();
3987+
}
3988+
3989+
public void preVisit(TPartitionBoundSpecSqlNode node) {
3990+
e_parent = (Element) elementStack.peek();
3991+
Element e_partition_bound_spec = xmldoc.createElement("partition_bound_spec");
3992+
e_partition_bound_spec.setAttribute("type",node.getSpecType().toString());
3993+
e_parent.appendChild(e_partition_bound_spec);
3994+
elementStack.push(e_partition_bound_spec);
3995+
switch (node.getSpecType()){
3996+
case typeIn:
3997+
node.getPartition_bound_expr_list().accept(this);
3998+
break;
3999+
case typeFromTo:
4000+
node.getPartition_bound_expr_list_from().accept(this);
4001+
node.getPartition_bound_expr_list_to().accept(this);
4002+
break;
4003+
}
4004+
38914005
elementStack.pop();
38924006
}
38934007

@@ -3938,6 +4052,10 @@ public void preVisit(TCreateTableSqlStatement stmt) {
39384052
case copy:
39394053
addElementOfNode("copy_table",stmt.getCloneSourceTable());
39404054
break;
4055+
case partitionOf:
4056+
addElementOfNode("parent_table",stmt.getSuperTableName());
4057+
stmt.getPartitionBoundSpec().accept(this);
4058+
break;
39414059
}
39424060

39434061
if (stmt.getTableOptions() != null) {
@@ -5599,12 +5717,18 @@ public void preVisit( TRepeatStmt stmt ) {
55995717

56005718
public void preVisit( TBlockSqlNode node ) {
56015719
e_parent = (Element) elementStack.peek( );
5602-
Element e_begin_end = xmldoc.createElement( "plsql_block" );
5720+
Element e_plsql_block = xmldoc.createElement( "plsql_block" );
56035721
if (node.getLabelName() != null){
5604-
e_begin_end.setAttribute("label",node.getLabelNameStr());
5722+
e_plsql_block.setAttribute("label",node.getLabelNameStr());
5723+
}
5724+
if (node.getParent() != null){
5725+
if (node.getParent() instanceof TStoredProcedureSqlStatement){
5726+
TStoredProcedureSqlStatement p = (TStoredProcedureSqlStatement)node.getParent();
5727+
e_plsql_block.setAttribute("parent_name",p.getStoredProcedureName().toString());
5728+
}
56055729
}
5606-
e_parent.appendChild( e_begin_end );
5607-
elementStack.push( e_begin_end );
5730+
e_parent.appendChild( e_plsql_block );
5731+
elementStack.push( e_plsql_block );
56085732
if (node.getDeclareStatements().size() > 0){
56095733
current_statement_list_tag = "declare_statement_list";
56105734
node.getDeclareStatements().accept(this);

‎src/test/java/common/testAsCanonical.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,4 +93,22 @@ public void testReplaceFdecrypt2(){
9393
assertTrue(sqlparser.sqlstatements.get(0).toString().trim().equalsIgnoreCase(sqlparser.sqltext.trim()));
9494
}
9595

96+
public void testReplaceFdecrypt3(){
97+
TGSqlParser sqlparser = new TGSqlParser(EDbVendor.dbvredshift);
98+
sqlparser.sqltext = "select business_partner_type,bp_cid, f_decrypt(name,z9rmt42l85qkf8v3dec23tep) as name , \n" +
99+
"name as encrypted_name, language_name,source_system \n" +
100+
"from public.mio03_business_partner where bp_cid = 'placeholder_str';";
101+
TBaseType.as_canonical_f_decrypt_replace_password = true;
102+
TBaseType.clearCryptFunctions();
103+
TBaseType.addToCryptFunctions("f_decrypt",2);
104+
105+
assertTrue(sqlparser.parse() == 0);
106+
// System.out.println(sqlparser.sqlstatements.get(0).asCanonical().trim());
107+
108+
assertTrue (sqlparser.sqlstatements.get(0).asCanonical().trim().equalsIgnoreCase("select business_partner_type,bp_cid, f_decrypt(name,'***') as name , \n" +
109+
"name as encrypted_name, language_name,source_system \n" +
110+
"from public.mio03_business_partner where bp_cid = 'placeholder_str'"));
111+
}
112+
113+
96114
}

‎src/test/java/databricks/testDatatypeCast.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ public void test1(){
1515
TSelectSqlStatement selectSqlStatement = (TSelectSqlStatement) sqlparser.sqlstatements.get (0);
1616
TExpression expression = selectSqlStatement.getResultColumnList().getResultColumn(0).getExpr();
1717
assertTrue(expression.getExpressionType() == EExpressionType.typecast_datatype_t);
18-
assertTrue(expression.getCastDatatype() == EDataType.bigint_t);
18+
assertTrue(expression.getTypeName().getDataType() == EDataType.bigint_t);
1919
assertTrue(expression.getLeftOperand().toString().equalsIgnoreCase("current_timestamp"));
2020
//System.out.println(expression.getLeftOperand().getExpressionType());
2121
}

0 commit comments

Comments
 (0)