Add support of SKIP options for JSON type

git-hulk · git-hulk · commit c9053081840d · 2025-01-07T21:25:14.000+08:00
diff --git a/parser/ast.go b/parser/ast.go
@@ -1371,7 +1371,7 @@ func (a *TableIndex) String() string {
 	builder.WriteString("INDEX")
 	builder.WriteByte(' ')
 	builder.WriteString(a.Name.String())
-	// a.ColumnDef = *Ident --- e.g. INDEX idx column TYPE ...
+	// a.ColumnDef = *Name --- e.g. INDEX idx column TYPE ...
 	// a.ColumnDef = *ParamExprList --- e.g. INDEX idx(column) TYPE ...
 	if _, ok := a.ColumnExpr.Expr.(*Ident); ok {
 		builder.WriteByte(' ')
@@ -3366,6 +3366,104 @@ func (s *ScalarType) Type() string {
 	return s.Name.Name
 }
 
+type JSONPath struct {
+	Idents []*Ident
+}
+
+func (j *JSONPath) String() string {
+	var builder strings.Builder
+	for i, ident := range j.Idents {
+		if i > 0 {
+			builder.WriteString(".")
+		}
+		builder.WriteString(ident.String())
+	}
+	return builder.String()
+}
+
+type JSONOption struct {
+	SkipPath  *JSONPath
+	SkipRegex *StringLiteral
+}
+
+func (j *JSONOption) String() string {
+	var builder strings.Builder
+	if j.SkipPath != nil {
+		builder.WriteString("SKIP ")
+		builder.WriteString(j.SkipPath.String())
+	}
+	if j.SkipRegex != nil {
+		builder.WriteString(" SKIP REGEXP ")
+		builder.WriteString(j.SkipRegex.String())
+	}
+	return builder.String()
+}
+
+type JSONOptions struct {
+	LParen Pos
+	RParen Pos
+	Items  []*JSONOption
+}
+
+func (j *JSONOptions) Pos() Pos {
+	return j.LParen
+}
+
+func (j *JSONOptions) End() Pos {
+	return j.RParen
+}
+
+func (j *JSONOptions) String() string {
+	var builder strings.Builder
+	builder.WriteByte('(')
+	for i, item := range j.Items {
+		if i > 0 {
+			builder.WriteString(", ")
+		}
+		builder.WriteString(item.String())
+	}
+	builder.WriteByte(')')
+	return builder.String()
+}
+
+type JSONType struct {
+	Name    *Ident
+	Options *JSONOptions
+}
+
+func (j *JSONType) Pos() Pos {
+	return j.Name.NamePos
+}
+
+func (j *JSONType) End() Pos {
+	if j.Options != nil {
+		return j.Options.RParen
+	}
+	return j.Name.NameEnd
+}
+
+func (j *JSONType) String() string {
+	var builder strings.Builder
+	builder.WriteString(j.Name.String())
+	if j.Options != nil {
+		builder.WriteString(j.Options.String())
+	}
+	return builder.String()
+}
+
+func (j *JSONType) Type() string {
+	return j.Name.Name
+}
+
+func (j *JSONType) Accept(visitor ASTVisitor) error {
+	visitor.enter(j)
+	defer visitor.leave(j)
+	if err := j.Name.Accept(visitor); err != nil {
+		return err
+	}
+	return visitor.VisitJSONType(j)
+}
+
 type PropertyType struct {
 	Name *Ident
 }
diff --git a/parser/ast_visitor.go b/parser/ast_visitor.go
@@ -75,6 +75,7 @@ type ASTVisitor interface {
 	VisitColumnDef(expr *ColumnDef) error
 	VisitColumnExpr(expr *ColumnExpr) error
 	VisitScalarType(expr *ScalarType) error
+	VisitJSONType(expr *JSONType) error
 	VisitPropertyType(expr *PropertyType) error
 	VisitTypeWithParams(expr *TypeWithParams) error
 	VisitComplexType(expr *ComplexType) error
@@ -693,6 +694,13 @@ func (v *DefaultASTVisitor) VisitScalarType(expr *ScalarType) error {
 	return nil
 }
 
+func (v *DefaultASTVisitor) VisitJSONType(expr *JSONType) error {
+	if v.Visit != nil {
+		return v.Visit(expr)
+	}
+	return nil
+}
+
 func (v *DefaultASTVisitor) VisitPropertyType(expr *PropertyType) error {
 	if v.Visit != nil {
 		return v.Visit(expr)
diff --git a/parser/keyword.go b/parser/keyword.go
@@ -105,6 +105,7 @@ const (
 	KeywordIs           = "IS"
 	KeywordIs_object_id = "IS_OBJECT_ID"
 	KeywordJoin         = "JOIN"
+	KeywordJSON         = "JSON"
 	KeywordKey          = "KEY"
 	KeywordKill         = "KILL"
 	KeywordLast         = "LAST"
@@ -158,6 +159,7 @@ const (
 	KeywordQuota        = "QUOTA"
 	KeywordRange        = "RANGE"
 	KeywordRefresh      = "REFRESH"
+	KeywordRegexp       = "REGEXP"
 	KeywordReload       = "RELOAD"
 	KeywordRemove       = "REMOVE"
 	KeywordRename       = "RENAME"
@@ -180,6 +182,7 @@ const (
 	KeywordSettings     = "SETTINGS"
 	KeywordShow         = "SHOW"
 	KeywordShutdown     = "SHUTDOWN"
+	KeywordSkip         = "SKIP"
 	KeywordSource       = "SOURCE"
 	KeywordStart        = "START"
 	KeywordStop         = "STOP"
@@ -329,6 +332,7 @@ var keywords = NewSet(
 	KeywordIs,
 	KeywordIs_object_id,
 	KeywordJoin,
+	KeywordJSON,
 	KeywordKey,
 	KeywordKill,
 	KeywordLast,
@@ -381,6 +385,8 @@ var keywords = NewSet(
 	KeywordQueues,
 	KeywordQuota,
 	KeywordRange,
+	KeywordRefresh,
+	KeywordRegexp,
 	KeywordReload,
 	KeywordRemove,
 	KeywordRename,
@@ -403,6 +409,7 @@ var keywords = NewSet(
 	KeywordSettings,
 	KeywordShow,
 	KeywordShutdown,
+	KeywordSkip,
 	KeywordSource,
 	KeywordStart,
 	KeywordStop,
diff --git a/parser/lexer_test.go b/parser/lexer_test.go
@@ -144,7 +144,7 @@ func TestConsumeNumber(t *testing.T) {
 		}
 	})
 
-	t.Run("Ident", func(t *testing.T) {
+	t.Run("Name", func(t *testing.T) {
 		idents := []string{
 			"`CASE`",
 			"`TEST`",
diff --git a/parser/parser_column.go b/parser/parser_column.go
@@ -810,10 +810,14 @@ func (p *Parser) parseColumnType(_ Pos) (ColumnType, error) { // nolint:funlen
 	if p.tryConsumeTokenKind(TokenKindLParen) != nil {
 		switch {
 		case p.matchTokenKind(TokenKindIdent):
-			if ident.Name == "Nested" {
+			switch ident.Name {
+			case "Nested":
 				return p.parseNestedType(ident, p.Pos())
+			case "JSON":
+				return p.parseJSONType(ident, p.Pos())
+			default:
+				return p.parseComplexType(ident, p.Pos())
 			}
-			return p.parseComplexType(ident, p.Pos())
 		case p.matchTokenKind(TokenKindString):
 			if peekToken, err := p.lexer.peekToken(); err == nil && peekToken.Kind == TokenKindSingleEQ {
 				// enum values
@@ -920,6 +924,81 @@ func (p *Parser) parseColumnTypeWithParams(name *Ident, pos Pos) (*TypeWithParam
 	}, nil
 }
 
+func (p *Parser) parseJSONPath() (*JSONPath, error) {
+	idents := make([]*Ident, 0)
+	ident, err := p.parseIdent()
+	if err != nil {
+		return nil, err
+	}
+	idents = append(idents, ident)
+
+	for !p.lexer.isEOF() && p.tryConsumeTokenKind(TokenKindDot) != nil {
+		ident, err := p.parseIdent()
+		if err != nil {
+			return nil, err
+		}
+		idents = append(idents, ident)
+	}
+	return &JSONPath{
+		Idents: idents,
+	}, nil
+}
+
+func (p *Parser) parseJSONOption() (*JSONOption, error) {
+	switch {
+	case p.tryConsumeKeyword(KeywordSkip) != nil:
+		if p.tryConsumeKeyword(KeywordRegexp) != nil {
+			regex, err := p.parseString(p.Pos())
+			if err != nil {
+				return nil, err
+			}
+			return &JSONOption{
+				SkipRegex: regex,
+			}, nil
+		}
+		jsonPath, err := p.parseJSONPath()
+		if err != nil {
+			return nil, err
+		}
+		return &JSONOption{
+			SkipPath: jsonPath,
+		}, nil
+	default:
+		return nil, fmt.Errorf("unexpected token kind: %s", p.lastTokenKind())
+	}
+}
+
+func (p *Parser) parseJSONType(name *Ident, pos Pos) (*JSONType, error) {
+	if p.matchTokenKind(TokenKindLParen) {
+		return &JSONType{Name: name}, nil
+	}
+
+	options := make([]*JSONOption, 0)
+	for !p.lexer.isEOF() && !p.matchTokenKind(TokenKindRParen) {
+		option, err := p.parseJSONOption()
+		if err != nil {
+			return nil, err
+		}
+		options = append(options, option)
+		if p.tryConsumeTokenKind(",") == nil {
+			break
+		}
+	}
+
+	rparenPos := p.Pos()
+	if _, err := p.consumeTokenKind(TokenKindRParen); err != nil {
+		return nil, err
+	}
+	return &JSONType{
+		Name: name,
+		Options: &JSONOptions{
+			LParen: pos,
+			RParen: rparenPos,
+			Items:  options,
+		},
+	}, nil
+}
+
 func (p *Parser) parseNestedType(name *Ident, pos Pos) (*NestedType, error) {
 	columns, err := p.parseTableColumns()
 	if err != nil {
diff --git a/parser/parser_table.go b/parser/parser_table.go
@@ -513,7 +513,7 @@ func (p *Parser) parseTableArgExpr(pos Pos) (Expr, error) {
 	case p.matchTokenKind(TokenKindInt), p.matchTokenKind(TokenKindString), p.matchKeyword("NULL"):
 		return p.parseLiteral(p.Pos())
 	default:
-		return nil, fmt.Errorf("unexpected token: %q, expected <Ident>, <literal>", p.last().String)
+		return nil, fmt.Errorf("unexpected token: %q, expected <Name>, <literal>", p.last().String)
 	}
 }
 
diff --git a/parser/testdata/ddl/create_table_basic.sql b/parser/testdata/ddl/create_table_basic.sql
@@ -22,6 +22,7 @@ CREATE TABLE IF NOT EXISTS test.events_local (
     f8 Datetime DEFAULT now(),
     f9 String MATERIALIZED toString(f7['f70']),
     f10 String ALIAS f11,
+    f12 JSON(SKIP a, SKIP a.b.c, SKIP REGEXP 'hello'),
 ) ENGINE = MergeTree
 PRIMARY KEY (f0, f1, f2)
 PARTITION BY toYYYYMMDD(f3)
diff --git a/parser/testdata/ddl/format/create_table_basic.sql b/parser/testdata/ddl/format/create_table_basic.sql
@@ -23,6 +23,7 @@ CREATE TABLE IF NOT EXISTS test.events_local (
     f8 Datetime DEFAULT now(),
     f9 String MATERIALIZED toString(f7['f70']),
     f10 String ALIAS f11,
+    f12 JSON(SKIP a, SKIP a.b.c, SKIP REGEXP 'hello'),
 ) ENGINE = MergeTree
 PRIMARY KEY (f0, f1, f2)
 PARTITION BY toYYYYMMDD(f3)
@@ -31,4 +32,4 @@ ORDER BY (f1,f2,f3)
 COMMENT 'Comment for table';
 
 -- Format SQL:
-CREATE TABLE IF NOT EXISTS test.events_local (f0 String, f1 String CODEC(ZSTD(1)), f2 VARCHAR(255), f3 Datetime, f4 Datetime, f5 Map(String, String), f6 String, f7 Nested(f70 UInt32, f71 UInt32, f72 DateTime, f73 Int64, f74 Int64, f75 String), f8 Datetime DEFAULT now(), f9 String MATERIALIZED toString(f7['f70']), f10 String ALIAS f11) ENGINE = MergeTree PRIMARY KEY (f0, f1, f2) PARTITION BY toYYYYMMDD(f3) TTL f3 + INTERVAL 6 MONTH ORDER BY (f1, f2, f3) COMMENT 'Comment for table';
+CREATE TABLE IF NOT EXISTS test.events_local (f0 String, f1 String CODEC(ZSTD(1)), f2 VARCHAR(255), f3 Datetime, f4 Datetime, f5 Map(String, String), f6 String, f7 Nested(f70 UInt32, f71 UInt32, f72 DateTime, f73 Int64, f74 Int64, f75 String), f8 Datetime DEFAULT now(), f9 String MATERIALIZED toString(f7['f70']), f10 String ALIAS f11, f12 JSON(SKIP a, SKIP a.b.c,  SKIP REGEXP 'hello')) ENGINE = MergeTree PRIMARY KEY (f0, f1, f2) PARTITION BY toYYYYMMDD(f3) TTL f3 + INTERVAL 6 MONTH ORDER BY (f1, f2, f3) COMMENT 'Comment for table';
diff --git a/parser/testdata/ddl/output/create_table_basic.sql.golden.json b/parser/testdata/ddl/output/create_table_basic.sql.golden.json

Original file line number	Diff line number	Diff line change
`@@ -144,7 +144,7 @@ func TestConsumeNumber(t *testing.T) {`
`144`	`144`	`}`
`145`	`145`	`})`
`146`	`146`
`147`		`- t.Run("Ident", func(t *testing.T) {`
	`147`	`+ t.Run("Name", func(t *testing.T) {`
`148`	`148`	`idents := []string{`
`149`	`149`	"`CASE`",
`150`	`150`	"`TEST`",
Original file line number	Diff line number	Diff line change
`@@ -513,7 +513,7 @@ func (p *Parser) parseTableArgExpr(pos Pos) (Expr, error) {`
`513`	`513`	`case p.matchTokenKind(TokenKindInt), p.matchTokenKind(TokenKindString), p.matchKeyword("NULL"):`
`514`	`514`	`return p.parseLiteral(p.Pos())`
`515`	`515`	`default:`
`516`		`- return nil, fmt.Errorf("unexpected token: %q, expected <Ident>, <literal>", p.last().String)`
	`516`	`+ return nil, fmt.Errorf("unexpected token: %q, expected <Name>, <literal>", p.last().String)`
`517`	`517`	`}`
`518`	`518`	`}`
`519`	`519`