Skip to content

Commit 2757feb

Browse files
git-hulkgit-hulk
authored
Add support of SKIP options for JSON type (#117)
1 parent 392b4ce commit 2757feb

9 files changed

+333
-57
lines changed

‎parser/ast.go

Lines changed: 99 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1371,7 +1371,7 @@ func (a *TableIndex) String() string {
13711371
builder.WriteString("INDEX")
13721372
builder.WriteByte(' ')
13731373
builder.WriteString(a.Name.String())
1374-
// a.ColumnDef = *Ident --- e.g. INDEX idx column TYPE ...
1374+
// a.ColumnDef = *Name --- e.g. INDEX idx column TYPE ...
13751375
// a.ColumnDef = *ParamExprList --- e.g. INDEX idx(column) TYPE ...
13761376
if _, ok := a.ColumnExpr.Expr.(*Ident); ok {
13771377
builder.WriteByte(' ')
@@ -3366,6 +3366,104 @@ func (s *ScalarType) Type() string {
33663366
return s.Name.Name
33673367
}
33683368

3369+
type JSONPath struct {
3370+
Idents []*Ident
3371+
}
3372+
3373+
func (j *JSONPath) String() string {
3374+
var builder strings.Builder
3375+
for i, ident := range j.Idents {
3376+
if i > 0 {
3377+
builder.WriteString(".")
3378+
}
3379+
builder.WriteString(ident.String())
3380+
}
3381+
return builder.String()
3382+
}
3383+
3384+
type JSONOption struct {
3385+
SkipPath *JSONPath
3386+
SkipRegex *StringLiteral
3387+
}
3388+
3389+
func (j *JSONOption) String() string {
3390+
var builder strings.Builder
3391+
if j.SkipPath != nil {
3392+
builder.WriteString("SKIP ")
3393+
builder.WriteString(j.SkipPath.String())
3394+
}
3395+
if j.SkipRegex != nil {
3396+
builder.WriteString(" SKIP REGEXP ")
3397+
builder.WriteString(j.SkipRegex.String())
3398+
}
3399+
return builder.String()
3400+
}
3401+
3402+
type JSONOptions struct {
3403+
LParen Pos
3404+
RParen Pos
3405+
Items []*JSONOption
3406+
}
3407+
3408+
func (j *JSONOptions) Pos() Pos {
3409+
return j.LParen
3410+
}
3411+
3412+
func (j *JSONOptions) End() Pos {
3413+
return j.RParen
3414+
}
3415+
3416+
func (j *JSONOptions) String() string {
3417+
var builder strings.Builder
3418+
builder.WriteByte('(')
3419+
for i, item := range j.Items {
3420+
if i > 0 {
3421+
builder.WriteString(", ")
3422+
}
3423+
builder.WriteString(item.String())
3424+
}
3425+
builder.WriteByte(')')
3426+
return builder.String()
3427+
}
3428+
3429+
type JSONType struct {
3430+
Name *Ident
3431+
Options *JSONOptions
3432+
}
3433+
3434+
func (j *JSONType) Pos() Pos {
3435+
return j.Name.NamePos
3436+
}
3437+
3438+
func (j *JSONType) End() Pos {
3439+
if j.Options != nil {
3440+
return j.Options.RParen
3441+
}
3442+
return j.Name.NameEnd
3443+
}
3444+
3445+
func (j *JSONType) String() string {
3446+
var builder strings.Builder
3447+
builder.WriteString(j.Name.String())
3448+
if j.Options != nil {
3449+
builder.WriteString(j.Options.String())
3450+
}
3451+
return builder.String()
3452+
}
3453+
3454+
func (j *JSONType) Type() string {
3455+
return j.Name.Name
3456+
}
3457+
3458+
func (j *JSONType) Accept(visitor ASTVisitor) error {
3459+
visitor.enter(j)
3460+
defer visitor.leave(j)
3461+
if err := j.Name.Accept(visitor); err != nil {
3462+
return err
3463+
}
3464+
return visitor.VisitJSONType(j)
3465+
}
3466+
33693467
type PropertyType struct {
33703468
Name *Ident
33713469
}

‎parser/ast_visitor.go

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ type ASTVisitor interface {
7575
VisitColumnDef(expr *ColumnDef) error
7676
VisitColumnExpr(expr *ColumnExpr) error
7777
VisitScalarType(expr *ScalarType) error
78+
VisitJSONType(expr *JSONType) error
7879
VisitPropertyType(expr *PropertyType) error
7980
VisitTypeWithParams(expr *TypeWithParams) error
8081
VisitComplexType(expr *ComplexType) error
@@ -693,6 +694,13 @@ func (v *DefaultASTVisitor) VisitScalarType(expr *ScalarType) error {
693694
return nil
694695
}
695696

697+
func (v *DefaultASTVisitor) VisitJSONType(expr *JSONType) error {
698+
if v.Visit != nil {
699+
return v.Visit(expr)
700+
}
701+
return nil
702+
}
703+
696704
func (v *DefaultASTVisitor) VisitPropertyType(expr *PropertyType) error {
697705
if v.Visit != nil {
698706
return v.Visit(expr)

‎parser/keyword.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,7 @@ const (
105105
KeywordIs = "IS"
106106
KeywordIs_object_id = "IS_OBJECT_ID"
107107
KeywordJoin = "JOIN"
108+
KeywordJSON = "JSON"
108109
KeywordKey = "KEY"
109110
KeywordKill = "KILL"
110111
KeywordLast = "LAST"
@@ -158,6 +159,7 @@ const (
158159
KeywordQuota = "QUOTA"
159160
KeywordRange = "RANGE"
160161
KeywordRefresh = "REFRESH"
162+
KeywordRegexp = "REGEXP"
161163
KeywordReload = "RELOAD"
162164
KeywordRemove = "REMOVE"
163165
KeywordRename = "RENAME"
@@ -180,6 +182,7 @@ const (
180182
KeywordSettings = "SETTINGS"
181183
KeywordShow = "SHOW"
182184
KeywordShutdown = "SHUTDOWN"
185+
KeywordSkip = "SKIP"
183186
KeywordSource = "SOURCE"
184187
KeywordStart = "START"
185188
KeywordStop = "STOP"
@@ -329,6 +332,7 @@ var keywords = NewSet(
329332
KeywordIs,
330333
KeywordIs_object_id,
331334
KeywordJoin,
335+
KeywordJSON,
332336
KeywordKey,
333337
KeywordKill,
334338
KeywordLast,
@@ -381,6 +385,8 @@ var keywords = NewSet(
381385
KeywordQueues,
382386
KeywordQuota,
383387
KeywordRange,
388+
KeywordRefresh,
389+
KeywordRegexp,
384390
KeywordReload,
385391
KeywordRemove,
386392
KeywordRename,
@@ -403,6 +409,7 @@ var keywords = NewSet(
403409
KeywordSettings,
404410
KeywordShow,
405411
KeywordShutdown,
412+
KeywordSkip,
406413
KeywordSource,
407414
KeywordStart,
408415
KeywordStop,

‎parser/lexer_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -144,7 +144,7 @@ func TestConsumeNumber(t *testing.T) {
144144
}
145145
})
146146

147-
t.Run("Ident", func(t *testing.T) {
147+
t.Run("Name", func(t *testing.T) {
148148
idents := []string{
149149
"`CASE`",
150150
"`TEST`",

‎parser/parser_column.go

Lines changed: 81 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -810,10 +810,14 @@ func (p *Parser) parseColumnType(_ Pos) (ColumnType, error) { // nolint:funlen
810810
if p.tryConsumeTokenKind(TokenKindLParen) != nil {
811811
switch {
812812
case p.matchTokenKind(TokenKindIdent):
813-
if ident.Name == "Nested" {
813+
switch ident.Name {
814+
case "Nested":
814815
return p.parseNestedType(ident, p.Pos())
816+
case "JSON":
817+
return p.parseJSONType(ident, p.Pos())
818+
default:
819+
return p.parseComplexType(ident, p.Pos())
815820
}
816-
return p.parseComplexType(ident, p.Pos())
817821
case p.matchTokenKind(TokenKindString):
818822
if peekToken, err := p.lexer.peekToken(); err == nil && peekToken.Kind == TokenKindSingleEQ {
819823
// enum values
@@ -920,6 +924,81 @@ func (p *Parser) parseColumnTypeWithParams(name *Ident, pos Pos) (*TypeWithParam
920924
}, nil
921925
}
922926

927+
func (p *Parser) parseJSONPath() (*JSONPath, error) {
928+
idents := make([]*Ident, 0)
929+
ident, err := p.parseIdent()
930+
if err != nil {
931+
return nil, err
932+
}
933+
idents = append(idents, ident)
934+
935+
for !p.lexer.isEOF() && p.tryConsumeTokenKind(TokenKindDot) != nil {
936+
ident, err := p.parseIdent()
937+
if err != nil {
938+
return nil, err
939+
}
940+
idents = append(idents, ident)
941+
}
942+
return &JSONPath{
943+
Idents: idents,
944+
}, nil
945+
}
946+
947+
func (p *Parser) parseJSONOption() (*JSONOption, error) {
948+
switch {
949+
case p.tryConsumeKeyword(KeywordSkip) != nil:
950+
if p.tryConsumeKeyword(KeywordRegexp) != nil {
951+
regex, err := p.parseString(p.Pos())
952+
if err != nil {
953+
return nil, err
954+
}
955+
return &JSONOption{
956+
SkipRegex: regex,
957+
}, nil
958+
}
959+
jsonPath, err := p.parseJSONPath()
960+
if err != nil {
961+
return nil, err
962+
}
963+
return &JSONOption{
964+
SkipPath: jsonPath,
965+
}, nil
966+
default:
967+
return nil, fmt.Errorf("unexpected token kind: %s", p.lastTokenKind())
968+
}
969+
}
970+
971+
func (p *Parser) parseJSONType(name *Ident, pos Pos) (*JSONType, error) {
972+
if p.matchTokenKind(TokenKindLParen) {
973+
return &JSONType{Name: name}, nil
974+
}
975+
976+
options := make([]*JSONOption, 0)
977+
for !p.lexer.isEOF() && !p.matchTokenKind(TokenKindRParen) {
978+
option, err := p.parseJSONOption()
979+
if err != nil {
980+
return nil, err
981+
}
982+
options = append(options, option)
983+
if p.tryConsumeTokenKind(",") == nil {
984+
break
985+
}
986+
}
987+
988+
rparenPos := p.Pos()
989+
if _, err := p.consumeTokenKind(TokenKindRParen); err != nil {
990+
return nil, err
991+
}
992+
return &JSONType{
993+
Name: name,
994+
Options: &JSONOptions{
995+
LParen: pos,
996+
RParen: rparenPos,
997+
Items: options,
998+
},
999+
}, nil
1000+
}
1001+
9231002
func (p *Parser) parseNestedType(name *Ident, pos Pos) (*NestedType, error) {
9241003
columns, err := p.parseTableColumns()
9251004
if err != nil {

‎parser/parser_table.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -513,7 +513,7 @@ func (p *Parser) parseTableArgExpr(pos Pos) (Expr, error) {
513513
case p.matchTokenKind(TokenKindInt), p.matchTokenKind(TokenKindString), p.matchKeyword("NULL"):
514514
return p.parseLiteral(p.Pos())
515515
default:
516-
return nil, fmt.Errorf("unexpected token: %q, expected <Ident>, <literal>", p.last().String)
516+
return nil, fmt.Errorf("unexpected token: %q, expected <Name>, <literal>", p.last().String)
517517
}
518518
}
519519

‎parser/testdata/ddl/create_table_basic.sql

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ CREATE TABLE IF NOT EXISTS test.events_local (
2222
f8 Datetime DEFAULT now(),
2323
f9 String MATERIALIZED toString(f7['f70']),
2424
f10 String ALIAS f11,
25+
f12 JSON(SKIP a, SKIP a.b.c, SKIP REGEXP 'hello'),
2526
) ENGINE = MergeTree
2627
PRIMARY KEY (f0, f1, f2)
2728
PARTITION BY toYYYYMMDD(f3)

‎parser/testdata/ddl/format/create_table_basic.sql

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ CREATE TABLE IF NOT EXISTS test.events_local (
2323
f8 Datetime DEFAULT now(),
2424
f9 String MATERIALIZED toString(f7['f70']),
2525
f10 String ALIAS f11,
26+
f12 JSON(SKIP a, SKIP a.b.c, SKIP REGEXP 'hello'),
2627
) ENGINE = MergeTree
2728
PRIMARY KEY (f0, f1, f2)
2829
PARTITION BY toYYYYMMDD(f3)
@@ -31,4 +32,4 @@ ORDER BY (f1,f2,f3)
3132
COMMENT 'Comment for table';
3233

3334
-- Format SQL:
34-
CREATE TABLE IF NOT EXISTS test.events_local (f0 String, f1 String CODEC(ZSTD(1)), f2 VARCHAR(255), f3 Datetime, f4 Datetime, f5 Map(String, String), f6 String, f7 Nested(f70 UInt32, f71 UInt32, f72 DateTime, f73 Int64, f74 Int64, f75 String), f8 Datetime DEFAULT now(), f9 String MATERIALIZED toString(f7['f70']), f10 String ALIAS f11) ENGINE = MergeTree PRIMARY KEY (f0, f1, f2) PARTITION BY toYYYYMMDD(f3) TTL f3 + INTERVAL 6 MONTH ORDER BY (f1, f2, f3) COMMENT 'Comment for table';
35+
CREATE TABLE IF NOT EXISTS test.events_local (f0 String, f1 String CODEC(ZSTD(1)), f2 VARCHAR(255), f3 Datetime, f4 Datetime, f5 Map(String, String), f6 String, f7 Nested(f70 UInt32, f71 UInt32, f72 DateTime, f73 Int64, f74 Int64, f75 String), f8 Datetime DEFAULT now(), f9 String MATERIALIZED toString(f7['f70']), f10 String ALIAS f11, f12 JSON(SKIP a, SKIP a.b.c, SKIP REGEXP 'hello')) ENGINE = MergeTree PRIMARY KEY (f0, f1, f2) PARTITION BY toYYYYMMDD(f3) TTL f3 + INTERVAL 6 MONTH ORDER BY (f1, f2, f3) COMMENT 'Comment for table';

0 commit comments

Comments
 (0)