Skip to content

Commit 37ddaa3

Browse files
yoffyoff
authored
Merge pull request #18702 from github/tausbn/python-allow-comments-in-subscripts
Python: Allow comments in subscripts
2 parents 381cc20 + 3d25cd3 commit 37ddaa3

File tree

7 files changed

+27998
-29379
lines changed

7 files changed

+27998
-29379
lines changed

Diff for: ‎python/extractor/tests/parser/subscripts.py

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
a[b]
2+
3+
c[d,e]
4+
5+
c1[d1,]
6+
7+
# And now with many comments
8+
9+
e[
10+
# comment1
11+
f
12+
# comment2
13+
]
14+
15+
g[
16+
# comment3
17+
h,
18+
# comment4
19+
i
20+
# comment5
21+
]

Diff for: ‎python/extractor/tsg-python/python.tsg

+7-53
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
(assignment !type) @assign
2323
{ let @assign.node = (ast-node @assign "Assign") }
2424

25-
[ (expression_list) (tuple) (tuple_pattern) (pattern_list) ] @tuple
25+
[ (expression_list) (tuple) (tuple_pattern) (pattern_list) (index_expression_list) ] @tuple
2626
{ let @tuple.node = (ast-node @tuple "Tuple") }
2727

2828
(list_pattern) @list
@@ -2543,66 +2543,16 @@
25432543

25442544
(subscript
25452545
value: (_) @value
2546+
subscript: (_) @index
25462547
) @subscript
25472548
{
25482549
attr (@subscript.node) value = @value.node
25492550
attr (@value.node) ctx = "load"
2550-
}
25512551

2552-
; Single subscript
2553-
(subscript
2554-
value: (_)
2555-
.
2556-
subscript: (_) @index
2557-
.
2558-
) @subscript
2559-
{
25602552
attr (@subscript.node) index = @index.node
25612553
attr (@index.node) ctx = "load"
25622554
}
25632555

2564-
; For expressions of the form `a[b, c]` we must explicitly synthesize an internal tuple node
2565-
; We do this and also hook it up:
2566-
(subscript
2567-
value: (_)
2568-
.
2569-
subscript: (_) @first
2570-
.
2571-
subscript: (_)
2572-
) @subscript
2573-
{
2574-
let @subscript.tuple = (ast-node @first "Tuple")
2575-
attr (@subscript.tuple) ctx = "load"
2576-
attr (@subscript.node) index = @subscript.tuple
2577-
edge @subscript.tuple -> @first.node
2578-
attr (@subscript.tuple -> @first.node) elts = (named-child-index @first)
2579-
attr (@first.node) ctx = "load"
2580-
}
2581-
2582-
(subscript
2583-
value: (_)
2584-
.
2585-
subscript: (_)
2586-
subscript: (_) @elt
2587-
) @subscript
2588-
{
2589-
edge @subscript.tuple -> @elt.node
2590-
attr (@subscript.tuple -> @elt.node) elts = (named-child-index @elt)
2591-
attr (@elt.node) ctx = "load"
2592-
}
2593-
2594-
2595-
; Set the end position correctly
2596-
(subscript
2597-
value: (_)
2598-
.
2599-
subscript: (_)
2600-
subscript: (_) @last
2601-
.
2602-
) @subscript
2603-
{
2604-
attr (@subscript.tuple) _location_end = (location-end @last)
2605-
}
26062556

26072557

26082558

@@ -3448,9 +3398,12 @@
34483398
; Left hand side of an assignment such as `[foo, bar] = ...`
34493399
(list_pattern element: (_) @elt) @parent
34503400

3451-
; An unadorned tuple (such as in `x = y, z`)
3401+
; An unadorned tuple such as in `x = y, z`
34523402
(expression_list element: (_) @elt) @parent
34533403

3404+
; An index containing multiple indices such as in `x[y, z]`
3405+
(index_expression_list element: (_) @elt) @parent
3406+
34543407
; A regular tuple such as `(x, y, z)`
34553408
(tuple element: (_) @elt) @parent
34563409

@@ -3486,6 +3439,7 @@
34863439
(pattern_list element: (_) @elt)
34873440
(list_pattern element: (_) @elt)
34883441
(expression_list element: (_) @elt)
3442+
(index_expression_list element: (_) @elt)
34893443
(parenthesized_expression inner: (_) @elt)
34903444
(set element: (_) @elt)
34913445
(match_sequence_pattern (_) @elt)

Diff for: ‎python/extractor/tsg-python/tsp/grammar.js

+9-2
Original file line numberDiff line numberDiff line change
@@ -929,11 +929,18 @@ module.exports = grammar({
929929
field('attribute', $.identifier)
930930
)),
931931

932+
_index_expression: $ => choice(
933+
$.list_splat,
934+
$.expression,
935+
$.slice
936+
),
937+
938+
index_expression_list: $ => open_sequence(field('element', $._index_expression)),
939+
932940
subscript: $ => prec(PREC.call, seq(
933941
field('value', $.primary_expression),
934942
'[',
935-
commaSep1(field('subscript', choice($.list_splat, $.expression, $.slice))),
936-
optional(','),
943+
field('subscript', choice($._index_expression, $.index_expression_list)),
937944
']'
938945
)),
939946

Diff for: ‎python/extractor/tsg-python/tsp/src/grammar.json

+93-67
Original file line numberDiff line numberDiff line change
@@ -5045,94 +5045,120 @@
50455045
]
50465046
}
50475047
},
5048-
"subscript": {
5049-
"type": "PREC",
5050-
"value": 21,
5048+
"_index_expression": {
5049+
"type": "CHOICE",
5050+
"members": [
5051+
{
5052+
"type": "SYMBOL",
5053+
"name": "list_splat"
5054+
},
5055+
{
5056+
"type": "SYMBOL",
5057+
"name": "expression"
5058+
},
5059+
{
5060+
"type": "SYMBOL",
5061+
"name": "slice"
5062+
}
5063+
]
5064+
},
5065+
"index_expression_list": {
5066+
"type": "PREC_RIGHT",
5067+
"value": 0,
50515068
"content": {
50525069
"type": "SEQ",
50535070
"members": [
50545071
{
50555072
"type": "FIELD",
5056-
"name": "value",
5073+
"name": "element",
50575074
"content": {
50585075
"type": "SYMBOL",
5059-
"name": "primary_expression"
5076+
"name": "_index_expression"
50605077
}
50615078
},
50625079
{
5063-
"type": "STRING",
5064-
"value": "["
5065-
},
5066-
{
5067-
"type": "SEQ",
5080+
"type": "CHOICE",
50685081
"members": [
50695082
{
5070-
"type": "FIELD",
5071-
"name": "subscript",
5072-
"content": {
5073-
"type": "CHOICE",
5074-
"members": [
5075-
{
5076-
"type": "SYMBOL",
5077-
"name": "list_splat"
5078-
},
5079-
{
5080-
"type": "SYMBOL",
5081-
"name": "expression"
5082-
},
5083-
{
5084-
"type": "SYMBOL",
5085-
"name": "slice"
5086-
}
5087-
]
5088-
}
5083+
"type": "STRING",
5084+
"value": ","
50895085
},
50905086
{
5091-
"type": "REPEAT",
5092-
"content": {
5093-
"type": "SEQ",
5094-
"members": [
5095-
{
5096-
"type": "STRING",
5097-
"value": ","
5098-
},
5099-
{
5100-
"type": "FIELD",
5101-
"name": "subscript",
5102-
"content": {
5103-
"type": "CHOICE",
5104-
"members": [
5105-
{
5106-
"type": "SYMBOL",
5107-
"name": "list_splat"
5108-
},
5109-
{
5110-
"type": "SYMBOL",
5111-
"name": "expression"
5112-
},
5113-
{
5087+
"type": "SEQ",
5088+
"members": [
5089+
{
5090+
"type": "REPEAT1",
5091+
"content": {
5092+
"type": "SEQ",
5093+
"members": [
5094+
{
5095+
"type": "STRING",
5096+
"value": ","
5097+
},
5098+
{
5099+
"type": "FIELD",
5100+
"name": "element",
5101+
"content": {
51145102
"type": "SYMBOL",
5115-
"name": "slice"
5103+
"name": "_index_expression"
51165104
}
5117-
]
5118-
}
5105+
}
5106+
]
51195107
}
5120-
]
5121-
}
5108+
},
5109+
{
5110+
"type": "CHOICE",
5111+
"members": [
5112+
{
5113+
"type": "STRING",
5114+
"value": ","
5115+
},
5116+
{
5117+
"type": "BLANK"
5118+
}
5119+
]
5120+
}
5121+
]
51225122
}
51235123
]
5124+
}
5125+
]
5126+
}
5127+
},
5128+
"subscript": {
5129+
"type": "PREC",
5130+
"value": 21,
5131+
"content": {
5132+
"type": "SEQ",
5133+
"members": [
5134+
{
5135+
"type": "FIELD",
5136+
"name": "value",
5137+
"content": {
5138+
"type": "SYMBOL",
5139+
"name": "primary_expression"
5140+
}
51245141
},
51255142
{
5126-
"type": "CHOICE",
5127-
"members": [
5128-
{
5129-
"type": "STRING",
5130-
"value": ","
5131-
},
5132-
{
5133-
"type": "BLANK"
5134-
}
5135-
]
5143+
"type": "STRING",
5144+
"value": "["
5145+
},
5146+
{
5147+
"type": "FIELD",
5148+
"name": "subscript",
5149+
"content": {
5150+
"type": "CHOICE",
5151+
"members": [
5152+
{
5153+
"type": "SYMBOL",
5154+
"name": "_index_expression"
5155+
},
5156+
{
5157+
"type": "SYMBOL",
5158+
"name": "index_expression_list"
5159+
}
5160+
]
5161+
}
51365162
},
51375163
{
51385164
"type": "STRING",

Diff for: ‎python/extractor/tsg-python/tsp/src/node-types.json

+29-1
Original file line numberDiff line numberDiff line change
@@ -1829,6 +1829,30 @@
18291829
}
18301830
}
18311831
},
1832+
{
1833+
"type": "index_expression_list",
1834+
"named": true,
1835+
"fields": {
1836+
"element": {
1837+
"multiple": true,
1838+
"required": true,
1839+
"types": [
1840+
{
1841+
"type": "expression",
1842+
"named": true
1843+
},
1844+
{
1845+
"type": "list_splat",
1846+
"named": true
1847+
},
1848+
{
1849+
"type": "slice",
1850+
"named": true
1851+
}
1852+
]
1853+
}
1854+
}
1855+
},
18321856
{
18331857
"type": "interpolation",
18341858
"named": true,
@@ -3200,13 +3224,17 @@
32003224
"named": true,
32013225
"fields": {
32023226
"subscript": {
3203-
"multiple": true,
3227+
"multiple": false,
32043228
"required": true,
32053229
"types": [
32063230
{
32073231
"type": "expression",
32083232
"named": true
32093233
},
3234+
{
3235+
"type": "index_expression_list",
3236+
"named": true
3237+
},
32103238
{
32113239
"type": "list_splat",
32123240
"named": true

0 commit comments

Comments
 (0)