Partial fix to handling of '#' in fstring format spec.

vpetrovykh · vpetrovykh · commit 8beee2f147ea · 2016-11-18T19:17:34.000-05:00
diff --git a/grammars/MagicPython.cson b/grammars/MagicPython.cson
@@ -869,6 +869,37 @@ repository:
         match: "(}(?!}))"
       }
     ]
+  "fstring-terminator":
+    patterns: [
+      {
+        match: '''
+          (?x)
+            (![rsa])
+              ( : \\w? [<>=^]? [-+ ]? \\#?
+                \\d* ,? (\\.\\d+)? [bcdeEfFgGnosxX%]? )?
+          
+        '''
+        captures:
+          "1":
+            name: "storage.type.format.python"
+          "2":
+            name: "support.other.format.python"
+      }
+      {
+        match: '''
+          (?x)
+            (![rsa])?
+              ( : \\w? [<>=^]? [-+ ]? \\#?
+                \\d* ,? (\\.\\d+)? [bcdeEfFgGnosxX%]? )
+          
+        '''
+        captures:
+          "1":
+            name: "storage.type.format.python"
+          "2":
+            name: "support.other.format.python"
+      }
+    ]
   "escape-sequence-unicode":
     patterns: [
       {
@@ -3409,9 +3440,6 @@ repository:
     begin: "(\\{)"
     end: '''
       (?x)
-        (![rsa])?
-          ( : \\w? [<>=^]? [-+ ]? \\#?
-            \\d* ,? (\\.\\d+)? [bcdeEfFgGnosxX%]? )?
         (\\})
       
     '''
@@ -3420,10 +3448,6 @@ repository:
         name: "constant.character.format.placeholder.other.python"
     endCaptures:
       "1":
-        name: "storage.type.format.python"
-      "2":
-        name: "support.other.format.python"
-      "4":
         name: "constant.character.format.placeholder.other.python"
     patterns: [
       {
@@ -3481,9 +3505,6 @@ repository:
     begin: "(\\{)"
     end: '''
       (?x)
-        (![rsa])?
-          ( : \\w? [<>=^]? [-+ ]? \\#?
-            \\d* ,? (\\.\\d+)? [bcdeEfFgGnosxX%]? )?
         (\\})
       
     '''
@@ -3492,10 +3513,6 @@ repository:
         name: "constant.character.format.placeholder.other.python"
     endCaptures:
       "1":
-        name: "storage.type.format.python"
-      "2":
-        name: "support.other.format.python"
-      "4":
         name: "constant.character.format.placeholder.other.python"
     patterns: [
       {
diff --git a/grammars/MagicPython.tmLanguage b/grammars/MagicPython.tmLanguage
@@ -1404,6 +1404,54 @@
           </dict>
         </array>
       </dict>
+      <key>fstring-terminator</key>
+      <dict>
+        <key>patterns</key>
+        <array>
+          <dict>
+            <key>match</key>
+            <string>(?x)
+  (![rsa])
+    ( : \w? [&lt;&gt;=^]? [-+ ]? \#?
+      \d* ,? (\.\d+)? [bcdeEfFgGnosxX%]? )?
+</string>
+            <key>captures</key>
+            <dict>
+              <key>1</key>
+              <dict>
+                <key>name</key>
+                <string>storage.type.format.python</string>
+              </dict>
+              <key>2</key>
+              <dict>
+                <key>name</key>
+                <string>support.other.format.python</string>
+              </dict>
+            </dict>
+          </dict>
+          <dict>
+            <key>match</key>
+            <string>(?x)
+  (![rsa])?
+    ( : \w? [&lt;&gt;=^]? [-+ ]? \#?
+      \d* ,? (\.\d+)? [bcdeEfFgGnosxX%]? )
+</string>
+            <key>captures</key>
+            <dict>
+              <key>1</key>
+              <dict>
+                <key>name</key>
+                <string>storage.type.format.python</string>
+              </dict>
+              <key>2</key>
+              <dict>
+                <key>name</key>
+                <string>support.other.format.python</string>
+              </dict>
+            </dict>
+          </dict>
+        </array>
+      </dict>
       <key>escape-sequence-unicode</key>
       <dict>
         <key>patterns</key>
@@ -5789,9 +5837,6 @@ indirectly through syntactic constructs
         <string>(\{)</string>
         <key>end</key>
         <string>(?x)
-  (![rsa])?
-    ( : \w? [&lt;&gt;=^]? [-+ ]? \#?
-      \d* ,? (\.\d+)? [bcdeEfFgGnosxX%]? )?
   (\})
 </string>
         <key>beginCaptures</key>
@@ -5805,16 +5850,6 @@ indirectly through syntactic constructs
         <key>endCaptures</key>
         <dict>
           <key>1</key>
-          <dict>
-            <key>name</key>
-            <string>storage.type.format.python</string>
-          </dict>
-          <key>2</key>
-          <dict>
-            <key>name</key>
-            <string>support.other.format.python</string>
-          </dict>
-          <key>4</key>
           <dict>
             <key>name</key>
             <string>constant.character.format.placeholder.other.python</string>
@@ -5906,9 +5941,6 @@ indirectly through syntactic constructs
         <string>(\{)</string>
         <key>end</key>
         <string>(?x)
-  (![rsa])?
-    ( : \w? [&lt;&gt;=^]? [-+ ]? \#?
-      \d* ,? (\.\d+)? [bcdeEfFgGnosxX%]? )?
   (\})
 </string>
         <key>beginCaptures</key>
@@ -5922,16 +5954,6 @@ indirectly through syntactic constructs
         <key>endCaptures</key>
         <dict>
           <key>1</key>
-          <dict>
-            <key>name</key>
-            <string>storage.type.format.python</string>
-          </dict>
-          <key>2</key>
-          <dict>
-            <key>name</key>
-            <string>support.other.format.python</string>
-          </dict>
-          <key>4</key>
           <dict>
             <key>name</key>
             <string>constant.character.format.placeholder.other.python</string>
diff --git a/grammars/src/MagicPython.syntax.yaml b/grammars/src/MagicPython.syntax.yaml
@@ -612,6 +612,26 @@ repository:
       - name: invalid.illegal.brace.python
         match: (}(?!}))
 
+  fstring-terminator:
+    patterns:
+      - match: |
+          (?x)
+            (![rsa])
+              ( : \w? [<>=^]? [-+ ]? \#?
+                \d* ,? (\.\d+)? [bcdeEfFgGnosxX%]? )?
+        captures:
+          '1': {name: storage.type.format.python}
+          '2': {name: support.other.format.python}
+
+      - match: |
+          (?x)
+            (![rsa])?
+              ( : \w? [<>=^]? [-+ ]? \#?
+                \d* ,? (\.\d+)? [bcdeEfFgGnosxX%]? )
+        captures:
+          '1': {name: storage.type.format.python}
+          '2': {name: support.other.format.python}
+
   escape-sequence-unicode:
     patterns:
       - name: constant.character.escape.python
diff --git a/grammars/src/pyfstring.inc.syntax.yaml b/grammars/src/pyfstring.inc.syntax.yaml
@@ -39,16 +39,11 @@ repository:
     begin: (\{)
     end: |
       (?x)
-        (![rsa])?
-          ( : \w? [<>=^]? [-+ ]? \#?
-            \d* ,? (\.\d+)? [bcdeEfFgGnosxX%]? )?
         (\})
     beginCaptures:
       '1': {name: constant.character.format.placeholder.other.python}
     endCaptures:
-      '1': {name: storage.type.format.python}
-      '2': {name: support.other.format.python}
-      '4': {name: constant.character.format.placeholder.other.python}
+      '1': {name: constant.character.format.placeholder.other.python}
     patterns:
       - include: '#fstring-terminator'
       - include: '#expression'
diff --git a/test/fstrings/comment1.py b/test/fstrings/comment1.py
@@ -0,0 +1,21 @@
+f'prefix{10 # comment, making the string technically illegal
+def foo(): pass
+
+
+
+
+f             : source.python, storage.type.string.python, string.quoted.single.python
+'             : punctuation.definition.string.begin.python, source.python, string.quoted.single.python
+prefix        : source.python, string.quoted.single.python
+{             : constant.character.format.placeholder.other.python, source.python
+10            : constant.numeric.dec.python, source.python
+              : source.python
+#             : comment.line.number-sign.python, punctuation.definition.comment.python, source.python
+ comment, making the string technically illegal : comment.line.number-sign.python, source.python
+def           : keyword.control.flow.python, source.python
+              : source.python
+foo           : meta.function-call.generic.python, meta.function-call.python, source.python
+(             : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python
+)             : meta.function-call.python, punctuation.definition.arguments.end.python, source.python
+:             : source.python, support.other.format.python
+pass          : keyword.control.flow.python, source.python
diff --git a/test/fstrings/comment2.py b/test/fstrings/comment2.py
@@ -0,0 +1,20 @@
+f'''
+    prefix{10 # comment, perfectly legal
+    } suffix'''
+
+
+
+
+f             : source.python, storage.type.string.python, string.quoted.multi.python
+'''           : punctuation.definition.string.begin.python, source.python, string.quoted.multi.python
+              : source.python, string.quoted.multi.python
+    prefix    : source.python, string.quoted.multi.python
+{             : constant.character.format.placeholder.other.python, source.python
+10            : constant.numeric.dec.python, source.python
+              : source.python
+#             : comment.line.number-sign.python, punctuation.definition.comment.python, source.python
+ comment, perfectly legal : comment.line.number-sign.python, source.python
+              : source.python
+}             : constant.character.format.placeholder.other.python, source.python
+ suffix       : source.python, string.quoted.multi.python
+'''           : punctuation.definition.string.end.python, source.python, string.quoted.multi.python
diff --git a/test/fstrings/comment3.py b/test/fstrings/comment3.py
@@ -0,0 +1,38 @@
+self.assertEqual(f'{10:#{3 != {4:5} and width}x}', '       0xa')
+
+
+
+
+self          : source.python, variable.language.special.self.python
+.             : source.python
+assertEqual   : meta.function-call.generic.python, meta.function-call.python, source.python
+(             : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python
+f             : meta.function-call.arguments.python, meta.function-call.python, source.python, storage.type.string.python, string.quoted.single.python
+'             : meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.begin.python, source.python, string.quoted.single.python
+{             : constant.character.format.placeholder.other.python, meta.function-call.arguments.python, meta.function-call.python, source.python
+10            : constant.numeric.dec.python, meta.function-call.arguments.python, meta.function-call.python, source.python
+:#            : meta.function-call.arguments.python, meta.function-call.python, source.python, support.other.format.python
+{             : meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.dict.begin.python, source.python
+3             : constant.numeric.dec.python, meta.function-call.arguments.python, meta.function-call.python, source.python
+              : meta.function-call.arguments.python, meta.function-call.python, source.python
+!=            : keyword.operator.comparison.python, meta.function-call.arguments.python, meta.function-call.python, source.python
+              : meta.function-call.arguments.python, meta.function-call.python, source.python
+{             : meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.dict.begin.python, source.python
+4             : constant.numeric.dec.python, meta.function-call.arguments.python, meta.function-call.python, source.python
+:             : meta.function-call.arguments.python, meta.function-call.python, source.python
+5             : constant.numeric.dec.python, meta.function-call.arguments.python, meta.function-call.python, source.python
+}             : meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.dict.end.python, source.python
+              : meta.function-call.arguments.python, meta.function-call.python, source.python
+and           : keyword.operator.logical.python, meta.function-call.arguments.python, meta.function-call.python, source.python
+              : meta.function-call.arguments.python, meta.function-call.python, source.python
+width         : meta.function-call.arguments.python, meta.function-call.python, source.python
+}             : meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.dict.end.python, source.python
+x             : meta.function-call.arguments.python, meta.function-call.python, source.python
+}             : constant.character.format.placeholder.other.python, meta.function-call.arguments.python, meta.function-call.python, source.python
+'             : meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.end.python, source.python, string.quoted.single.python
+,             : meta.function-call.arguments.python, meta.function-call.python, punctuation.separator.arguments.python, source.python
+              : meta.function-call.arguments.python, meta.function-call.python, source.python
+'             : meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.begin.python, source.python, string.quoted.single.python
+       0xa    : meta.function-call.arguments.python, meta.function-call.python, source.python, string.quoted.single.python
+'             : meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.end.python, source.python, string.quoted.single.python
+)             : meta.function-call.python, punctuation.definition.arguments.end.python, source.python
diff --git a/test/fstrings/nested1.py b/test/fstrings/nested1.py
@@ -52,7 +52,7 @@ def           : keyword.control.flow.python, meta.function-call.arguments.python
 aaa           : meta.function-call.arguments.python, meta.function-call.generic.python, meta.function-call.python, source.python
 (             : meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.arguments.begin.python, source.python
 )             : meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.arguments.end.python, source.python
-:             : meta.function-call.arguments.python, meta.function-call.python, source.python
+:             : meta.function-call.arguments.python, meta.function-call.python, source.python, support.other.format.python
 pass          : keyword.control.flow.python, meta.function-call.arguments.python, meta.function-call.python, source.python
 }             : constant.character.format.placeholder.other.python, meta.function-call.arguments.python, meta.function-call.python, source.python
 111           : meta.function-call.arguments.python, meta.function-call.python, source.python, string.quoted.single.python
diff --git a/test/fstrings/nested2.py b/test/fstrings/nested2.py
@@ -0,0 +1,31 @@
+f"result: {value:{60}.{16!s:2}{'qwerty'[2]}}"
+
+
+
+f             : source.python, storage.type.string.python, string.quoted.single.python
+"             : punctuation.definition.string.begin.python, source.python, string.quoted.single.python
+result:       : source.python, string.quoted.single.python
+{             : constant.character.format.placeholder.other.python, source.python
+value         : source.python
+:             : source.python, support.other.format.python
+{             : punctuation.definition.dict.begin.python, source.python
+60            : constant.numeric.dec.python, source.python
+}             : punctuation.definition.dict.end.python, source.python
+.             : source.python
+{             : punctuation.definition.dict.begin.python, source.python
+16            : constant.numeric.dec.python, source.python
+!             : invalid.illegal.operator.python, source.python
+s             : source.python
+:             : source.python
+2             : constant.numeric.dec.python, source.python
+}             : punctuation.definition.dict.end.python, source.python
+{             : punctuation.definition.dict.begin.python, source.python
+'             : punctuation.definition.string.begin.python, source.python, string.quoted.single.python
+qwerty        : source.python, string.quoted.single.python
+'             : punctuation.definition.string.end.python, source.python, string.quoted.single.python
+[             : punctuation.definition.list.begin.python, source.python
+2             : constant.numeric.dec.python, source.python
+]             : punctuation.definition.list.end.python, source.python
+}             : punctuation.definition.dict.end.python, source.python
+}             : constant.character.format.placeholder.other.python, source.python
+"             : punctuation.definition.string.end.python, source.python, string.quoted.single.python
diff --git a/test/fstrings/simple1.py b/test/fstrings/simple1.py
@@ -45,8 +45,7 @@
 [             : punctuation.definition.list.begin.python, source.python
               : source.python
 ]             : punctuation.definition.list.end.python, source.python
-:             : source.python
-Xd            : source.python
+:Xd           : source.python, support.other.format.python
 >             : keyword.operator.comparison.python, source.python
 +             : keyword.operator.arithmetic.python, source.python
 10d           : invalid.illegal.name.python, source.python
diff --git a/test/fstrings/simple6.py b/test/fstrings/simple6.py
@@ -21,7 +21,7 @@ def           : keyword.control.flow.python, source.python
 aaa           : meta.function-call.generic.python, meta.function-call.python, source.python
 (             : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python
 )             : meta.function-call.python, punctuation.definition.arguments.end.python, source.python
-:             : source.python
+:             : source.python, support.other.format.python
 pass          : keyword.control.flow.python, source.python
 }             : constant.character.format.placeholder.other.python, source.python
 '             : punctuation.definition.string.end.python, source.python, string.quoted.single.python
