Tokenize @ and . in decorators (#110)

chbk · 1st1 · commit 4030374b5245 · 2018-10-04T16:02:19.000-04:00
* Tokenize `@` in decorators

* Tokenize periods in decorators
diff --git a/grammars/MagicPython.cson b/grammars/MagicPython.cson
@@ -1479,7 +1479,7 @@ repository:
     begin: '''
       (?x)
         ^\\s*
-        (@) \\s* (?=[[:alpha:]_]\\w*)
+        ((@)) \\s* (?=[[:alpha:]_]\\w*)
       
     '''
     end: '''
@@ -1493,6 +1493,8 @@ repository:
     beginCaptures:
       "1":
         name: "entity.name.function.decorator.python"
+      "2":
+        name: "punctuation.definition.decorator.python"
     endCaptures:
       "1":
         name: "punctuation.definition.arguments.end.python"
@@ -1518,9 +1520,12 @@ repository:
         name: "entity.name.function.decorator.python"
         match: '''
           (?x)
-            ([[:alpha:]_]\\w*) | \\.
+            ([[:alpha:]_]\\w*) | (\\.)
           
         '''
+        captures:
+          "2":
+            name: "punctuation.separator.period.python"
       }
       {
         include: "#line-continuation"
diff --git a/grammars/MagicPython.tmLanguage b/grammars/MagicPython.tmLanguage
@@ -2331,7 +2331,7 @@
         <key>begin</key>
         <string>(?x)
   ^\s*
-  (@) \s* (?=[[:alpha:]_]\w*)
+  ((@)) \s* (?=[[:alpha:]_]\w*)
 </string>
         <key>end</key>
         <string>(?x)
@@ -2347,6 +2347,11 @@
             <key>name</key>
             <string>entity.name.function.decorator.python</string>
           </dict>
+          <key>2</key>
+          <dict>
+            <key>name</key>
+            <string>punctuation.definition.decorator.python</string>
+          </dict>
         </dict>
         <key>endCaptures</key>
         <dict>
@@ -2390,8 +2395,16 @@
             <string>entity.name.function.decorator.python</string>
             <key>match</key>
             <string>(?x)
-  ([[:alpha:]_]\w*) | \.
+  ([[:alpha:]_]\w*) | (\.)
 </string>
+            <key>captures</key>
+            <dict>
+              <key>2</key>
+              <dict>
+                <key>name</key>
+                <string>punctuation.separator.period.python</string>
+              </dict>
+            </dict>
           </dict>
           <dict>
             <key>include</key>
diff --git a/grammars/src/MagicPython.syntax.yaml b/grammars/src/MagicPython.syntax.yaml
@@ -1138,7 +1138,7 @@ repository:
     begin: |
       (?x)
         ^\s*
-        (@) \s* (?=[[:alpha:]_]\w*)
+        ((@)) \s* (?=[[:alpha:]_]\w*)
     end: |
         (?x)
           ( \) )
@@ -1147,6 +1147,7 @@ repository:
           | (?=\n|\#)
     beginCaptures:
       '1': {name: entity.name.function.decorator.python}
+      '2': {name: punctuation.definition.decorator.python}
     endCaptures:
       '1': {name: punctuation.definition.arguments.end.python}
       '2': {name: invalid.illegal.decorator.python}
@@ -1162,7 +1163,9 @@ repository:
       - name: entity.name.function.decorator.python
         match: |
           (?x)
-            ([[:alpha:]_]\w*) | \.
+            ([[:alpha:]_]\w*) | (\.)
+        captures:
+          '2': {name: punctuation.separator.period.python}
       - include: '#line-continuation'
       - name: invalid.illegal.decorator.python
         match: |
diff --git a/misc/scopes b/misc/scopes
@@ -84,6 +84,7 @@ punctuation.comment.end.regexp
 punctuation.definition.arguments.begin.python
 punctuation.definition.arguments.end.python
 punctuation.definition.comment.python
+punctuation.definition.decorator.python
 punctuation.definition.dict.begin.python
 punctuation.definition.dict.end.python
 punctuation.definition.inheritance.begin.python
diff --git a/test/atom-spec/python-spec.js b/test/atom-spec/python-spec.js
diff --git a/test/classes/class12.py b/test/classes/class12.py
@@ -17,7 +17,7 @@ class         : meta.class.python, source.python, storage.type.class.python
 F             : entity.name.type.class.python, meta.class.python, source.python
 :             : meta.class.python, punctuation.section.class.begin.python, source.python
               : meta.function.decorator.python, source.python
-@             : entity.name.function.decorator.python, meta.function.decorator.python, source.python
+@             : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.definition.decorator.python, source.python
 classmethod   : meta.function.decorator.python, source.python, support.type.python
               : meta.function.python, source.python
 def           : meta.function.python, source.python, storage.type.function.python
diff --git a/test/classes/class2.py b/test/classes/class2.py
@@ -5,7 +5,7 @@ class Spam(Foo.Bar, Bar.name={'very': 'odd'}):
 
 
 
-@             : entity.name.function.decorator.python, meta.function.decorator.python, source.python
+@             : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.definition.decorator.python, source.python
 dec           : entity.name.function.decorator.python, meta.function.decorator.python, source.python
 #             : comment.line.number-sign.python, punctuation.definition.comment.python, source.python
  Bar.name=... is not legal, but the test is for highlighter not breaking badly : comment.line.number-sign.python, source.python
diff --git a/test/docstrings/regr1.py b/test/docstrings/regr1.py
@@ -8,7 +8,7 @@ def foo():
 #             : comment.line.number-sign.python, punctuation.definition.comment.python, source.python
 :             : comment.line.number-sign.python, source.python
               : meta.function.decorator.python, source.python
-@             : entity.name.function.decorator.python, meta.function.decorator.python, source.python
+@             : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.definition.decorator.python, source.python
 asd           : entity.name.function.decorator.python, meta.function.decorator.python, source.python
               : meta.function.python, source.python
 def           : meta.function.python, source.python, storage.type.function.python
diff --git a/test/functions/async1.py b/test/functions/async1.py
@@ -7,7 +7,7 @@ async def foo():
 
 
 
-@             : entity.name.function.decorator.python, meta.function.decorator.python, source.python
+@             : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.definition.decorator.python, source.python
 foo           : entity.name.function.decorator.python, meta.function.decorator.python, source.python
 async         : meta.function.python, source.python, storage.type.function.async.python
               : meta.function.python, source.python
diff --git a/test/functions/decorators1.py b/test/functions/decorators1.py
@@ -8,25 +8,25 @@ def decorated(a): pass
 
 
 
-@             : entity.name.function.decorator.python, meta.function.decorator.python, source.python
+@             : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.definition.decorator.python, source.python
 some_decorator : entity.name.function.decorator.python, meta.function.decorator.python, source.python
               : meta.function.decorator.python, source.python
 #             : comment.line.number-sign.python, punctuation.definition.comment.python, source.python
  with comment : comment.line.number-sign.python, source.python
-@             : entity.name.function.decorator.python, meta.function.decorator.python, source.python
+@             : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.definition.decorator.python, source.python
 some          : entity.name.function.decorator.python, meta.function.decorator.python, source.python
-.             : entity.name.function.decorator.python, meta.function.decorator.python, source.python
+.             : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.separator.period.python, source.python
 class         : keyword.control.flow.python, meta.function.decorator.python, source.python
-.             : entity.name.function.decorator.python, meta.function.decorator.python, source.python
+.             : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.separator.period.python, source.python
 decorator     : entity.name.function.decorator.python, meta.function.decorator.python, source.python
-@             : entity.name.function.decorator.python, meta.function.decorator.python, source.python
+@             : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.definition.decorator.python, source.python
 some_decorator : entity.name.function.decorator.python, meta.function.decorator.python, source.python
 (             : meta.function.decorator.python, punctuation.definition.arguments.begin.python, source.python
 1             : constant.numeric.dec.python, meta.function-call.arguments.python, meta.function.decorator.python, source.python
 )             : meta.function.decorator.python, punctuation.definition.arguments.end.python, source.python
-@             : entity.name.function.decorator.python, meta.function.decorator.python, source.python
+@             : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.definition.decorator.python, source.python
 some          : entity.name.function.decorator.python, meta.function.decorator.python, source.python
-.             : entity.name.function.decorator.python, meta.function.decorator.python, source.python
+.             : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.separator.period.python, source.python
 decorator     : entity.name.function.decorator.python, meta.function.decorator.python, source.python
               : meta.function.decorator.python, source.python
 (             : meta.function.decorator.python, punctuation.definition.arguments.begin.python, source.python
@@ -35,7 +35,7 @@ class         : keyword.control.flow.python, meta.function.decorator.python, sou
               : meta.function-call.arguments.python, meta.function.decorator.python, source.python
 3             : constant.numeric.dec.python, meta.function-call.arguments.python, meta.function.decorator.python, source.python
 )             : meta.function.decorator.python, punctuation.definition.arguments.end.python, source.python
-@             : entity.name.function.decorator.python, meta.function.decorator.python, source.python
+@             : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.definition.decorator.python, source.python
 some_decorator : entity.name.function.decorator.python, meta.function.decorator.python, source.python
 (             : meta.function.decorator.python, punctuation.definition.arguments.begin.python, source.python
 a             : meta.function-call.arguments.python, meta.function.decorator.python, source.python, variable.parameter.function-call.python
@@ -58,7 +58,7 @@ class         : keyword.control.flow.python, meta.function.decorator.python, sou
 **            : keyword.operator.unpacking.arguments.python, meta.function-call.arguments.python, meta.function.decorator.python, source.python
 kwargs        : meta.function-call.arguments.python, meta.function.decorator.python, source.python
 )             : meta.function.decorator.python, punctuation.definition.arguments.end.python, source.python
-@             : entity.name.function.decorator.python, meta.function.decorator.python, source.python
+@             : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.definition.decorator.python, source.python
 classmethod   : meta.function.decorator.python, source.python, support.type.python
 def           : meta.function.python, source.python, storage.type.function.python
               : meta.function.python, source.python
diff --git a/test/functions/decorators2.py b/test/functions/decorators2.py
@@ -3,19 +3,19 @@ def foo(): pass
 
 
 
-@             : entity.name.function.decorator.python, meta.function.decorator.python, source.python
+@             : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.definition.decorator.python, source.python
               : meta.function.decorator.python, source.python
 f             : entity.name.function.decorator.python, meta.function.decorator.python, source.python
               : meta.function.decorator.python, source.python
-.             : entity.name.function.decorator.python, meta.function.decorator.python, source.python
+.             : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.separator.period.python, source.python
               : meta.function.decorator.python, source.python
 bar           : entity.name.function.decorator.python, meta.function.decorator.python, source.python
               : meta.function.decorator.python, source.python
-.             : entity.name.function.decorator.python, meta.function.decorator.python, source.python
+.             : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.separator.period.python, source.python
               : meta.function.decorator.python, source.python
 a             : entity.name.function.decorator.python, meta.function.decorator.python, source.python
               : meta.function.decorator.python, source.python
-.             : entity.name.function.decorator.python, meta.function.decorator.python, source.python
+.             : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.separator.period.python, source.python
               : meta.function.decorator.python, source.python
 b             : entity.name.function.decorator.python, meta.function.decorator.python, source.python
 def           : meta.function.python, source.python, storage.type.function.python
diff --git a/test/functions/decorators3.py b/test/functions/decorators3.py
@@ -3,11 +3,11 @@ def foo(): pass
 
 
 
-@             : entity.name.function.decorator.python, meta.function.decorator.python, source.python
+@             : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.definition.decorator.python, source.python
               : meta.function.decorator.python, source.python
 f             : entity.name.function.decorator.python, meta.function.decorator.python, source.python
               : meta.function.decorator.python, source.python
-.             : entity.name.function.decorator.python, meta.function.decorator.python, source.python
+.             : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.separator.period.python, source.python
               : meta.function.decorator.python, source.python
 bar           : entity.name.function.decorator.python, meta.function.decorator.python, source.python
               : meta.function.decorator.python, source.python
diff --git a/test/functions/decorators4.py b/test/functions/decorators4.py
@@ -3,15 +3,15 @@ def foo(): pass
 
 
 
-@             : entity.name.function.decorator.python, meta.function.decorator.python, source.python
+@             : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.definition.decorator.python, source.python
 a             : entity.name.function.decorator.python, meta.function.decorator.python, source.python
-.             : entity.name.function.decorator.python, meta.function.decorator.python, source.python
+.             : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.separator.period.python, source.python
 b             : entity.name.function.decorator.python, meta.function.decorator.python, source.python
-.             : entity.name.function.decorator.python, meta.function.decorator.python, source.python
+.             : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.separator.period.python, source.python
 c             : entity.name.function.decorator.python, meta.function.decorator.python, source.python
-.             : entity.name.function.decorator.python, meta.function.decorator.python, source.python
+.             : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.separator.period.python, source.python
 None          : keyword.illegal.name.python, meta.function.decorator.python, source.python
-.             : entity.name.function.decorator.python, meta.function.decorator.python, source.python
+.             : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.separator.period.python, source.python
 z             : entity.name.function.decorator.python, meta.function.decorator.python, source.python
 def           : meta.function.python, source.python, storage.type.function.python
               : meta.function.python, source.python
diff --git a/test/functions/decorators5.py b/test/functions/decorators5.py
@@ -7,24 +7,24 @@ def foo(): pass
 
 
 
-@             : entity.name.function.decorator.python, meta.function.decorator.python, source.python
+@             : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.definition.decorator.python, source.python
 a             : entity.name.function.decorator.python, meta.function.decorator.python, source.python
-.             : entity.name.function.decorator.python, meta.function.decorator.python, source.python
+.             : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.separator.period.python, source.python
               : meta.function.decorator.python, source.python
 \             : meta.function.decorator.python, punctuation.separator.continuation.line.python, source.python
               : meta.function.decorator.python, source.python
               : meta.function.decorator.python, source.python
 b             : entity.name.function.decorator.python, meta.function.decorator.python, source.python
               : meta.function.decorator.python, source.python
-.             : entity.name.function.decorator.python, meta.function.decorator.python, source.python
+.             : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.separator.period.python, source.python
               : meta.function.decorator.python, source.python
 \             : meta.function.decorator.python, punctuation.separator.continuation.line.python, source.python
               : meta.function.decorator.python, source.python
               : meta.function.decorator.python, source.python
 c             : entity.name.function.decorator.python, meta.function.decorator.python, source.python
-.             : entity.name.function.decorator.python, meta.function.decorator.python, source.python
+.             : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.separator.period.python, source.python
 None          : keyword.illegal.name.python, meta.function.decorator.python, source.python
-.             : entity.name.function.decorator.python, meta.function.decorator.python, source.python
+.             : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.separator.period.python, source.python
 z             : entity.name.function.decorator.python, meta.function.decorator.python, source.python
               : meta.function.decorator.python, source.python
 \             : meta.function.decorator.python, punctuation.separator.continuation.line.python, source.python
@@ -36,9 +36,9 @@ def foo(): pass
 =             : keyword.operator.assignment.python, meta.function-call.arguments.python, meta.function.decorator.python, source.python
 1             : constant.numeric.dec.python, meta.function-call.arguments.python, meta.function.decorator.python, source.python
 )             : meta.function.decorator.python, punctuation.definition.arguments.end.python, source.python
-@             : entity.name.function.decorator.python, meta.function.decorator.python, source.python
+@             : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.definition.decorator.python, source.python
 foo           : entity.name.function.decorator.python, meta.function.decorator.python, source.python
-.             : entity.name.function.decorator.python, meta.function.decorator.python, source.python
+.             : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.separator.period.python, source.python
 ok            : entity.name.function.decorator.python, meta.function.decorator.python, source.python
 def           : meta.function.python, source.python, storage.type.function.python
               : meta.function.python, source.python
diff --git a/test/functions/decorators6.py b/test/functions/decorators6.py
diff --git a/test/functions/decorators7.py b/test/functions/decorators7.py
diff --git a/test/functions/decorators8.py b/test/functions/decorators8.py
diff --git a/test/unicode/unicode1.py b/test/unicode/unicode1.py
