From 8d11bdedcac08f80ed54b3097c621efb85a762e3 Mon Sep 17 00:00:00 2001 From: Maxim Sokolov Date: Mon, 25 Jan 2016 20:35:11 +0300 Subject: [PATCH 01/21] Improve functions Recognize function bodies. Tokenize rest parameter Tokenize arrow functions with one parameter Don't scope not built-in objects as 'support' --- grammars/javascript.cson | 575 ++++++++++++++++++------------------ spec/javascript-spec.coffee | 306 +++++++++---------- 2 files changed, 440 insertions(+), 441 deletions(-) diff --git a/grammars/javascript.cson b/grammars/javascript.cson index 378b6392..448a5c8a 100644 --- a/grammars/javascript.cson +++ b/grammars/javascript.cson @@ -281,239 +281,255 @@ 'name': 'variable.language.js' } { - 'captures': - '1': - 'name': 'support.class.js' - '2': - 'name': 'support.constant.js' - '3': - 'name': 'keyword.operator.assignment.js' - 'comment': 'match stuff like: Sound.prototype = { … } when extending an object' - 'match': '([a-zA-Z_?.$][\\w?.$]*)\\.(prototype)\\s*(=)\\s*' - 'name': 'meta.class.js' - } - { - 'begin': '([a-zA-Z_?.$][\\w?.$]*)\\.(prototype)\\.([a-zA-Z_?.$][\\w?.$]*)\\s*(=)\\s*(?:(async)(?:\\s+))?(function\\*?)\\s*(\\*?)(\\()' - 'beginCaptures': - '1': - 'name': 'support.class.js' - '2': - 'name': 'support.constant.js' - '3': - 'name': 'entity.name.function.js' - '4': - 'name': 'keyword.operator.assignment.js' - '5': - 'name': 'storage.modifier.js' - '6': - 'name': 'storage.type.function.js' - '7': - 'name': 'storage.type.function.js' - '8': - 'name': 'punctuation.definition.parameters.begin.js' - 'comment': 'match stuff like: Sound.prototype.play = function() { … }' - 'end': '(\\))' - 'endCaptures': - '1': - 'name': 'punctuation.definition.parameters.end.js' - 'name': 'meta.function.prototype.js' + # [async] function [name](params) + # function* name(params) – generator function declaration + 'begin': '(?=(?:\\basync\\b\\s*)?\\bfunction\\b)' + 'end': '(?<=})' 'patterns': [ { - 'include': '#function-params' + 'begin': '\\G' + 'end': '(?<=\\))' + 'name': 'meta.function.js' + 'patterns': [ + { + 'include': '#function_innards' + } + ] + } + { + 'include': '#function_body' + } + { + 'include': '#comments' } ] } { - 'captures': - '1': - 'name': 'support.class.js' - '2': - 'name': 'support.constant.js' - '3': - 'name': 'entity.name.function.js' - '4': - 'name': 'keyword.operator.assignment.js' - 'comment': 'match stuff like: Sound.prototype.play = myfunc' - 'match': '([a-zA-Z_?.$][\\w?.$]*)\\.(prototype)\\.([a-zA-Z_?.$][\\w?.$]*)\\s*(=)\\s*' - 'name': 'meta.function.js' + # [.]foo = function... + 'begin': '(?=(\\.)?[a-zA-Z_$][\\w$]*\\s*=\\s*(\\basync\\b\\s*)?\\bfunction\\b)' + 'end': '(?<=})' + 'patterns': [ + { + 'begin': '\\G' + 'end': '(?<=\\))' + 'name': 'meta.function.js' + 'patterns': [ + { + 'match': '(\\.)?([a-zA-Z_$][\\w$]*)\\s*(=)\\s*' + 'captures': + '1': + 'name': 'meta.delimiter.method.period.js' + '2': + 'name': 'entity.name.function.js' + '3': + 'name': 'keyword.operator.assignment.js' + } + { + 'include': '#function_innards' + } + ] + } + { + 'include': '#function_body' + } + { + 'include': '#comments' + } + ] } { - 'begin': '([a-zA-Z_?.$][\\w?.$]*)\\.([a-zA-Z_?.$][\\w?.$]*)\\s*(=)\\s*(?:(async)(?:\\s+))?(function\\*?)\\s*(\\*?)\\s*([a-zA-Z_?$][\\w?$]*)?\\s*(\\()' - 'beginCaptures': - '1': - 'name': 'support.class.js' - '2': - 'name': 'entity.name.function.js' - '3': - 'name': 'keyword.operator.assignment.js' - '4': - 'name': 'storage.modifier.js' - '5': - 'name': 'storage.type.function.js' - '6': - 'name': 'storage.type.function.js' - '7': - 'name': 'entity.name.function.js' - '8': - 'name': 'punctuation.definition.parameters.begin.js' - 'comment': 'match stuff like: Sound.play = function() { … }' - 'end': '(\\))' - 'endCaptures': - '1': - 'name': 'punctuation.definition.parameters.end.js' - 'name': 'meta.function.js' + # foo: function... + 'begin': '(?=\\b[a-zA-Z_$][\\w$]*\\s*:\\s*(\\basync\\b\\s*)?\\bfunction\\b)' + 'end': '(?<=})' 'patterns': [ { - 'include': '#function-params' + 'begin': '\\G' + 'end': '(?<=\\))' + 'name': 'meta.function.json.js' + 'patterns': [ + { + 'match': '\\b([a-zA-Z_$][\\w$]*)\\s*(:)\\s*' + 'captures': + '1': + 'name': 'entity.name.function.js' + '2': + 'name': 'keyword.operator.assignment.js' + } + { + 'include': '#function_innards' + } + ] + } + { + 'include': '#function_body' + } + { + 'include': '#comments' } ] } { - 'begin': '([a-zA-Z_?$][\\w?$]*)\\s*(=)\\s*(?:(async)(?:\\s+))?(function\\*?)\\s*(\\*?)\\s*([a-zA-Z_?$][\\w?$]*)?\\s*(\\()' - 'beginCaptures': - '1': - 'name': 'entity.name.function.js' - '2': - 'name': 'keyword.operator.assignment.js' - '3': - 'name': 'storage.modifier.js' - '4': - 'name': 'storage.type.function.js' - '5': - 'name': 'storage.type.function.js' - '6': - 'name': 'entity.name.function.js' - '7': - 'name': 'punctuation.definition.parameters.begin.js' - 'comment': 'match stuff like: play = function() { … }' - 'end': '(\\))' - 'endCaptures': - '1': - 'name': 'punctuation.definition.parameters.end.js' - 'name': 'meta.function.js' + # "foo": function... + 'begin': '(?=((\'[^\']*?\')|("[^"]*?"))\\s*:\\s*(\\basync\\b\\s*)?\\bfunction\\b)' + 'end': '(?<=})' 'patterns': [ { - 'include': '#function-params' + 'begin': '\\G' + 'end': '(?<=\\))' + 'name': 'meta.function.json.js' + 'patterns': [ + { + 'match': '(?:((\')([^\']*?)(\'))|((")([^"]*?)(")))\\s*(:)' + 'captures': + '1': + 'name': 'string.quoted.single.js' + '2': + 'name': 'punctuation.definition.string.begin.js' + '3': + 'name': 'entity.name.function.js' + '4': + 'name': 'punctuation.definition.string.end.js' + '5': + 'name': 'string.quoted.double.js' + '6': + 'name': 'punctuation.definition.string.begin.js' + '7': + 'name': 'entity.name.function.js' + '8': + 'name': 'punctuation.definition.string.end.js' + '9': + 'name': 'keyword.operator.assignment.js' + } + { + 'include': '#function_innards' + } + ] + } + { + 'include': '#function_body' + } + { + 'include': '#comments' } ] } { - 'begin': '\\b(?:(async)(?:\\s+))?(function\\*?)\\s*((\\*)|(?:(?:\\s+)(\\*?[a-zA-Z_$][\\w$]*)))?\\s*(\\()' - 'beginCaptures': - '1': - 'name': 'storage.modifier.js' - '2': - 'name': 'storage.type.function.js' - '4': - 'name': 'storage.type.function.js' - '5': - 'name': 'entity.name.function.js' - '6': - 'name': 'punctuation.definition.parameters.begin.js' - 'comment': 'match regular function like: function myFunc(arg) { … }' - 'end': '(\\))' - 'endCaptures': - '1': - 'name': 'punctuation.definition.parameters.end.js' - 'name': 'meta.function.js' + # ES6 method definitions: foo() {} + 'begin': '''(?x) + (?= + (?! + (break|case|catch|continue|do|else|finally|for|function|if|export| + import|package|return|switch|throw|try|while|with) + [\\s\\(] + ) + \\b[a-zA-Z_$][a-zA-Z_$0-9]* + \\s*\\(\\s* + (("[^"]*")|(\\'[^\\']*\\')|((?!"|\\'|\\(|\\)).))* + \\)\\s*\\{ + ) + ''' + 'end': '(?<=})' 'patterns': [ { - 'include': '#function-params' + 'begin': '\\G' + 'end': '(?<=\\))' + 'name': 'meta.function.method.definition.js' + 'patterns': [ + { + 'match': '\\b([a-zA-Z_$][\\w$]*)' + 'name': 'entity.name.function.js' + } + { + 'include': '#function_params' + } + ] + } + { + 'include': '#function_body' + } + { + 'include': '#comments' } ] } { - 'begin': '\\b([a-zA-Z_?.$][\\w?.$]*)\\s*(:)\\s*\\b(?:(async)(?:\\s+))?(function\\*?)\\s*(\\*?)\\s*([a-zA-Z_?$][\\w?$]*)?\\s*(\\()' - 'beginCaptures': - '1': - 'name': 'entity.name.function.js' - '2': - 'name': 'keyword.operator.assignment.js' - '3': - 'name': 'storage.modifier.js' - '4': - 'name': 'storage.type.function.js' - '5': - 'name': 'storage.type.function.js' - '6': - 'name': 'entity.name.function.js' - '7': - 'name': 'punctuation.definition.parameters.begin.js' - 'comment': 'match stuff like: foobar: function() { … }' - 'end': '(\\))' - 'endCaptures': - '1': - 'name': 'punctuation.definition.parameters.end.js' - 'name': 'meta.function.json.js' + # [param|(params)] => [expression|{statements}] + 'begin': '(?=(?)' + 'end': '(?!(\\s*{|\\G\\(|\\G[\\w$]+|/\\*))((?=\\s*\\S)|(?<=}))' 'patterns': [ { - 'include': '#function-params' + 'begin': '\\G' + 'end': '(?<=(=>))' + 'name': 'meta.function.arrow.js' + 'patterns': [ + { + 'include': '#arrow_function_innards' + } + ] + } + { + 'include': '#function_body' + } + { + 'include': '#comments' } ] } { - 'begin': '(?:((\')([^\']*?)(\'))|((")([^"]*?)(")))\\s*(:)\\s*\\b(?:(async)(?:\\s+))?(function\\*?)\\s*(\\*?)\\s*([a-zA-Z_?$][\\w?$]*)?\\s*(\\()' - 'beginCaptures': - '1': - 'name': 'string.quoted.single.js' - '2': - 'name': 'punctuation.definition.string.begin.js' - '3': - 'name': 'entity.name.function.js' - '4': - 'name': 'punctuation.definition.string.end.js' - '5': - 'name': 'string.quoted.double.js' - '6': - 'name': 'punctuation.definition.string.begin.js' - '7': - 'name': 'entity.name.function.js' - '8': - 'name': 'punctuation.definition.string.end.js' - '9': - 'name': 'keyword.operator.assignment.js' - '10': - 'name': 'storage.modifier.js' - '11': - 'name': 'storage.type.function.js' - '12': - 'name': 'storage.type.function.js' - '13': - 'name': 'entity.name.function.js' - '14': - 'name': 'punctuation.definition.parameters.begin.js' - 'comment': 'Attempt to match "foo": function' - 'end': '(\\))' - 'endCaptures': - '1': - 'name': 'punctuation.definition.parameters.end.js' - 'name': 'meta.function.json.js' + # [.]foo = ... => ... + 'begin': '(?=(\\.)?[a-zA-Z_$][\\w$]*\\s*(=)\\s*((\\(([^\\(\\)]*)?\\))|[\\w$]+)\\s*=>)' + 'end': '''(?x) + (?!( + \\s*{|\\G(\\.)?[a-zA-Z_$][\\w$]*\\s*(=)\\s*\\(| + \\G(\\.)?[a-zA-Z_$][\\w$]*\\s*(=)\\s*[\\w$]+| + /\\* + ))((?=\\s*\\S)|(?<=})) + ''' 'patterns': [ { - 'include': '#function-params' + 'begin': '\\G' + 'end': '(?<=(=>))' + 'name': 'meta.function.arrow.js' + 'patterns': [ + { + 'match': '\\G(\\.)?([a-zA-Z_$][\\w$]*)\\s*(=)' + 'captures': + '1': + 'name': 'meta.delimiter.method.period.js' + '2': + 'name': 'entity.name.function.js' + '3': + 'name': 'keyword.operator.assignment.js' + } + { + 'include': '#arrow_function_innards' + } + ] + } + { + 'include': '#function_body' + } + { + 'include': '#comments' } ] } { - 'begin': '\\b(constructor)\\s*(\\()' + 'match': '=>', + 'name': 'storage.type.arrow.js' + } + { + 'begin': '\\b(constructor)\\s*(?=\\()' 'beginCaptures': '1': 'name': 'entity.name.function.constructor.js' - '2': - 'name': 'punctuation.definition.parameters.begin.js' - 'end': '(\\))' - 'endCaptures': - '1': - 'name': 'punctuation.definition.parameters.end.js' + 'end': '(?<=\\})' 'patterns': [ { - 'include': '#function-params' + 'include': '#function_params' } ] } - { - 'include': '#methods' - } { 'match': '''(?x) \\b(class) @@ -541,72 +557,6 @@ 'name': 'entity.name.type.js' 'name': 'meta.class.js' } - { - 'name': 'meta.function.arrow.js' - 'begin': '(?)' - 'beginCaptures': - '1': - 'name': 'punctuation.definition.parameters.begin.js' - 'end': '(\\))\\s*(=>)' - 'endCaptures': - '1': - 'name': 'punctuation.definition.parameters.end.js' - '2': - 'name': 'storage.type.arrow.js' - 'patterns': [ - { - 'include': '#function-params' - } - ] - } - { - 'comment': 'match stuff like: play = (a,b) => { … }' - 'name': 'meta.function.arrow.js' - 'begin': '([a-zA-Z_?$][\\w?$]*)\\s*(=)\\s*(\\()(?=(?:[^\\(\\)]*)?\\)\\s*=>)' - 'beginCaptures': - '1': - 'name': 'entity.name.function.js' - '2': - 'name': 'keyword.operator.assignment.js' - '3': - 'name': 'punctuation.definition.parameters.begin.js' - 'end': '(\\))\\s*(=>)' - 'endCaptures': - '1': - 'name': 'punctuation.definition.parameters.end.js' - '2': - 'name': 'storage.type.arrow.js' - 'patterns': [ - { - 'include': '#function-params' - } - ] - } - { - 'comment': 'match stuff like: Sound.play = (a,b) => { … }' - 'name': 'meta.function.arrow.js' - 'begin': '([a-zA-Z_?$]*)\\.([a-zA-Z_?$][\\w?$]*)\\s*(=)\\s*(\\()(?=(?:[^\\(\\)]*)?\\)\\s*=>)' - 'beginCaptures': - '1': - 'name': 'support.class.js' - '2': - 'name': 'entity.name.function.js' - '3': - 'name': 'keyword.operator.assignment.js' - '4': - 'name': 'punctuation.definition.parameters.begin.js' - 'end': '(\\))\\s*(=>)' - 'endCaptures': - '1': - 'name': 'punctuation.definition.parameters.end.js' - '2': - 'name': 'storage.type.arrow.js' - 'patterns': [ - { - 'include': '#function-params' - } - ] - } { 'captures': '1': @@ -643,10 +593,6 @@ 'match': '()' 'name': 'comment.block.html.js' } - { - 'match': '=>', - 'name': 'storage.type.arrow.js' - } { 'match': '(? + 'begin': '(\\()(?=(?:[^\\(\\)]*)?\\)\\s*=>)' 'beginCaptures': '1': - 'name': 'entity.name.function.js' - '2': 'name': 'punctuation.definition.parameters.begin.js' - 'end': '(\\))' + 'end': '(\\))\\s*(=>)' 'endCaptures': '1': 'name': 'punctuation.definition.parameters.end.js' + '2': + 'name': 'storage.type.arrow.js' 'patterns': [ { - 'include': '#function-params' - } - { - 'include': '#strings' + 'include': '#function_params' } ] } ] - 'function-params': + 'function_params': 'patterns': [ { - 'include': '#numbers' - } - { - 'include': '#strings' - } - { - 'match': '\\(|\\)' - 'name': 'meta.brace.round.js' + 'begin': '(\\()' + 'beginCaptures': + '1': + 'name': 'punctuation.definition.parameters.begin.bracket.round.js' + 'end': '(\\))' + 'endCaptures': + '1': + 'name': 'punctuation.definition.parameters.end.bracket.round.js' + 'name': 'meta.parameters.js' + 'patterns': [ + { + 'match': '(\\.\\.\\.)([a-zA-Z_$][a-zA-Z_$0-9]*)' + 'captures': + '1': + 'name': 'keyword.operator.spread.js' + '2': + 'name': 'variable.parameter.rest.function.js' + } + { + 'include': '$self' + } + { + 'match': '[a-zA-Z_$][a-zA-Z_$0-9]*' + 'name': 'variable.parameter.function.js' + } + ] } + ] + 'function_body': + 'patterns': [ { - 'match': '\\[|\\]' - 'name': 'meta.brace.square.js' + 'begin': '{' + 'beginCaptures': + '0': + 'name': 'punctuation.definition.function.body.begin.bracket.curly.js' + 'end': '}' + 'endCaptures': + '0': + 'name': 'punctuation.definition.function.body.end.bracket.curly.js' + 'patterns': [ + { + 'include': '$self' + } + ] } + ] + 'function_innards': + 'patterns': [ { - 'match': '{|}' - 'name': 'meta.brace.curly.js' + 'match': '''(?x) + (?:\\b(async)\\b\\s*)? + \\b(function) + \\s*(\\*?)\\s* + ([a-zA-Z_$][\\w$]*)? + \\s*(?=\\() + ''' + 'captures': + '1': + 'name': 'storage.modifier.async.js' + '2': + 'name': 'storage.type.function.js' + '3': + 'name': 'storage.modifier.generator.asterisk.js' + '4': + 'name': 'entity.name.function.js' } { - 'match': ',' - 'name': 'meta.object.delimiter.js' + 'include': '#function_params' } + ] + 'arrow_function_innards': + 'patterns': [ { - 'match': '=' - 'name': 'keyword.operator.assignment.js' + 'match': '=>' + 'name': 'storage.type.arrow.js' } { - 'include': '#comments' + 'include': '#function_params' } { - 'match': '[a-zA-Z_$][a-zA-Z_$0-9]*' - 'name': 'variable.parameter.function.js' + 'match': '([a-zA-Z_$][a-zA-Z_$0-9]*)(?=\\s*=>)' + 'captures': + '0': + 'name': 'meta.parameters.js' + '1': + 'name': 'variable.parameter.function.js' } - ] + ] 'arguments': 'patterns': [ { diff --git a/spec/javascript-spec.coffee b/spec/javascript-spec.coffee index fe0673a9..aa0ca957 100644 --- a/spec/javascript-spec.coffee +++ b/spec/javascript-spec.coffee @@ -509,14 +509,14 @@ describe "Javascript grammar", -> expect(tokens[0]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.js', 'punctuation.definition.string.begin.js'] expect(tokens[1]).toEqual value: 'hey ', scopes: ['source.js', 'string.quoted.template.js'] expect(tokens[2]).toEqual value: '${', scopes: ['source.js', 'string.quoted.template.js', 'source.js.embedded.source', 'punctuation.section.embedded.js'] - expect(tokens[3]).toEqual value: '(', scopes: ['source.js', 'string.quoted.template.js', 'source.js.embedded.source', 'meta.function.arrow.js', 'punctuation.definition.parameters.begin.js'] - expect(tokens[4]).toEqual value: ')', scopes: ['source.js', 'string.quoted.template.js', 'source.js.embedded.source', 'meta.function.arrow.js', 'punctuation.definition.parameters.end.js'] + expect(tokens[3]).toEqual value: '(', scopes: ['source.js', 'string.quoted.template.js', 'source.js.embedded.source', 'meta.function.arrow.js', 'meta.parameters.js', 'punctuation.definition.parameters.begin.bracket.round.js'] + expect(tokens[4]).toEqual value: ')', scopes: ['source.js', 'string.quoted.template.js', 'source.js.embedded.source', 'meta.function.arrow.js', 'meta.parameters.js', 'punctuation.definition.parameters.end.bracket.round.js'] expect(tokens[6]).toEqual value: '=>', scopes: ['source.js', 'string.quoted.template.js', 'source.js.embedded.source', 'meta.function.arrow.js', 'storage.type.arrow.js'] - expect(tokens[8]).toEqual value: '{', scopes: ['source.js', 'string.quoted.template.js', 'source.js.embedded.source', 'meta.brace.curly.js'] + expect(tokens[8]).toEqual value: '{', scopes: ['source.js', 'string.quoted.template.js', 'source.js.embedded.source', 'punctuation.definition.function.body.begin.bracket.curly.js'] expect(tokens[9]).toEqual value: 'return', scopes: ['source.js', 'string.quoted.template.js', 'source.js.embedded.source', 'keyword.control.js'] expect(tokens[10]).toEqual value: ' hi', scopes: ['source.js', 'string.quoted.template.js', 'source.js.embedded.source'] expect(tokens[11]).toEqual value: ';', scopes: ['source.js', 'string.quoted.template.js', 'source.js.embedded.source', 'punctuation.terminator.statement.js'] - expect(tokens[12]).toEqual value: '}', scopes: ['source.js', 'string.quoted.template.js', 'source.js.embedded.source', 'meta.brace.curly.js'] + expect(tokens[12]).toEqual value: '}', scopes: ['source.js', 'string.quoted.template.js', 'source.js.embedded.source', 'punctuation.definition.function.body.end.bracket.curly.js'] expect(tokens[13]).toEqual value: '}', scopes: ['source.js', 'string.quoted.template.js', 'source.js.embedded.source', 'punctuation.section.embedded.js'] expect(tokens[14]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.js', 'punctuation.definition.string.end.js'] @@ -632,14 +632,13 @@ describe "Javascript grammar", -> expect(tokens[4]).toEqual value: '$$', scopes: ['source.js', 'meta.class.js', 'entity.name.type.js'] it "tokenizes constructors", -> - {tokens} = grammar.tokenizeLine('constructor(a, b)') + {tokens} = grammar.tokenizeLine('constructor(p1, p2)') expect(tokens[0]).toEqual value: 'constructor', scopes: ['source.js', 'entity.name.function.constructor.js'] - expect(tokens[1]).toEqual value: '(', scopes: ['source.js', 'punctuation.definition.parameters.begin.js'] - expect(tokens[2]).toEqual value: 'a', scopes: ['source.js', 'variable.parameter.function.js'] - expect(tokens[3]).toEqual value: ',', scopes: ['source.js', 'meta.object.delimiter.js'] - expect(tokens[4]).toEqual value: ' ', scopes: ['source.js'] - expect(tokens[5]).toEqual value: 'b', scopes: ['source.js', 'variable.parameter.function.js'] - expect(tokens[6]).toEqual value: ')', scopes: ['source.js', 'punctuation.definition.parameters.end.js'] + expect(tokens[1]).toEqual value: '(', scopes: ['source.js', 'meta.parameters.js', 'punctuation.definition.parameters.begin.bracket.round.js'] + expect(tokens[2]).toEqual value: 'p1', scopes: ['source.js', 'meta.parameters.js', 'variable.parameter.function.js'] + expect(tokens[3]).toEqual value: ',', scopes: ['source.js', 'meta.parameters.js', 'meta.delimiter.object.comma.js'] + expect(tokens[5]).toEqual value: 'p2', scopes: ['source.js', 'meta.parameters.js', 'variable.parameter.function.js'] + expect(tokens[6]).toEqual value: ')', scopes: ['source.js', 'meta.parameters.js', 'punctuation.definition.parameters.end.bracket.round.js'] describe "ES6 import", -> it "tokenizes import", -> @@ -755,11 +754,6 @@ describe "Javascript grammar", -> expect(tokens[0]).toEqual value: 'export', scopes: ['source.js', 'meta.export.js', 'keyword.control.js'] expect(tokens[2]).toEqual value: 'function', scopes: ['source.js', 'meta.function.js', 'storage.type.function.js'] expect(tokens[4]).toEqual value: 'func', scopes: ['source.js', 'meta.function.js', 'entity.name.function.js'] - expect(tokens[5]).toEqual value: '(', scopes: ['source.js', 'meta.function.js', 'punctuation.definition.parameters.begin.js'] - expect(tokens[6]).toEqual value: 'p1', scopes: ['source.js', 'meta.function.js', 'variable.parameter.function.js'] - expect(tokens[7]).toEqual value: ',', scopes: ['source.js', 'meta.function.js', 'meta.object.delimiter.js'] - expect(tokens[9]).toEqual value: 'p2', scopes: ['source.js', 'meta.function.js', 'variable.parameter.function.js'] - expect(tokens[10]).toEqual value: ')', scopes: ['source.js', 'meta.function.js', 'punctuation.definition.parameters.end.js'] it "tokenizes named class export", -> {tokens} = grammar.tokenizeLine('export class Foo {}') @@ -825,18 +819,11 @@ describe "Javascript grammar", -> expect(tokens[0]).toEqual value: 'export', scopes: ['source.js', 'meta.export.js', 'keyword.control.js'] expect(tokens[2]).toEqual value: 'default', scopes: ['source.js', 'meta.export.js', 'variable.language.default.js'] expect(tokens[4]).toEqual value: 'function', scopes: ['source.js', 'meta.function.js', 'storage.type.function.js'] - expect(tokens[6]).toEqual value: '(', scopes: ['source.js', 'meta.function.js', 'punctuation.definition.parameters.begin.js'] - expect(tokens[7]).toEqual value: ')', scopes: ['source.js', 'meta.function.js', 'punctuation.definition.parameters.end.js'] - expect(tokens[9]).toEqual value: '{', scopes: ['source.js', 'punctuation.section.scope.begin.js'] - expect(tokens[10]).toEqual value: '}', scopes: ['source.js', 'punctuation.section.scope.end.js'] {tokens} = grammar.tokenizeLine('export default function func() {}') expect(tokens[0]).toEqual value: 'export', scopes: ['source.js', 'meta.export.js', 'keyword.control.js'] expect(tokens[2]).toEqual value: 'default', scopes: ['source.js', 'meta.export.js', 'variable.language.default.js'] expect(tokens[4]).toEqual value: 'function', scopes: ['source.js', 'meta.function.js', 'storage.type.function.js'] - expect(tokens[6]).toEqual value: 'func', scopes: ['source.js', 'meta.function.js', 'entity.name.function.js'] - expect(tokens[7]).toEqual value: '(', scopes: ['source.js', 'meta.function.js', 'punctuation.definition.parameters.begin.js'] - expect(tokens[8]).toEqual value: ')', scopes: ['source.js', 'meta.function.js', 'punctuation.definition.parameters.end.js'] it "tokenizes comments in statement", -> lines = grammar.tokenizeLines ''' @@ -924,12 +911,15 @@ describe "Javascript grammar", -> {tokens} = grammar.tokenizeLine('default: ') expect(tokens[0]).toEqual value: 'default', scopes: ['source.js', 'keyword.control.js'] - describe "non-anonymous functions", -> - it "tokenizes regular functions", -> - {tokens} = grammar.tokenizeLine('function nonAnonymous(){}') + describe "functions", -> + it "tokenizes regular function declarations", -> + {tokens} = grammar.tokenizeLine('function foo(){}') expect(tokens[0]).toEqual value: 'function', scopes: ['source.js', 'meta.function.js', 'storage.type.function.js'] - expect(tokens[2]).toEqual value: 'nonAnonymous', scopes: ['source.js', 'meta.function.js', 'entity.name.function.js'] - expect(tokens[3]).toEqual value: '(', scopes: ['source.js', 'meta.function.js', 'punctuation.definition.parameters.begin.js'] + expect(tokens[2]).toEqual value: 'foo', scopes: ['source.js', 'meta.function.js', 'entity.name.function.js'] + expect(tokens[3]).toEqual value: '(', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'punctuation.definition.parameters.begin.bracket.round.js'] + expect(tokens[4]).toEqual value: ')', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'punctuation.definition.parameters.end.bracket.round.js'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'punctuation.definition.function.body.begin.bracket.curly.js'] + expect(tokens[6]).toEqual value: '}', scopes: ['source.js', 'punctuation.definition.function.body.end.bracket.curly.js'] {tokens} = grammar.tokenizeLine('function $abc$(){}') expect(tokens[2]).toEqual value: '$abc$', scopes: ['source.js', 'meta.function.js', 'entity.name.function.js'] @@ -937,112 +927,159 @@ describe "Javascript grammar", -> {tokens} = grammar.tokenizeLine('function $$(){}') expect(tokens[2]).toEqual value: '$$', scopes: ['source.js', 'meta.function.js', 'entity.name.function.js'] - it "tokenizes methods", -> - {tokens} = grammar.tokenizeLine('Foo.method = function nonAnonymous(') - expect(tokens[0]).toEqual value: 'Foo', scopes: ['source.js', 'meta.function.js', 'support.class.js'] + it "tokenizes anonymous functions", -> + {tokens} = grammar.tokenizeLine('function (){}') + expect(tokens[0]).toEqual value: 'function', scopes: ['source.js', 'meta.function.js', 'storage.type.function.js'] + expect(tokens[2]).toEqual value: '(', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'punctuation.definition.parameters.begin.bracket.round.js'] + expect(tokens[3]).toEqual value: ')', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'punctuation.definition.parameters.end.bracket.round.js'] + expect(tokens[4]).toEqual value: '{', scopes: ['source.js', 'punctuation.definition.function.body.begin.bracket.curly.js'] + expect(tokens[5]).toEqual value: '}', scopes: ['source.js', 'punctuation.definition.function.body.end.bracket.curly.js'] + + it "tokenizes async functions", -> + {tokens} = grammar.tokenizeLine('async function foo(){}') + expect(tokens[0]).toEqual value: 'async', scopes: ['source.js', 'meta.function.js', 'storage.modifier.async.js'] + expect(tokens[2]).toEqual value: 'function', scopes: ['source.js', 'meta.function.js', 'storage.type.function.js'] + expect(tokens[4]).toEqual value: 'foo', scopes: ['source.js', 'meta.function.js', 'entity.name.function.js'] + + it "tokenizes functions as object properties", -> + {tokens} = grammar.tokenizeLine('obj.method = function foo(') + expect(tokens[0]).toEqual value: 'obj', scopes: ['source.js', 'variable.other.object.js'] expect(tokens[2]).toEqual value: 'method', scopes: ['source.js', 'meta.function.js', 'entity.name.function.js'] expect(tokens[4]).toEqual value: '=', scopes: ['source.js', 'meta.function.js', 'keyword.operator.assignment.js'] expect(tokens[6]).toEqual value: 'function', scopes: ['source.js', 'meta.function.js', 'storage.type.function.js'] - expect(tokens[8]).toEqual value: 'nonAnonymous', scopes: ['source.js', 'meta.function.js', 'entity.name.function.js'] - expect(tokens[9]).toEqual value: '(', scopes: ['source.js', 'meta.function.js', 'punctuation.definition.parameters.begin.js'] + expect(tokens[8]).toEqual value: 'foo', scopes: ['source.js', 'meta.function.js', 'entity.name.function.js'] + expect(tokens[9]).toEqual value: '(', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'punctuation.definition.parameters.begin.bracket.round.js'] - it "tokenizes methods", -> + it "tokenizes ES6 method definitions", -> {tokens} = grammar.tokenizeLine('f(a, b) {}') - expect(tokens[0]).toEqual value: 'f', scopes: ['source.js', 'meta.method.js', 'entity.name.function.js'] - expect(tokens[1]).toEqual value: '(', scopes: ['source.js', 'meta.method.js', 'punctuation.definition.parameters.begin.js'] - expect(tokens[2]).toEqual value: 'a', scopes: ['source.js', 'meta.method.js', 'variable.parameter.function.js'] - expect(tokens[3]).toEqual value: ',', scopes: ['source.js', 'meta.method.js', 'meta.object.delimiter.js'] - expect(tokens[5]).toEqual value: 'b', scopes: ['source.js', 'meta.method.js', 'variable.parameter.function.js'] - expect(tokens[6]).toEqual value: ')', scopes: ['source.js', 'meta.method.js', 'punctuation.definition.parameters.end.js'] + expect(tokens[0]).toEqual value: 'f', scopes: ['source.js', 'meta.function.method.definition.js', 'entity.name.function.js'] + expect(tokens[1]).toEqual value: '(', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'punctuation.definition.parameters.begin.bracket.round.js'] + expect(tokens[2]).toEqual value: 'a', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'variable.parameter.function.js'] + expect(tokens[3]).toEqual value: ',', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'meta.delimiter.object.comma.js'] + expect(tokens[5]).toEqual value: 'b', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'variable.parameter.function.js'] + expect(tokens[6]).toEqual value: ')', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'punctuation.definition.parameters.end.bracket.round.js'] + + {tokens} = grammar.tokenizeLine('async foo(){}') + expect(tokens[0]).toEqual value: 'async', scopes: ['source.js', 'storage.modifier.js'] + expect(tokens[2]).toEqual value: 'foo', scopes: ['source.js', 'meta.function.method.definition.js', 'entity.name.function.js'] {tokens} = grammar.tokenizeLine('hi({host, root = "./", plugins = [a, "b", "c", d]}) {}') - expect(tokens[0]).toEqual value: 'hi', scopes: ['source.js', 'meta.method.js', 'entity.name.function.js'] - expect(tokens[1]).toEqual value: '(', scopes: ['source.js', 'meta.method.js', 'punctuation.definition.parameters.begin.js'] - expect(tokens[2]).toEqual value: '{', scopes: ['source.js', 'meta.method.js', 'meta.brace.curly.js'] - expect(tokens[3]).toEqual value: 'host', scopes: ['source.js', 'meta.method.js', 'variable.parameter.function.js'] - expect(tokens[4]).toEqual value: ',', scopes: ['source.js', 'meta.method.js', 'meta.object.delimiter.js'] - expect(tokens[6]).toEqual value: 'root', scopes: ['source.js', 'meta.method.js', 'variable.parameter.function.js'] - expect(tokens[8]).toEqual value: '=', scopes: ['source.js', 'meta.method.js', 'keyword.operator.assignment.js'] - expect(tokens[10]).toEqual value: '"', scopes: ['source.js', 'meta.method.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] - expect(tokens[11]).toEqual value: './', scopes: ['source.js', 'meta.method.js', 'string.quoted.double.js'] - expect(tokens[12]).toEqual value: '"', scopes: ['source.js', 'meta.method.js', 'string.quoted.double.js', 'punctuation.definition.string.end.js'] - expect(tokens[13]).toEqual value: ',', scopes: ['source.js', 'meta.method.js', 'meta.object.delimiter.js'] - expect(tokens[15]).toEqual value: 'plugins', scopes: ['source.js', 'meta.method.js', 'variable.parameter.function.js'] - expect(tokens[17]).toEqual value: '=', scopes: ['source.js', 'meta.method.js', 'keyword.operator.assignment.js'] - expect(tokens[19]).toEqual value: '[', scopes: ['source.js', 'meta.method.js', 'meta.brace.square.js'] - expect(tokens[20]).toEqual value: 'a', scopes: ['source.js', 'meta.method.js', 'variable.parameter.function.js'] - expect(tokens[21]).toEqual value: ',', scopes: ['source.js', 'meta.method.js', 'meta.object.delimiter.js'] - expect(tokens[23]).toEqual value: '"', scopes: ['source.js', 'meta.method.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] - expect(tokens[26]).toEqual value: ',', scopes: ['source.js', 'meta.method.js', 'meta.object.delimiter.js'] - expect(tokens[28]).toEqual value: '"', scopes: ['source.js', 'meta.method.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] - expect(tokens[33]).toEqual value: 'd', scopes: ['source.js', 'meta.method.js', 'variable.parameter.function.js'] - expect(tokens[34]).toEqual value: ']', scopes: ['source.js', 'meta.method.js', 'meta.brace.square.js'] - expect(tokens[35]).toEqual value: '}', scopes: ['source.js', 'meta.method.js', 'meta.brace.curly.js'] - expect(tokens[36]).toEqual value: ')', scopes: ['source.js', 'meta.method.js', 'punctuation.definition.parameters.end.js'] - - it "tokenizes functions", -> - {tokens} = grammar.tokenizeLine('var func = function nonAnonymous(') + expect(tokens[0]).toEqual value: 'hi', scopes: ['source.js', 'meta.function.method.definition.js', 'entity.name.function.js'] + expect(tokens[1]).toEqual value: '(', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'punctuation.definition.parameters.begin.bracket.round.js'] + expect(tokens[2]).toEqual value: '{', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'meta.brace.curly.js'] + expect(tokens[3]).toEqual value: 'host', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'variable.parameter.function.js'] + expect(tokens[4]).toEqual value: ',', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'meta.delimiter.object.comma.js'] + expect(tokens[6]).toEqual value: 'root', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'variable.parameter.function.js'] + expect(tokens[8]).toEqual value: '=', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'keyword.operator.assignment.js'] + expect(tokens[10]).toEqual value: '"', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(tokens[11]).toEqual value: './', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'string.quoted.double.js'] + expect(tokens[12]).toEqual value: '"', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'string.quoted.double.js', 'punctuation.definition.string.end.js'] + expect(tokens[13]).toEqual value: ',', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'meta.delimiter.object.comma.js'] + expect(tokens[15]).toEqual value: 'plugins', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'variable.parameter.function.js'] + expect(tokens[17]).toEqual value: '=', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'keyword.operator.assignment.js'] + expect(tokens[19]).toEqual value: '[', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'meta.brace.square.js'] + expect(tokens[20]).toEqual value: 'a', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'variable.parameter.function.js'] + expect(tokens[21]).toEqual value: ',', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'meta.delimiter.object.comma.js'] + expect(tokens[23]).toEqual value: '"', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(tokens[26]).toEqual value: ',', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'meta.delimiter.object.comma.js'] + expect(tokens[28]).toEqual value: '"', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(tokens[33]).toEqual value: 'd', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'variable.parameter.function.js'] + expect(tokens[34]).toEqual value: ']', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'meta.brace.square.js'] + expect(tokens[35]).toEqual value: '}', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'meta.brace.curly.js'] + expect(tokens[36]).toEqual value: ')', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'punctuation.definition.parameters.end.bracket.round.js'] + + {tokens} = grammar.tokenizeLine('write("){");') + expect(tokens[0]).toEqual value: 'write', scopes: ['source.js', 'meta.function-call.js', 'entity.name.function.js'] + + it "tokenizes named function expressions", -> + {tokens} = grammar.tokenizeLine('var func = function foo(){}') + expect(tokens[0]).toEqual value: 'var', scopes: ['source.js', 'storage.type.var.js'] + expect(tokens[2]).toEqual value: 'func', scopes: ['source.js', 'meta.function.js', 'entity.name.function.js'] + expect(tokens[4]).toEqual value: '=', scopes: ['source.js', 'meta.function.js', 'keyword.operator.assignment.js'] + expect(tokens[6]).toEqual value: 'function', scopes: ['source.js', 'meta.function.js', 'storage.type.function.js'] + expect(tokens[8]).toEqual value: 'foo', scopes: ['source.js', 'meta.function.js', 'entity.name.function.js'] + expect(tokens[9]).toEqual value: '(', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'punctuation.definition.parameters.begin.bracket.round.js'] + + it "tokenizes anonymous function expressions", -> + {tokens} = grammar.tokenizeLine('var func = function(){}') expect(tokens[0]).toEqual value: 'var', scopes: ['source.js', 'storage.type.var.js'] expect(tokens[2]).toEqual value: 'func', scopes: ['source.js', 'meta.function.js', 'entity.name.function.js'] expect(tokens[4]).toEqual value: '=', scopes: ['source.js', 'meta.function.js', 'keyword.operator.assignment.js'] expect(tokens[6]).toEqual value: 'function', scopes: ['source.js', 'meta.function.js', 'storage.type.function.js'] - expect(tokens[8]).toEqual value: 'nonAnonymous', scopes: ['source.js', 'meta.function.js', 'entity.name.function.js'] - expect(tokens[9]).toEqual value: '(', scopes: ['source.js', 'meta.function.js', 'punctuation.definition.parameters.begin.js'] + expect(tokens[7]).toEqual value: '(', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'punctuation.definition.parameters.begin.bracket.round.js'] - it "tokenizes object functions", -> - {tokens} = grammar.tokenizeLine('foo: function nonAnonymous(') - expect(tokens[0]).toEqual value: 'foo', scopes: ['source.js', 'meta.function.json.js', 'entity.name.function.js'] + it "tokenizes functions in object literals", -> + {tokens} = grammar.tokenizeLine('func: function foo(') + expect(tokens[0]).toEqual value: 'func', scopes: ['source.js', 'meta.function.json.js', 'entity.name.function.js'] expect(tokens[1]).toEqual value: ':', scopes: ['source.js', 'meta.function.json.js', 'keyword.operator.assignment.js'] expect(tokens[3]).toEqual value: 'function', scopes: ['source.js', 'meta.function.json.js', 'storage.type.function.js'] - expect(tokens[5]).toEqual value: 'nonAnonymous', scopes: ['source.js', 'meta.function.json.js', 'entity.name.function.js'] - expect(tokens[6]).toEqual value: '(', scopes: ['source.js', 'meta.function.json.js', 'punctuation.definition.parameters.begin.js'] + expect(tokens[5]).toEqual value: 'foo', scopes: ['source.js', 'meta.function.json.js', 'entity.name.function.js'] + expect(tokens[6]).toEqual value: '(', scopes: ['source.js', 'meta.function.json.js', 'meta.parameters.js', 'punctuation.definition.parameters.begin.bracket.round.js'] - it "tokenizes quoted object functions", -> - {tokens} = grammar.tokenizeLine('"foo": function nonAnonymous(') - expect(tokens[1]).toEqual value: 'foo', scopes: ['source.js', 'meta.function.json.js', 'string.quoted.double.js', 'entity.name.function.js'] + {tokens} = grammar.tokenizeLine('"func": function foo(') + expect(tokens[1]).toEqual value: 'func', scopes: ['source.js', 'meta.function.json.js', 'string.quoted.double.js', 'entity.name.function.js'] expect(tokens[3]).toEqual value: ':', scopes: ['source.js', 'meta.function.json.js', 'keyword.operator.assignment.js'] expect(tokens[5]).toEqual value: 'function', scopes: ['source.js', 'meta.function.json.js', 'storage.type.function.js'] - expect(tokens[7]).toEqual value: 'nonAnonymous', scopes: ['source.js', 'meta.function.json.js', 'entity.name.function.js'] - expect(tokens[8]).toEqual value: '(', scopes: ['source.js', 'meta.function.json.js', 'punctuation.definition.parameters.begin.js'] + expect(tokens[7]).toEqual value: 'foo', scopes: ['source.js', 'meta.function.json.js', 'entity.name.function.js'] + expect(tokens[8]).toEqual value: '(', scopes: ['source.js', 'meta.function.json.js', 'meta.parameters.js', 'punctuation.definition.parameters.begin.bracket.round.js'] - it "tokenizes async functions", -> - {tokens} = grammar.tokenizeLine('async function f(){}') - expect(tokens[0]).toEqual value: 'async', scopes: ['source.js', 'meta.function.js', 'storage.modifier.js'] - expect(tokens[2]).toEqual value: 'function', scopes: ['source.js', 'meta.function.js', 'storage.type.function.js'] - expect(tokens[4]).toEqual value: 'f', scopes: ['source.js', 'meta.function.js', 'entity.name.function.js'] + it "tokenizes arrow functions", -> + {tokens} = grammar.tokenizeLine('x => x * x') + expect(tokens[0]).toEqual value: 'x', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'variable.parameter.function.js'] + expect(tokens[2]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.js', 'storage.type.arrow.js'] + expect(tokens[3]).toEqual value: ' x ', scopes: ['source.js'] - {tokens} = grammar.tokenizeLine('async f(){}') - expect(tokens[0]).toEqual value: 'async', scopes: ['source.js', 'storage.modifier.js'] - expect(tokens[2]).toEqual value: 'f', scopes: ['source.js', 'meta.method.js', 'entity.name.function.js'] - - it "tokenizes arrow functions with params", -> - {tokens} = grammar.tokenizeLine('(param1,param2)=>{}') - expect(tokens[0]).toEqual value: '(', scopes: ['source.js', 'meta.function.arrow.js', 'punctuation.definition.parameters.begin.js'] - expect(tokens[1]).toEqual value: 'param1', scopes: ['source.js', 'meta.function.arrow.js', 'variable.parameter.function.js'] - expect(tokens[3]).toEqual value: 'param2', scopes: ['source.js', 'meta.function.arrow.js', 'variable.parameter.function.js'] - expect(tokens[4]).toEqual value: ')', scopes: ['source.js', 'meta.function.arrow.js', 'punctuation.definition.parameters.end.js'] - expect(tokens[5]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.js', 'storage.type.arrow.js'] - - it "tokenizes stored arrow functions with params", -> - {tokens} = grammar.tokenizeLine('var func = (param1,param2)=>{}') + {tokens} = grammar.tokenizeLine('() => {}') + expect(tokens[0]).toEqual value: '(', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'punctuation.definition.parameters.begin.bracket.round.js'] + expect(tokens[1]).toEqual value: ')', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'punctuation.definition.parameters.end.bracket.round.js'] + expect(tokens[3]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.js', 'storage.type.arrow.js'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'punctuation.definition.function.body.begin.bracket.curly.js'] + expect(tokens[6]).toEqual value: '}', scopes: ['source.js', 'punctuation.definition.function.body.end.bracket.curly.js'] + + {tokens} = grammar.tokenizeLine('(p1, p2) => {}') + expect(tokens[0]).toEqual value: '(', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'punctuation.definition.parameters.begin.bracket.round.js'] + expect(tokens[1]).toEqual value: 'p1', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'variable.parameter.function.js'] + expect(tokens[2]).toEqual value: ',', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'meta.delimiter.object.comma.js'] + expect(tokens[4]).toEqual value: 'p2', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'variable.parameter.function.js'] + expect(tokens[5]).toEqual value: ')', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'punctuation.definition.parameters.end.bracket.round.js'] + expect(tokens[7]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.js', 'storage.type.arrow.js'] + expect(tokens[9]).toEqual value: '{', scopes: ['source.js', 'punctuation.definition.function.body.begin.bracket.curly.js'] + expect(tokens[10]).toEqual value: '}', scopes: ['source.js', 'punctuation.definition.function.body.end.bracket.curly.js'] + + it "tokenizes stored arrow functions", -> + {tokens} = grammar.tokenizeLine('var func = (p1, p2) => {}') expect(tokens[0]).toEqual value: 'var', scopes: ['source.js', 'storage.type.var.js'] expect(tokens[2]).toEqual value: 'func', scopes: ['source.js', 'meta.function.arrow.js', 'entity.name.function.js'] expect(tokens[4]).toEqual value: '=', scopes: ['source.js', 'meta.function.arrow.js', 'keyword.operator.assignment.js'] - expect(tokens[7]).toEqual value: 'param1', scopes: ['source.js', 'meta.function.arrow.js', 'variable.parameter.function.js'] - expect(tokens[9]).toEqual value: 'param2', scopes: ['source.js', 'meta.function.arrow.js', 'variable.parameter.function.js'] - expect(tokens[10]).toEqual value: ')', scopes: ['source.js', 'meta.function.arrow.js', 'punctuation.definition.parameters.end.js'] - expect(tokens[11]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.js', 'storage.type.arrow.js'] - - it "tokenizes arrow functions with params stored in object properties", -> - {tokens} = grammar.tokenizeLine('Utils.isEmpty = (param1, param2) => {}') - expect(tokens[0]).toEqual value: 'Utils', scopes: ['source.js', 'meta.function.arrow.js', 'support.class.js'] + expect(tokens[11]).toEqual value: ')', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'punctuation.definition.parameters.end.bracket.round.js'] + expect(tokens[13]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.js', 'storage.type.arrow.js'] + + it "tokenizes arrow functions as object properties", -> + {tokens} = grammar.tokenizeLine('Utils.isEmpty = (p1, p2) => {}') + expect(tokens[0]).toEqual value: 'Utils', scopes: ['source.js', 'variable.other.object.js'] expect(tokens[2]).toEqual value: 'isEmpty', scopes: ['source.js', 'meta.function.arrow.js', 'entity.name.function.js'] expect(tokens[4]).toEqual value: '=', scopes: ['source.js', 'meta.function.arrow.js', 'keyword.operator.assignment.js'] - expect(tokens[7]).toEqual value: 'param1', scopes: ['source.js', 'meta.function.arrow.js', 'variable.parameter.function.js'] - expect(tokens[10]).toEqual value: 'param2', scopes: ['source.js', 'meta.function.arrow.js', 'variable.parameter.function.js'] - expect(tokens[11]).toEqual value: ')', scopes: ['source.js', 'meta.function.arrow.js', 'punctuation.definition.parameters.end.js'] + expect(tokens[11]).toEqual value: ')', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'punctuation.definition.parameters.end.bracket.round.js'] expect(tokens[13]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.js', 'storage.type.arrow.js'] + it "tokenizes default parameters", -> + {tokens} = grammar.tokenizeLine('function multiply(a, b = 1){}') + expect(tokens[7]).toEqual value: 'b', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'variable.parameter.function.js'] + expect(tokens[9]).toEqual value: '=', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'keyword.operator.assignment.js'] + expect(tokens[11]).toEqual value: '1', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'constant.numeric.decimal.js'] + + {tokens} = grammar.tokenizeLine('function callSomething(thing = this.something()) {}') + expect(tokens[4]).toEqual value: 'thing', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'variable.parameter.function.js'] + expect(tokens[6]).toEqual value: '=', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'keyword.operator.assignment.js'] + expect(tokens[8]).toEqual value: 'this', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'variable.language.js'] + expect(tokens[9]).toEqual value: '.', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'meta.method-call.js', 'meta.delimiter.method.period.js'] + expect(tokens[10]).toEqual value: 'something', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'meta.method-call.js', 'entity.name.function.js'] + + it "tokenizes rest parameter", -> + {tokens} = grammar.tokenizeLine('(...args) => args[0]') + expect(tokens[1]).toEqual value: '...', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'keyword.operator.spread.js'] + expect(tokens[2]).toEqual value: 'args', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'variable.parameter.rest.function.js'] + describe "variables", -> it "tokenizes 'this'", -> {tokens} = grammar.tokenizeLine('this') @@ -1165,14 +1202,14 @@ describe "Javascript grammar", -> expect(tokens[1]).toEqual value: '(', scopes: ['source.js', 'meta.function-call.js', 'punctuation.definition.arguments.begin.js'] expect(tokens[2]).toEqual value: 'function', scopes: ['source.js', 'meta.function-call.js', 'meta.function.js', 'storage.type.function.js'] expect(tokens[4]).toEqual value: 'b', scopes: ['source.js', 'meta.function-call.js', 'meta.function.js', 'entity.name.function.js'] - expect(tokens[5]).toEqual value: '(', scopes: ['source.js', 'meta.function-call.js', 'meta.function.js', 'punctuation.definition.parameters.begin.js'] - expect(tokens[6]).toEqual value: 'p', scopes: ['source.js', 'meta.function-call.js', 'meta.function.js', 'variable.parameter.function.js'] - expect(tokens[7]).toEqual value: ')', scopes: ['source.js', 'meta.function-call.js', 'meta.function.js', 'punctuation.definition.parameters.end.js'] - expect(tokens[9]).toEqual value: '{', scopes: ['source.js', 'meta.function-call.js', 'meta.brace.curly.js'] + expect(tokens[5]).toEqual value: '(', scopes: ['source.js', 'meta.function-call.js', 'meta.function.js', 'meta.parameters.js', 'punctuation.definition.parameters.begin.bracket.round.js'] + expect(tokens[6]).toEqual value: 'p', scopes: ['source.js', 'meta.function-call.js', 'meta.function.js', 'meta.parameters.js', 'variable.parameter.function.js'] + expect(tokens[7]).toEqual value: ')', scopes: ['source.js', 'meta.function-call.js', 'meta.function.js', 'meta.parameters.js', 'punctuation.definition.parameters.end.bracket.round.js'] + expect(tokens[9]).toEqual value: '{', scopes: ['source.js', 'meta.function-call.js', 'punctuation.definition.function.body.begin.bracket.curly.js'] expect(tokens[11]).toEqual value: 'return', scopes: ['source.js', 'meta.function-call.js', 'keyword.control.js'] expect(tokens[12]).toEqual value: ' p', scopes: ['source.js', 'meta.function-call.js'] expect(tokens[13]).toEqual value: ';', scopes: ['source.js', 'meta.function-call.js', 'punctuation.terminator.statement.js'] - expect(tokens[15]).toEqual value: '}', scopes: ['source.js', 'meta.function-call.js', 'meta.brace.curly.js'] + expect(tokens[15]).toEqual value: '}', scopes: ['source.js', 'meta.function-call.js', 'punctuation.definition.function.body.end.bracket.curly.js'] expect(tokens[16]).toEqual value: ')', scopes: ['source.js', 'meta.function-call.js', 'punctuation.definition.arguments.end.js'] describe "method calls", -> @@ -1281,15 +1318,6 @@ describe "Javascript grammar", -> expect(tokens[8]).toEqual value: "'", scopes: ['source.js', 'meta.method-call.js', 'string.quoted.single.js', 'punctuation.definition.string.end.js'] expect(tokens[9]).toEqual value: ")", scopes: ['source.js', 'meta.method-call.js', 'punctuation.definition.arguments.end.js'] - {tokens} = grammar.tokenizeLine('write("){");') - expect(tokens[0]).toEqual value: 'write', scopes: ['source.js', 'meta.method.js', 'entity.name.function.js'] - expect(tokens[1]).toEqual value: '(', scopes: ['source.js', 'meta.method.js', 'punctuation.definition.parameters.begin.js'] - expect(tokens[2]).toEqual value: '"', scopes: ['source.js', 'meta.method.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] - expect(tokens[3]).toEqual value: '){', scopes: ['source.js', 'meta.method.js', 'string.quoted.double.js'] - expect(tokens[4]).toEqual value: '"', scopes: ['source.js', 'meta.method.js', 'string.quoted.double.js', 'punctuation.definition.string.end.js'] - expect(tokens[5]).toEqual value: ')', scopes: ['source.js', 'meta.method.js', 'punctuation.definition.parameters.end.js'] - expect(tokens[6]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js'] - delimsByScope = "string.quoted.double.js": '"' "string.quoted.single.js": "'" @@ -1350,36 +1378,10 @@ describe "Javascript grammar", -> expect(tokens[6]).toEqual value: ' comment', scopes: ['source.js', 'comment.line.double-slash.js'] it "tokenizes comments inside function parameters correctly", -> - {tokens} = grammar.tokenizeLine('function test(arg1 /*, arg2 */) {}') - expect(tokens[0]).toEqual value: 'function', scopes: ['source.js', 'meta.function.js', 'storage.type.function.js'] - expect(tokens[1]).toEqual value: ' ', scopes: ['source.js', 'meta.function.js'] - expect(tokens[2]).toEqual value: 'test', scopes: ['source.js', 'meta.function.js', 'entity.name.function.js'] - expect(tokens[3]).toEqual value: '(', scopes: ['source.js', 'meta.function.js', 'punctuation.definition.parameters.begin.js'] - expect(tokens[4]).toEqual value: 'arg1', scopes: ['source.js', 'meta.function.js', 'variable.parameter.function.js'] - expect(tokens[5]).toEqual value: ' ', scopes: ['source.js', 'meta.function.js'] - expect(tokens[6]).toEqual value: '/*', scopes: ['source.js', 'meta.function.js', 'comment.block.js', 'punctuation.definition.comment.js'] - expect(tokens[7]).toEqual value: ', arg2 ', scopes: ['source.js', 'meta.function.js', 'comment.block.js'] - expect(tokens[8]).toEqual value: '*/', scopes: ['source.js', 'meta.function.js', 'comment.block.js', 'punctuation.definition.comment.js'] - expect(tokens[9]).toEqual value: ')', scopes: ['source.js', 'meta.function.js', 'punctuation.definition.parameters.end.js'] - expect(tokens[10]).toEqual value: ' ', scopes: ['source.js'] - expect(tokens[11]).toEqual value: '{', scopes: ['source.js', 'punctuation.section.scope.begin.js'] - expect(tokens[12]).toEqual value: '}', scopes: ['source.js', 'punctuation.section.scope.end.js'] - - {tokens} = grammar.tokenizeLine('foo: function (/**Bar*/bar){') - expect(tokens[5]).toEqual value: '(', scopes: ['source.js', 'meta.function.json.js', 'punctuation.definition.parameters.begin.js'] - expect(tokens[6]).toEqual value: '/**', scopes: ['source.js', 'meta.function.json.js', 'comment.block.documentation.js', 'punctuation.definition.comment.js'] - expect(tokens[7]).toEqual value: 'Bar', scopes: ['source.js', 'meta.function.json.js', 'comment.block.documentation.js'] - expect(tokens[8]).toEqual value: '*/', scopes: ['source.js', 'meta.function.json.js', 'comment.block.documentation.js', 'punctuation.definition.comment.js'] - expect(tokens[9]).toEqual value: 'bar', scopes: ['source.js', 'meta.function.json.js', 'variable.parameter.function.js'] - - {tokens} = grammar.tokenizeLine('function test(bar, // comment') - expect(tokens[0]).toEqual value: 'function', scopes: ['source.js', 'meta.function.js', 'storage.type.function.js'] - expect(tokens[2]).toEqual value: 'test', scopes: ['source.js', 'meta.function.js', 'entity.name.function.js'] - expect(tokens[3]).toEqual value: '(', scopes: ['source.js', 'meta.function.js', 'punctuation.definition.parameters.begin.js'] - expect(tokens[4]).toEqual value: 'bar', scopes: ['source.js', 'meta.function.js', 'variable.parameter.function.js'] - expect(tokens[5]).toEqual value: ',', scopes: ['source.js', 'meta.function.js', 'meta.object.delimiter.js'] - expect(tokens[7]).toEqual value: '//', scopes: ['source.js', 'meta.function.js', 'comment.line.double-slash.js', 'punctuation.definition.comment.js'] - expect(tokens[8]).toEqual value: ' comment', scopes: ['source.js', 'meta.function.js', 'comment.line.double-slash.js'] + {tokens} = grammar.tokenizeLine('function test(p1 /*, p2 */) {}') + expect(tokens[6]).toEqual value: '/*', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'comment.block.js', 'punctuation.definition.comment.js'] + expect(tokens[7]).toEqual value: ', p2 ', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'comment.block.js'] + expect(tokens[8]).toEqual value: '*/', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'comment.block.js', 'punctuation.definition.comment.js'] describe "console", -> it "tokenizes the console keyword", -> From a5d1a47858a2103406663f76a3e7c4bd5ac8df99 Mon Sep 17 00:00:00 2001 From: Maxim Sokolov Date: Sun, 7 Feb 2016 12:08:52 +0300 Subject: [PATCH 02/21] Allow comments in function declaration --- grammars/javascript.cson | 54 ++++++++++++++++++++++++++----------- spec/javascript-spec.coffee | 45 +++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 16 deletions(-) diff --git a/grammars/javascript.cson b/grammars/javascript.cson index 448a5c8a..b77c8d44 100644 --- a/grammars/javascript.cson +++ b/grammars/javascript.cson @@ -454,8 +454,22 @@ } { # [param|(params)] => [expression|{statements}] - 'begin': '(?=(?)' - 'end': '(?!(\\s*{|\\G\\(|\\G[\\w$]+|/\\*))((?=\\s*\\S)|(?<=}))' + 'begin': '''(?x) + (?= + (? + ) + ''' + 'end': '''(?x) + (?<=})| + ((?! + \\s*{| + \\G\\(| + \\G[\\w$]+| + \\s*/\\*|// + )(?=\\s*\\S)) + ''' 'patterns': [ { 'begin': '\\G' @@ -477,13 +491,22 @@ } { # [.]foo = ... => ... - 'begin': '(?=(\\.)?[a-zA-Z_$][\\w$]*\\s*(=)\\s*((\\(([^\\(\\)]*)?\\))|[\\w$]+)\\s*=>)' + 'begin': '''(?x) + (?= + (\\.)?[a-zA-Z_$][\\w$]* + \\s*(=)\\s* + ((\\(([^\\(\\)]*)?\\))|[\\w$]+) + \\s*=> + ) + ''' 'end': '''(?x) - (?!( - \\s*{|\\G(\\.)?[a-zA-Z_$][\\w$]*\\s*(=)\\s*\\(| + (?<=})| + ((?! + \\s*{| + \\G(\\.)?[a-zA-Z_$][\\w$]*\\s*(=)\\s*\\(| \\G(\\.)?[a-zA-Z_$][\\w$]*\\s*(=)\\s*[\\w$]+| - /\\* - ))((?=\\s*\\S)|(?<=})) + \\s*/\\*|// + )(?=\\s*\\S)) ''' 'patterns': [ { @@ -1140,13 +1163,7 @@ 'function_innards': 'patterns': [ { - 'match': '''(?x) - (?:\\b(async)\\b\\s*)? - \\b(function) - \\s*(\\*?)\\s* - ([a-zA-Z_$][\\w$]*)? - \\s*(?=\\() - ''' + 'match': '(?:\\b(async)\\b\\s*)?\\b(function)\\b(?:\\s*(\\*))?' 'captures': '1': 'name': 'storage.modifier.async.js' @@ -1154,12 +1171,17 @@ 'name': 'storage.type.function.js' '3': 'name': 'storage.modifier.generator.asterisk.js' - '4': - 'name': 'entity.name.function.js' + } + { + 'match': '[a-zA-Z_$][\\w$]*(?=\\s*\\()' + 'name': 'entity.name.function.js' } { 'include': '#function_params' } + { + 'include': '#comments' + } ] 'arrow_function_innards': 'patterns': [ diff --git a/spec/javascript-spec.coffee b/spec/javascript-spec.coffee index aa0ca957..01ca22f6 100644 --- a/spec/javascript-spec.coffee +++ b/spec/javascript-spec.coffee @@ -1377,6 +1377,51 @@ describe "Javascript grammar", -> expect(tokens[5]).toEqual value: '//', scopes: ['source.js', 'comment.line.double-slash.js', 'punctuation.definition.comment.js'] expect(tokens[6]).toEqual value: ' comment', scopes: ['source.js', 'comment.line.double-slash.js'] + it "tokenizes comments inside function declarations", -> + {tokens} = grammar.tokenizeLine('function /* */ foo() /* */ {}') + expect(tokens[0]).toEqual value: 'function', scopes: ['source.js', 'meta.function.js', 'storage.type.function.js'] + expect(tokens[2]).toEqual value: '/*', scopes: ['source.js', 'meta.function.js', 'comment.block.js', 'punctuation.definition.comment.js'] + expect(tokens[4]).toEqual value: '*/', scopes: ['source.js', 'meta.function.js', 'comment.block.js', 'punctuation.definition.comment.js'] + expect(tokens[6]).toEqual value: 'foo', scopes: ['source.js', 'meta.function.js', 'entity.name.function.js'] + expect(tokens[10]).toEqual value: '/*', scopes: ['source.js', 'comment.block.js', 'punctuation.definition.comment.js'] + expect(tokens[12]).toEqual value: '*/', scopes: ['source.js', 'comment.block.js', 'punctuation.definition.comment.js'] + + {tokens} = grammar.tokenizeLine('x => /* */ {}') + expect(tokens[0]).toEqual value: 'x', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'variable.parameter.function.js'] + expect(tokens[2]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.js', 'storage.type.arrow.js'] + expect(tokens[4]).toEqual value: '/*', scopes: ['source.js', 'comment.block.js', 'punctuation.definition.comment.js'] + expect(tokens[6]).toEqual value: '*/', scopes: ['source.js', 'comment.block.js', 'punctuation.definition.comment.js'] + expect(tokens[8]).toEqual value: '{', scopes: ['source.js', 'punctuation.definition.function.body.begin.bracket.curly.js'] + + {tokens} = grammar.tokenizeLine('.foo = x => /* */ {}') + expect(tokens[1]).toEqual value: 'foo', scopes: ['source.js', 'meta.function.arrow.js', 'entity.name.function.js'] + expect(tokens[5]).toEqual value: 'x', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'variable.parameter.function.js'] + expect(tokens[7]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.js', 'storage.type.arrow.js'] + expect(tokens[9]).toEqual value: '/*', scopes: ['source.js', 'comment.block.js', 'punctuation.definition.comment.js'] + expect(tokens[11]).toEqual value: '*/', scopes: ['source.js', 'comment.block.js', 'punctuation.definition.comment.js'] + expect(tokens[13]).toEqual value: '{', scopes: ['source.js', 'punctuation.definition.function.body.begin.bracket.curly.js'] + + lines = grammar.tokenizeLines ''' + function + // comment + foo() {} + ''' + expect(lines[0][0]).toEqual value: 'function', scopes: ['source.js', 'meta.function.js', 'storage.type.function.js'] + expect(lines[1][0]).toEqual value: '//', scopes: ['source.js', 'meta.function.js', 'comment.line.double-slash.js', 'punctuation.definition.comment.js'] + expect(lines[1][1]).toEqual value: ' comment', scopes: ['source.js', 'meta.function.js', 'comment.line.double-slash.js'] + expect(lines[2][0]).toEqual value: 'foo', scopes: ['source.js', 'meta.function.js', 'entity.name.function.js'] + + lines = grammar.tokenizeLines ''' + x => + // comment + {} + ''' + expect(lines[0][0]).toEqual value: 'x', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'variable.parameter.function.js'] + expect(lines[0][2]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.js', 'storage.type.arrow.js'] + expect(lines[1][0]).toEqual value: '//', scopes: ['source.js', 'comment.line.double-slash.js', 'punctuation.definition.comment.js'] + expect(lines[1][1]).toEqual value: ' comment', scopes: ['source.js', 'comment.line.double-slash.js'] + expect(lines[2][0]).toEqual value: '{', scopes: ['source.js', 'punctuation.definition.function.body.begin.bracket.curly.js'] + it "tokenizes comments inside function parameters correctly", -> {tokens} = grammar.tokenizeLine('function test(p1 /*, p2 */) {}') expect(tokens[6]).toEqual value: '/*', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'comment.block.js', 'punctuation.definition.comment.js'] From d7cdde0fb36aacab813396760529b1e52f2b0ff2 Mon Sep 17 00:00:00 2001 From: Maxim Sokolov Date: Sun, 7 Feb 2016 17:58:25 +0300 Subject: [PATCH 03/21] Wrap '=>' in 'meta.function.arrow' --- grammars/javascript.cson | 8 ++++++-- spec/javascript-spec.coffee | 3 +++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/grammars/javascript.cson b/grammars/javascript.cson index b77c8d44..8504da49 100644 --- a/grammars/javascript.cson +++ b/grammars/javascript.cson @@ -538,8 +538,12 @@ ] } { - 'match': '=>', - 'name': 'storage.type.arrow.js' + 'match': '(=>)' + 'captures': + '0': + 'name': 'meta.function.arrow.js' + '1': + 'name': 'storage.type.arrow.js' } { 'begin': '\\b(constructor)\\s*(?=\\()' diff --git a/spec/javascript-spec.coffee b/spec/javascript-spec.coffee index 01ca22f6..b10df413 100644 --- a/spec/javascript-spec.coffee +++ b/spec/javascript-spec.coffee @@ -1024,6 +1024,9 @@ describe "Javascript grammar", -> expect(tokens[8]).toEqual value: '(', scopes: ['source.js', 'meta.function.json.js', 'meta.parameters.js', 'punctuation.definition.parameters.begin.bracket.round.js'] it "tokenizes arrow functions", -> + {tokens} = grammar.tokenizeLine('=>') + expect(tokens[0]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.js', 'storage.type.arrow.js'] + {tokens} = grammar.tokenizeLine('x => x * x') expect(tokens[0]).toEqual value: 'x', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'variable.parameter.function.js'] expect(tokens[2]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.js', 'storage.type.arrow.js'] From a835e95d3490614eb825caf001612d6047a66db6 Mon Sep 17 00:00:00 2001 From: Maxim Sokolov Date: Sun, 7 Feb 2016 18:00:55 +0300 Subject: [PATCH 04/21] Add 'function' subscope to 'storage.type.arrow' --- grammars/javascript.cson | 6 +++--- spec/javascript-spec.coffee | 20 ++++++++++---------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/grammars/javascript.cson b/grammars/javascript.cson index 8504da49..76bf2f68 100644 --- a/grammars/javascript.cson +++ b/grammars/javascript.cson @@ -543,7 +543,7 @@ '0': 'name': 'meta.function.arrow.js' '1': - 'name': 'storage.type.arrow.js' + 'name': 'storage.type.function.arrow.js' } { 'begin': '\\b(constructor)\\s*(?=\\()' @@ -1107,7 +1107,7 @@ '1': 'name': 'punctuation.definition.parameters.end.js' '2': - 'name': 'storage.type.arrow.js' + 'name': 'storage.type.function.arrow.js' 'patterns': [ { 'include': '#function_params' @@ -1191,7 +1191,7 @@ 'patterns': [ { 'match': '=>' - 'name': 'storage.type.arrow.js' + 'name': 'storage.type.function.arrow.js' } { 'include': '#function_params' diff --git a/spec/javascript-spec.coffee b/spec/javascript-spec.coffee index b10df413..d1a056e1 100644 --- a/spec/javascript-spec.coffee +++ b/spec/javascript-spec.coffee @@ -511,7 +511,7 @@ describe "Javascript grammar", -> expect(tokens[2]).toEqual value: '${', scopes: ['source.js', 'string.quoted.template.js', 'source.js.embedded.source', 'punctuation.section.embedded.js'] expect(tokens[3]).toEqual value: '(', scopes: ['source.js', 'string.quoted.template.js', 'source.js.embedded.source', 'meta.function.arrow.js', 'meta.parameters.js', 'punctuation.definition.parameters.begin.bracket.round.js'] expect(tokens[4]).toEqual value: ')', scopes: ['source.js', 'string.quoted.template.js', 'source.js.embedded.source', 'meta.function.arrow.js', 'meta.parameters.js', 'punctuation.definition.parameters.end.bracket.round.js'] - expect(tokens[6]).toEqual value: '=>', scopes: ['source.js', 'string.quoted.template.js', 'source.js.embedded.source', 'meta.function.arrow.js', 'storage.type.arrow.js'] + expect(tokens[6]).toEqual value: '=>', scopes: ['source.js', 'string.quoted.template.js', 'source.js.embedded.source', 'meta.function.arrow.js', 'storage.type.function.arrow.js'] expect(tokens[8]).toEqual value: '{', scopes: ['source.js', 'string.quoted.template.js', 'source.js.embedded.source', 'punctuation.definition.function.body.begin.bracket.curly.js'] expect(tokens[9]).toEqual value: 'return', scopes: ['source.js', 'string.quoted.template.js', 'source.js.embedded.source', 'keyword.control.js'] expect(tokens[10]).toEqual value: ' hi', scopes: ['source.js', 'string.quoted.template.js', 'source.js.embedded.source'] @@ -1025,17 +1025,17 @@ describe "Javascript grammar", -> it "tokenizes arrow functions", -> {tokens} = grammar.tokenizeLine('=>') - expect(tokens[0]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.js', 'storage.type.arrow.js'] + expect(tokens[0]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.js', 'storage.type.function.arrow.js'] {tokens} = grammar.tokenizeLine('x => x * x') expect(tokens[0]).toEqual value: 'x', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'variable.parameter.function.js'] - expect(tokens[2]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.js', 'storage.type.arrow.js'] + expect(tokens[2]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.js', 'storage.type.function.arrow.js'] expect(tokens[3]).toEqual value: ' x ', scopes: ['source.js'] {tokens} = grammar.tokenizeLine('() => {}') expect(tokens[0]).toEqual value: '(', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'punctuation.definition.parameters.begin.bracket.round.js'] expect(tokens[1]).toEqual value: ')', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'punctuation.definition.parameters.end.bracket.round.js'] - expect(tokens[3]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.js', 'storage.type.arrow.js'] + expect(tokens[3]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.js', 'storage.type.function.arrow.js'] expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'punctuation.definition.function.body.begin.bracket.curly.js'] expect(tokens[6]).toEqual value: '}', scopes: ['source.js', 'punctuation.definition.function.body.end.bracket.curly.js'] @@ -1045,7 +1045,7 @@ describe "Javascript grammar", -> expect(tokens[2]).toEqual value: ',', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'meta.delimiter.object.comma.js'] expect(tokens[4]).toEqual value: 'p2', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'variable.parameter.function.js'] expect(tokens[5]).toEqual value: ')', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'punctuation.definition.parameters.end.bracket.round.js'] - expect(tokens[7]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.js', 'storage.type.arrow.js'] + expect(tokens[7]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.js', 'storage.type.function.arrow.js'] expect(tokens[9]).toEqual value: '{', scopes: ['source.js', 'punctuation.definition.function.body.begin.bracket.curly.js'] expect(tokens[10]).toEqual value: '}', scopes: ['source.js', 'punctuation.definition.function.body.end.bracket.curly.js'] @@ -1055,7 +1055,7 @@ describe "Javascript grammar", -> expect(tokens[2]).toEqual value: 'func', scopes: ['source.js', 'meta.function.arrow.js', 'entity.name.function.js'] expect(tokens[4]).toEqual value: '=', scopes: ['source.js', 'meta.function.arrow.js', 'keyword.operator.assignment.js'] expect(tokens[11]).toEqual value: ')', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'punctuation.definition.parameters.end.bracket.round.js'] - expect(tokens[13]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.js', 'storage.type.arrow.js'] + expect(tokens[13]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.js', 'storage.type.function.arrow.js'] it "tokenizes arrow functions as object properties", -> {tokens} = grammar.tokenizeLine('Utils.isEmpty = (p1, p2) => {}') @@ -1063,7 +1063,7 @@ describe "Javascript grammar", -> expect(tokens[2]).toEqual value: 'isEmpty', scopes: ['source.js', 'meta.function.arrow.js', 'entity.name.function.js'] expect(tokens[4]).toEqual value: '=', scopes: ['source.js', 'meta.function.arrow.js', 'keyword.operator.assignment.js'] expect(tokens[11]).toEqual value: ')', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'punctuation.definition.parameters.end.bracket.round.js'] - expect(tokens[13]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.js', 'storage.type.arrow.js'] + expect(tokens[13]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.js', 'storage.type.function.arrow.js'] it "tokenizes default parameters", -> {tokens} = grammar.tokenizeLine('function multiply(a, b = 1){}') @@ -1391,7 +1391,7 @@ describe "Javascript grammar", -> {tokens} = grammar.tokenizeLine('x => /* */ {}') expect(tokens[0]).toEqual value: 'x', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'variable.parameter.function.js'] - expect(tokens[2]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.js', 'storage.type.arrow.js'] + expect(tokens[2]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.js', 'storage.type.function.arrow.js'] expect(tokens[4]).toEqual value: '/*', scopes: ['source.js', 'comment.block.js', 'punctuation.definition.comment.js'] expect(tokens[6]).toEqual value: '*/', scopes: ['source.js', 'comment.block.js', 'punctuation.definition.comment.js'] expect(tokens[8]).toEqual value: '{', scopes: ['source.js', 'punctuation.definition.function.body.begin.bracket.curly.js'] @@ -1399,7 +1399,7 @@ describe "Javascript grammar", -> {tokens} = grammar.tokenizeLine('.foo = x => /* */ {}') expect(tokens[1]).toEqual value: 'foo', scopes: ['source.js', 'meta.function.arrow.js', 'entity.name.function.js'] expect(tokens[5]).toEqual value: 'x', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'variable.parameter.function.js'] - expect(tokens[7]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.js', 'storage.type.arrow.js'] + expect(tokens[7]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.js', 'storage.type.function.arrow.js'] expect(tokens[9]).toEqual value: '/*', scopes: ['source.js', 'comment.block.js', 'punctuation.definition.comment.js'] expect(tokens[11]).toEqual value: '*/', scopes: ['source.js', 'comment.block.js', 'punctuation.definition.comment.js'] expect(tokens[13]).toEqual value: '{', scopes: ['source.js', 'punctuation.definition.function.body.begin.bracket.curly.js'] @@ -1420,7 +1420,7 @@ describe "Javascript grammar", -> {} ''' expect(lines[0][0]).toEqual value: 'x', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'variable.parameter.function.js'] - expect(lines[0][2]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.js', 'storage.type.arrow.js'] + expect(lines[0][2]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.js', 'storage.type.function.arrow.js'] expect(lines[1][0]).toEqual value: '//', scopes: ['source.js', 'comment.line.double-slash.js', 'punctuation.definition.comment.js'] expect(lines[1][1]).toEqual value: ' comment', scopes: ['source.js', 'comment.line.double-slash.js'] expect(lines[2][0]).toEqual value: '{', scopes: ['source.js', 'punctuation.definition.function.body.begin.bracket.curly.js'] From d0d23e4c87f0a391e7e4ef86990c1e1d63d7531e Mon Sep 17 00:00:00 2001 From: Maxim Sokolov Date: Sun, 7 Feb 2016 19:07:03 +0300 Subject: [PATCH 05/21] :fire: Remove '?:' --- grammars/javascript.cson | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grammars/javascript.cson b/grammars/javascript.cson index 76bf2f68..92147289 100644 --- a/grammars/javascript.cson +++ b/grammars/javascript.cson @@ -283,7 +283,7 @@ { # [async] function [name](params) # function* name(params) – generator function declaration - 'begin': '(?=(?:\\basync\\b\\s*)?\\bfunction\\b)' + 'begin': '(?=(\\basync\\b\\s*)?\\bfunction\\b)' 'end': '(?<=})' 'patterns': [ { From fd2eab66c46531bb98cd57786f1bf3ad2aac14e1 Mon Sep 17 00:00:00 2001 From: Maxim Sokolov Date: Mon, 8 Feb 2016 15:05:17 +0300 Subject: [PATCH 06/21] :art: --- grammars/javascript.cson | 8 ++++---- spec/javascript-spec.coffee | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/grammars/javascript.cson b/grammars/javascript.cson index 963ec68a..7142fc62 100644 --- a/grammars/javascript.cson +++ b/grammars/javascript.cson @@ -1108,13 +1108,13 @@ 'function_params': 'patterns': [ { - 'begin': '(\\()' + 'begin': '\\(' 'beginCaptures': - '1': + '0': 'name': 'punctuation.definition.parameters.begin.bracket.round.js' - 'end': '(\\))' + 'end': '\\)' 'endCaptures': - '1': + '0': 'name': 'punctuation.definition.parameters.end.bracket.round.js' 'name': 'meta.parameters.js' 'patterns': [ diff --git a/spec/javascript-spec.coffee b/spec/javascript-spec.coffee index 5130688f..b74a271b 100644 --- a/spec/javascript-spec.coffee +++ b/spec/javascript-spec.coffee @@ -1406,7 +1406,7 @@ describe "Javascript grammar", -> expect(tokens[10]).toEqual value: '/*', scopes: ['source.js', 'comment.block.js', 'punctuation.definition.comment.js'] expect(tokens[12]).toEqual value: '*/', scopes: ['source.js', 'comment.block.js', 'punctuation.definition.comment.js'] - {tokens} = grammar.tokenizeLine('x => /* */ {}') + {tokens} = grammar.tokenizeLine('x => /* */ {}') expect(tokens[0]).toEqual value: 'x', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'variable.parameter.function.js'] expect(tokens[2]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.js', 'storage.type.function.arrow.js'] expect(tokens[4]).toEqual value: '/*', scopes: ['source.js', 'comment.block.js', 'punctuation.definition.comment.js'] From c108216d443936d29cf6ec80c1aaf51177a906ca Mon Sep 17 00:00:00 2001 From: Maxim Sokolov Date: Mon, 8 Feb 2016 15:08:07 +0300 Subject: [PATCH 07/21] :fire: Remove extra 'asterisk' subscope --- grammars/javascript.cson | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grammars/javascript.cson b/grammars/javascript.cson index 7142fc62..a9aaba51 100644 --- a/grammars/javascript.cson +++ b/grammars/javascript.cson @@ -1164,7 +1164,7 @@ '2': 'name': 'storage.type.function.js' '3': - 'name': 'storage.modifier.generator.asterisk.js' + 'name': 'storage.modifier.generator.js' } { 'match': '[a-zA-Z_$][\\w$]*(?=\\s*\\()' From 5c12bc315496b02a6ef8ff953fe9c69110d5110e Mon Sep 17 00:00:00 2001 From: Maxim Sokolov Date: Mon, 8 Feb 2016 15:09:50 +0300 Subject: [PATCH 08/21] :memo: Add missing 'the' article --- spec/javascript-spec.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/javascript-spec.coffee b/spec/javascript-spec.coffee index b74a271b..94829c70 100644 --- a/spec/javascript-spec.coffee +++ b/spec/javascript-spec.coffee @@ -1078,7 +1078,7 @@ describe "Javascript grammar", -> expect(tokens[9]).toEqual value: '.', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'meta.method-call.js', 'meta.delimiter.method.period.js'] expect(tokens[10]).toEqual value: 'something', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'meta.method-call.js', 'entity.name.function.js'] - it "tokenizes rest parameter", -> + it "tokenizes the rest parameter", -> {tokens} = grammar.tokenizeLine('(...args) => args[0]') expect(tokens[1]).toEqual value: '...', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'keyword.operator.spread.js'] expect(tokens[2]).toEqual value: 'args', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'variable.parameter.rest.function.js'] From 2f5562766dd1338c888cff0970da8e65f4ef0ee0 Mon Sep 17 00:00:00 2001 From: Maxim Sokolov Date: Mon, 8 Feb 2016 15:14:51 +0300 Subject: [PATCH 09/21] :white_check_mark: Test `write("){");` --- spec/javascript-spec.coffee | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/spec/javascript-spec.coffee b/spec/javascript-spec.coffee index 94829c70..a9a400d3 100644 --- a/spec/javascript-spec.coffee +++ b/spec/javascript-spec.coffee @@ -990,6 +990,12 @@ describe "Javascript grammar", -> {tokens} = grammar.tokenizeLine('write("){");') expect(tokens[0]).toEqual value: 'write', scopes: ['source.js', 'meta.function-call.js', 'entity.name.function.js'] + expect(tokens[1]).toEqual value: '(', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(tokens[2]).toEqual value: '"', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(tokens[3]).toEqual value: '){', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'string.quoted.double.js'] + expect(tokens[4]).toEqual value: '"', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'string.quoted.double.js', 'punctuation.definition.string.end.js'] + expect(tokens[5]).toEqual value: ')', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + expect(tokens[6]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js'] it "tokenizes named function expressions", -> {tokens} = grammar.tokenizeLine('var func = function foo(){}') From 079e8e947e90de51a54e91c1050cd3172a082877 Mon Sep 17 00:00:00 2001 From: Maxim Sokolov Date: Mon, 8 Feb 2016 15:22:09 +0300 Subject: [PATCH 10/21] :white_check_mark: Test generator functions --- spec/javascript-spec.coffee | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/spec/javascript-spec.coffee b/spec/javascript-spec.coffee index a9a400d3..7b51af82 100644 --- a/spec/javascript-spec.coffee +++ b/spec/javascript-spec.coffee @@ -1029,6 +1029,22 @@ describe "Javascript grammar", -> expect(tokens[7]).toEqual value: 'foo', scopes: ['source.js', 'meta.function.json.js', 'entity.name.function.js'] expect(tokens[8]).toEqual value: '(', scopes: ['source.js', 'meta.function.json.js', 'meta.parameters.js', 'punctuation.definition.parameters.begin.bracket.round.js'] + it "tokenizes generator functions", -> + {tokens} = grammar.tokenizeLine('function* foo(){}') + expect(tokens[0]).toEqual value: 'function', scopes: ['source.js', 'meta.function.js', 'storage.type.function.js'] + expect(tokens[1]).toEqual value: '*', scopes: ['source.js', 'meta.function.js', 'storage.modifier.generator.js'] + expect(tokens[3]).toEqual value: 'foo', scopes: ['source.js', 'meta.function.js', 'entity.name.function.js'] + expect(tokens[4]).toEqual value: '(', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'punctuation.definition.parameters.begin.bracket.round.js'] + expect(tokens[5]).toEqual value: ')', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'punctuation.definition.parameters.end.bracket.round.js'] + expect(tokens[6]).toEqual value: '{', scopes: ['source.js', 'punctuation.definition.function.body.begin.bracket.curly.js'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'punctuation.definition.function.body.end.bracket.curly.js'] + + {tokens} = grammar.tokenizeLine('function *foo(){}') + expect(tokens[2]).toEqual value: '*', scopes: ['source.js', 'meta.function.js', 'storage.modifier.generator.js'] + + {tokens} = grammar.tokenizeLine('function *(){}') + expect(tokens[2]).toEqual value: '*', scopes: ['source.js', 'meta.function.js', 'storage.modifier.generator.js'] + it "tokenizes arrow functions", -> {tokens} = grammar.tokenizeLine('=>') expect(tokens[0]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.js', 'storage.type.function.arrow.js'] From 9ad6d95570cf2a01b01dd45be8d419a2d904d5f5 Mon Sep 17 00:00:00 2001 From: Maxim Sokolov Date: Mon, 8 Feb 2016 15:27:27 +0300 Subject: [PATCH 11/21] :white_check_mark: Test functions following language variables --- spec/javascript-spec.coffee | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/spec/javascript-spec.coffee b/spec/javascript-spec.coffee index 7b51af82..7fa45ec4 100644 --- a/spec/javascript-spec.coffee +++ b/spec/javascript-spec.coffee @@ -944,12 +944,19 @@ describe "Javascript grammar", -> it "tokenizes functions as object properties", -> {tokens} = grammar.tokenizeLine('obj.method = function foo(') expect(tokens[0]).toEqual value: 'obj', scopes: ['source.js', 'variable.other.object.js'] + expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'meta.function.js', 'meta.delimiter.method.period.js'] expect(tokens[2]).toEqual value: 'method', scopes: ['source.js', 'meta.function.js', 'entity.name.function.js'] expect(tokens[4]).toEqual value: '=', scopes: ['source.js', 'meta.function.js', 'keyword.operator.assignment.js'] expect(tokens[6]).toEqual value: 'function', scopes: ['source.js', 'meta.function.js', 'storage.type.function.js'] expect(tokens[8]).toEqual value: 'foo', scopes: ['source.js', 'meta.function.js', 'entity.name.function.js'] expect(tokens[9]).toEqual value: '(', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'punctuation.definition.parameters.begin.bracket.round.js'] + {tokens} = grammar.tokenizeLine('this.register = function(') + expect(tokens[0]).toEqual value: 'this', scopes: ['source.js', 'variable.language.js'] + expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'meta.function.js', 'meta.delimiter.method.period.js'] + expect(tokens[2]).toEqual value: 'register', scopes: ['source.js', 'meta.function.js', 'entity.name.function.js'] + expect(tokens[6]).toEqual value: 'function', scopes: ['source.js', 'meta.function.js', 'storage.type.function.js'] + it "tokenizes ES6 method definitions", -> {tokens} = grammar.tokenizeLine('f(a, b) {}') expect(tokens[0]).toEqual value: 'f', scopes: ['source.js', 'meta.function.method.definition.js', 'entity.name.function.js'] From 7165e972a7e692dd0115a3a203cca586ce95de53 Mon Sep 17 00:00:00 2001 From: Maxim Sokolov Date: Mon, 8 Feb 2016 19:46:22 +0300 Subject: [PATCH 12/21] :white_check_mark: Test arrow functions multiline parameters --- spec/javascript-spec.coffee | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/spec/javascript-spec.coffee b/spec/javascript-spec.coffee index 7fa45ec4..34da0b02 100644 --- a/spec/javascript-spec.coffee +++ b/spec/javascript-spec.coffee @@ -1053,9 +1053,6 @@ describe "Javascript grammar", -> expect(tokens[2]).toEqual value: '*', scopes: ['source.js', 'meta.function.js', 'storage.modifier.generator.js'] it "tokenizes arrow functions", -> - {tokens} = grammar.tokenizeLine('=>') - expect(tokens[0]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.js', 'storage.type.function.arrow.js'] - {tokens} = grammar.tokenizeLine('x => x * x') expect(tokens[0]).toEqual value: 'x', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'variable.parameter.function.js'] expect(tokens[2]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.js', 'storage.type.function.arrow.js'] @@ -1078,6 +1075,12 @@ describe "Javascript grammar", -> expect(tokens[9]).toEqual value: '{', scopes: ['source.js', 'punctuation.definition.function.body.begin.bracket.curly.js'] expect(tokens[10]).toEqual value: '}', scopes: ['source.js', 'punctuation.definition.function.body.end.bracket.curly.js'] + lines = grammar.tokenizeLines """ + a = (x, + y) => {} + """ + expect(lines[1][3]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.js', 'storage.type.function.arrow.js'] + it "tokenizes stored arrow functions", -> {tokens} = grammar.tokenizeLine('var func = (p1, p2) => {}') expect(tokens[0]).toEqual value: 'var', scopes: ['source.js', 'storage.type.var.js'] From 34a82b4edc42769390390f5090c3dbd1f4f9babc Mon Sep 17 00:00:00 2001 From: Maxim Sokolov Date: Mon, 8 Feb 2016 19:48:31 +0300 Subject: [PATCH 13/21] :art: --- grammars/javascript.cson | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grammars/javascript.cson b/grammars/javascript.cson index a9aaba51..dc1eff38 100644 --- a/grammars/javascript.cson +++ b/grammars/javascript.cson @@ -1194,7 +1194,7 @@ '1': 'name': 'variable.parameter.function.js' } - ] + ] 'arguments': 'patterns': [ { From f5cf7671e39ff0cb3c6026908fa355f525822c0a Mon Sep 17 00:00:00 2001 From: Maxim Sokolov Date: Tue, 9 Feb 2016 07:50:22 +0300 Subject: [PATCH 14/21] :fire: Remove unused code --- grammars/javascript.cson | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/grammars/javascript.cson b/grammars/javascript.cson index dc1eff38..a6af102b 100644 --- a/grammars/javascript.cson +++ b/grammars/javascript.cson @@ -1084,27 +1084,6 @@ ] } ] - 'arrow_functions': - 'patterns': [ - { - # (a,b) => - 'begin': '(\\()(?=(?:[^\\(\\)]*)?\\)\\s*=>)' - 'beginCaptures': - '1': - 'name': 'punctuation.definition.parameters.begin.js' - 'end': '(\\))\\s*(=>)' - 'endCaptures': - '1': - 'name': 'punctuation.definition.parameters.end.js' - '2': - 'name': 'storage.type.function.arrow.js' - 'patterns': [ - { - 'include': '#function_params' - } - ] - } - ] 'function_params': 'patterns': [ { From b14ae08da96164ad3d29535a6bdfca6ab1aa8a9d Mon Sep 17 00:00:00 2001 From: Maxim Sokolov Date: Tue, 9 Feb 2016 18:59:10 +0300 Subject: [PATCH 15/21] Allow whitespaces before line commet in arrow function --- grammars/javascript.cson | 4 ++-- spec/javascript-spec.coffee | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/grammars/javascript.cson b/grammars/javascript.cson index a6af102b..4d9046b2 100644 --- a/grammars/javascript.cson +++ b/grammars/javascript.cson @@ -467,7 +467,7 @@ \\s*{| \\G\\(| \\G[\\w$]+| - \\s*/\\*|// + \\s*/\\*|\\s*// )(?=\\s*\\S)) ''' 'patterns': [ @@ -505,7 +505,7 @@ \\s*{| \\G(\\.)?[a-zA-Z_$][\\w$]*\\s*(=)\\s*\\(| \\G(\\.)?[a-zA-Z_$][\\w$]*\\s*(=)\\s*[\\w$]+| - \\s*/\\*|// + \\s*/\\*|\\s*// )(?=\\s*\\S)) ''' 'patterns': [ diff --git a/spec/javascript-spec.coffee b/spec/javascript-spec.coffee index 34da0b02..385509b9 100644 --- a/spec/javascript-spec.coffee +++ b/spec/javascript-spec.coffee @@ -1465,13 +1465,13 @@ describe "Javascript grammar", -> lines = grammar.tokenizeLines ''' x => - // comment + // comment {} ''' expect(lines[0][0]).toEqual value: 'x', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'variable.parameter.function.js'] expect(lines[0][2]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.js', 'storage.type.function.arrow.js'] - expect(lines[1][0]).toEqual value: '//', scopes: ['source.js', 'comment.line.double-slash.js', 'punctuation.definition.comment.js'] - expect(lines[1][1]).toEqual value: ' comment', scopes: ['source.js', 'comment.line.double-slash.js'] + expect(lines[1][1]).toEqual value: '//', scopes: ['source.js', 'comment.line.double-slash.js', 'punctuation.definition.comment.js'] + expect(lines[1][2]).toEqual value: ' comment', scopes: ['source.js', 'comment.line.double-slash.js'] expect(lines[2][0]).toEqual value: '{', scopes: ['source.js', 'punctuation.definition.function.body.begin.bracket.curly.js'] it "tokenizes comments inside function parameters correctly", -> From fb802210ce2bd6ef4bd096325edf98186ae10bd9 Mon Sep 17 00:00:00 2001 From: Maxim Sokolov Date: Tue, 9 Feb 2016 19:02:39 +0300 Subject: [PATCH 16/21] :art: Replace [a-zA-Z_$0-9] -> [\\w$] --- grammars/javascript.cson | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/grammars/javascript.cson b/grammars/javascript.cson index 4d9046b2..68270d3a 100644 --- a/grammars/javascript.cson +++ b/grammars/javascript.cson @@ -422,7 +422,7 @@ import|package|return|switch|throw|try|while|with) [\\s\\(] ) - \\b[a-zA-Z_$][a-zA-Z_$0-9]* + \\b[a-zA-Z_$][\\w$]* \\s*\\(\\s* (("[^"]*")|(\\'[^\\']*\\')|((?!"|\\'|\\(|\\)).))* \\)\\s*\\{ @@ -1166,7 +1166,7 @@ 'include': '#function_params' } { - 'match': '([a-zA-Z_$][a-zA-Z_$0-9]*)(?=\\s*=>)' + 'match': '([a-zA-Z_$][\\w$]*)(?=\\s*=>)' 'captures': '0': 'name': 'meta.parameters.js' From cfcfac8b3cf5dc5e7fadd6ccc9bc21f10178e9fb Mon Sep 17 00:00:00 2001 From: Maxim Sokolov Date: Tue, 9 Feb 2016 19:07:21 +0300 Subject: [PATCH 17/21] Tokenize illegal parameters --- grammars/javascript.cson | 8 ++++++++ spec/javascript-spec.coffee | 9 +++++++++ 2 files changed, 17 insertions(+) diff --git a/grammars/javascript.cson b/grammars/javascript.cson index 68270d3a..4251c500 100644 --- a/grammars/javascript.cson +++ b/grammars/javascript.cson @@ -1173,6 +1173,14 @@ '1': 'name': 'variable.parameter.function.js' } + { + 'match': '(\\d[\\w$]*)' + 'captures': + '0': + 'name': 'meta.parameters.js' + '1': + 'name': 'invalid.illegal.identifier.js' + } ] 'arguments': 'patterns': [ diff --git a/spec/javascript-spec.coffee b/spec/javascript-spec.coffee index 385509b9..8663daa2 100644 --- a/spec/javascript-spec.coffee +++ b/spec/javascript-spec.coffee @@ -1115,6 +1115,15 @@ describe "Javascript grammar", -> expect(tokens[1]).toEqual value: '...', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'keyword.operator.spread.js'] expect(tokens[2]).toEqual value: 'args', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'variable.parameter.rest.function.js'] + it "tokenizes illegal parameters", -> + {tokens} = grammar.tokenizeLine('0abc => {}') + expect(tokens[0]).toEqual value: '0abc', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'invalid.illegal.identifier.js'] + expect(tokens[2]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.js', 'storage.type.function.arrow.js'] + + {tokens} = grammar.tokenizeLine('(0abc) => {}') + expect(tokens[1]).toEqual value: '0abc', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'invalid.illegal.identifier.js'] + expect(tokens[4]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.js', 'storage.type.function.arrow.js'] + describe "variables", -> it "tokenizes 'this'", -> {tokens} = grammar.tokenizeLine('this') From ff14ff6f5ab6cd9c7eddcbd4f568aa6f22ba7f21 Mon Sep 17 00:00:00 2001 From: Maxim Sokolov Date: Thu, 11 Feb 2016 19:41:14 +0300 Subject: [PATCH 18/21] Use begin/end match for curly brackets --- grammars/javascript.cson | 19 ++++++++++--- spec/javascript-spec.coffee | 53 ++++++++++++++++++++++++------------- 2 files changed, 50 insertions(+), 22 deletions(-) diff --git a/grammars/javascript.cson b/grammars/javascript.cson index 4251c500..5861361b 100644 --- a/grammars/javascript.cson +++ b/grammars/javascript.cson @@ -856,8 +856,19 @@ 'match': '({)(})' } { - 'match': '{|}' - 'name': 'meta.brace.curly.js' + 'begin': '{' + 'beginCaptures': + '0': + 'name': 'meta.brace.curly.js' + 'end': '}' + 'endCaptures': + '0': + 'name': 'meta.brace.curly.js' + 'patterns': [ + { + 'include': '$self' + } + ] } { 'begin': '\\(' @@ -869,7 +880,9 @@ '0': 'name': 'meta.brace.round.js' 'patterns': [ - 'include': '$self' + { + 'include': '$self' + } ] } { diff --git a/spec/javascript-spec.coffee b/spec/javascript-spec.coffee index 8663daa2..2cdef5d1 100644 --- a/spec/javascript-spec.coffee +++ b/spec/javascript-spec.coffee @@ -921,6 +921,21 @@ describe "Javascript grammar", -> expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'punctuation.definition.function.body.begin.bracket.curly.js'] expect(tokens[6]).toEqual value: '}', scopes: ['source.js', 'punctuation.definition.function.body.end.bracket.curly.js'] + lines = grammar.tokenizeLines ''' + function foo() { + if(something){ } + } + ''' + expect(lines[0][0]).toEqual value: 'function', scopes: ['source.js', 'meta.function.js', 'storage.type.function.js'] + expect(lines[0][2]).toEqual value: 'foo', scopes: ['source.js', 'meta.function.js', 'entity.name.function.js'] + expect(lines[0][3]).toEqual value: '(', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'punctuation.definition.parameters.begin.bracket.round.js'] + expect(lines[0][4]).toEqual value: ')', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'punctuation.definition.parameters.end.bracket.round.js'] + expect(lines[0][6]).toEqual value: '{', scopes: ['source.js', 'punctuation.definition.function.body.begin.bracket.curly.js'] + expect(lines[1][1]).toEqual value: 'if', scopes: ['source.js', 'keyword.control.js'] + expect(lines[1][5]).toEqual value: '{', scopes: ['source.js', 'meta.brace.curly.js'] + expect(lines[1][7]).toEqual value: '}', scopes: ['source.js', 'meta.brace.curly.js'] + expect(lines[2][0]).toEqual value: '}', scopes: ['source.js', 'punctuation.definition.function.body.end.bracket.curly.js'] + {tokens} = grammar.tokenizeLine('function $abc$(){}') expect(tokens[2]).toEqual value: '$abc$', scopes: ['source.js', 'meta.function.js', 'entity.name.function.js'] @@ -974,26 +989,26 @@ describe "Javascript grammar", -> expect(tokens[0]).toEqual value: 'hi', scopes: ['source.js', 'meta.function.method.definition.js', 'entity.name.function.js'] expect(tokens[1]).toEqual value: '(', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'punctuation.definition.parameters.begin.bracket.round.js'] expect(tokens[2]).toEqual value: '{', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'meta.brace.curly.js'] - expect(tokens[3]).toEqual value: 'host', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'variable.parameter.function.js'] + expect(tokens[3]).toEqual value: 'host', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js'] expect(tokens[4]).toEqual value: ',', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'meta.delimiter.object.comma.js'] - expect(tokens[6]).toEqual value: 'root', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'variable.parameter.function.js'] - expect(tokens[8]).toEqual value: '=', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'keyword.operator.assignment.js'] - expect(tokens[10]).toEqual value: '"', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] - expect(tokens[11]).toEqual value: './', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'string.quoted.double.js'] - expect(tokens[12]).toEqual value: '"', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'string.quoted.double.js', 'punctuation.definition.string.end.js'] - expect(tokens[13]).toEqual value: ',', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'meta.delimiter.object.comma.js'] - expect(tokens[15]).toEqual value: 'plugins', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'variable.parameter.function.js'] - expect(tokens[17]).toEqual value: '=', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'keyword.operator.assignment.js'] - expect(tokens[19]).toEqual value: '[', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'meta.brace.square.js'] - expect(tokens[20]).toEqual value: 'a', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'variable.parameter.function.js'] - expect(tokens[21]).toEqual value: ',', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'meta.delimiter.object.comma.js'] - expect(tokens[23]).toEqual value: '"', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] - expect(tokens[26]).toEqual value: ',', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'meta.delimiter.object.comma.js'] - expect(tokens[28]).toEqual value: '"', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] - expect(tokens[33]).toEqual value: 'd', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'variable.parameter.function.js'] - expect(tokens[34]).toEqual value: ']', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'meta.brace.square.js'] - expect(tokens[35]).toEqual value: '}', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'meta.brace.curly.js'] - expect(tokens[36]).toEqual value: ')', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'punctuation.definition.parameters.end.bracket.round.js'] + expect(tokens[5]).toEqual value: ' root ', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js'] + expect(tokens[6]).toEqual value: '=', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'keyword.operator.assignment.js'] + expect(tokens[8]).toEqual value: '"', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(tokens[9]).toEqual value: './', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'string.quoted.double.js'] + expect(tokens[10]).toEqual value: '"', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'string.quoted.double.js', 'punctuation.definition.string.end.js'] + expect(tokens[11]).toEqual value: ',', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'meta.delimiter.object.comma.js'] + expect(tokens[12]).toEqual value: ' plugins ', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js'] + expect(tokens[13]).toEqual value: '=', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'keyword.operator.assignment.js'] + expect(tokens[15]).toEqual value: '[', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'meta.brace.square.js'] + expect(tokens[16]).toEqual value: 'a', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js'] + expect(tokens[17]).toEqual value: ',', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'meta.delimiter.object.comma.js'] + expect(tokens[19]).toEqual value: '"', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(tokens[22]).toEqual value: ',', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'meta.delimiter.object.comma.js'] + expect(tokens[24]).toEqual value: '"', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(tokens[28]).toEqual value: ' d', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js'] + expect(tokens[29]).toEqual value: ']', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'meta.brace.square.js'] + expect(tokens[30]).toEqual value: '}', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'meta.brace.curly.js'] + expect(tokens[31]).toEqual value: ')', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'punctuation.definition.parameters.end.bracket.round.js'] {tokens} = grammar.tokenizeLine('write("){");') expect(tokens[0]).toEqual value: 'write', scopes: ['source.js', 'meta.function-call.js', 'entity.name.function.js'] From e7ca46a589eff6c9c632e8ae92b7806ef0dd6411 Mon Sep 17 00:00:00 2001 From: Maxim Sokolov Date: Sat, 20 Feb 2016 20:42:41 +0300 Subject: [PATCH 19/21] Tokenize function body properly --- grammars/javascript.cson | 84 ++++++++++++++++++++-------------------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/grammars/javascript.cson b/grammars/javascript.cson index 5861361b..03136256 100644 --- a/grammars/javascript.cson +++ b/grammars/javascript.cson @@ -286,6 +286,12 @@ 'begin': '(?=(\\basync\\b\\s*)?\\bfunction\\b)' 'end': '(?<=})' 'patterns': [ + { + 'include': '#comments' + } + { + 'include': '#function_body' + } { 'begin': '\\G' 'end': '(?<=\\))' @@ -296,12 +302,6 @@ } ] } - { - 'include': '#function_body' - } - { - 'include': '#comments' - } ] } { @@ -309,6 +309,12 @@ 'begin': '(?=(\\.)?[a-zA-Z_$][\\w$]*\\s*=\\s*(\\basync\\b\\s*)?\\bfunction\\b)' 'end': '(?<=})' 'patterns': [ + { + 'include': '#comments' + } + { + 'include': '#function_body' + } { 'begin': '\\G' 'end': '(?<=\\))' @@ -329,12 +335,6 @@ } ] } - { - 'include': '#function_body' - } - { - 'include': '#comments' - } ] } { @@ -342,6 +342,12 @@ 'begin': '(?=\\b[a-zA-Z_$][\\w$]*\\s*:\\s*(\\basync\\b\\s*)?\\bfunction\\b)' 'end': '(?<=})' 'patterns': [ + { + 'include': '#comments' + } + { + 'include': '#function_body' + } { 'begin': '\\G' 'end': '(?<=\\))' @@ -360,12 +366,6 @@ } ] } - { - 'include': '#function_body' - } - { - 'include': '#comments' - } ] } { @@ -373,6 +373,12 @@ 'begin': '(?=((\'[^\']*?\')|("[^"]*?"))\\s*:\\s*(\\basync\\b\\s*)?\\bfunction\\b)' 'end': '(?<=})' 'patterns': [ + { + 'include': '#comments' + } + { + 'include': '#function_body' + } { 'begin': '\\G' 'end': '(?<=\\))' @@ -405,12 +411,6 @@ } ] } - { - 'include': '#function_body' - } - { - 'include': '#comments' - } ] } { @@ -430,6 +430,12 @@ ''' 'end': '(?<=})' 'patterns': [ + { + 'include': '#comments' + } + { + 'include': '#function_body' + } { 'begin': '\\G' 'end': '(?<=\\))' @@ -444,12 +450,6 @@ } ] } - { - 'include': '#function_body' - } - { - 'include': '#comments' - } ] } { @@ -471,6 +471,12 @@ )(?=\\s*\\S)) ''' 'patterns': [ + { + 'include': '#comments' + } + { + 'include': '#function_body' + } { 'begin': '\\G' 'end': '(?<=(=>))' @@ -481,12 +487,6 @@ } ] } - { - 'include': '#function_body' - } - { - 'include': '#comments' - } ] } { @@ -509,6 +509,12 @@ )(?=\\s*\\S)) ''' 'patterns': [ + { + 'include': '#comments' + } + { + 'include': '#function_body' + } { 'begin': '\\G' 'end': '(?<=(=>))' @@ -529,12 +535,6 @@ } ] } - { - 'include': '#function_body' - } - { - 'include': '#comments' - } ] } { From f141d73ae6c9700781bd2d7cc5d3ef562962591e Mon Sep 17 00:00:00 2001 From: Maxim Sokolov Date: Wed, 24 Feb 2016 16:04:45 +0300 Subject: [PATCH 20/21] Tokenize arrow functions in object literals --- grammars/javascript.cson | 104 ++++++++++++++++++++++++++++++++++++ spec/javascript-spec.coffee | 13 +++++ 2 files changed, 117 insertions(+) diff --git a/grammars/javascript.cson b/grammars/javascript.cson index 03136256..ea2680b4 100644 --- a/grammars/javascript.cson +++ b/grammars/javascript.cson @@ -537,6 +537,110 @@ } ] } + { + # foo: ... => ... + 'begin': '''(?x) + (?= + \\b[a-zA-Z_$][\\w$]* + \\s*:\\s* + ((\\(([^\\(\\)]*)?\\))|[\\w$]+) + \\s*=> + ) + ''' + 'end': '''(?x) + (?<=})| + ((?! + \\s*{| + \\G[\\w$]+:| + \\s*/\\*|\\s*// + )(?=\\s*\\S)) + ''' + 'patterns': [ + { + 'include': '#comments' + } + { + 'include': '#function_body' + } + { + 'begin': '\\G' + 'end': '(?<=(=>))' + 'name': 'meta.function.arrow.json.js' + 'patterns': [ + { + 'match': '\\b([a-zA-Z_$][\\w$]*)\\s*(:)\\s*' + 'captures': + '1': + 'name': 'entity.name.function.js' + '2': + 'name': 'keyword.operator.assignment.js' + } + { + 'include': '#arrow_function_innards' + } + ] + } + ] + } + { + # "foo": ... => ... + 'begin': '''(?x) + (?= + ((\'[^\']*?\')|("[^"]*?")) + \\s*:\\s* + ((\\(([^\\(\\)]*)?\\))|[\\w$]+) + \\s*=> + ) + ''' + 'end': '''(?x) + (?<=})| + ((?! + \\G((\'[^\']*?\')|("[^"]*?"))| + \\s*{| + \\s*/\\*|\\s*// + )(?=\\s*\\S)) + ''' + 'patterns': [ + { + 'include': '#comments' + } + { + 'include': '#function_body' + } + { + 'begin': '\\G' + 'end': '(?<=(=>))' + 'name': 'meta.function.arrow.json.js' + 'patterns': [ + { + 'match': '(?:((\')([^\']*?)(\'))|((")([^"]*?)(")))\\s*(:)' + 'captures': + '1': + 'name': 'string.quoted.single.js' + '2': + 'name': 'punctuation.definition.string.begin.js' + '3': + 'name': 'entity.name.function.js' + '4': + 'name': 'punctuation.definition.string.end.js' + '5': + 'name': 'string.quoted.double.js' + '6': + 'name': 'punctuation.definition.string.begin.js' + '7': + 'name': 'entity.name.function.js' + '8': + 'name': 'punctuation.definition.string.end.js' + '9': + 'name': 'keyword.operator.assignment.js' + } + { + 'include': '#arrow_function_innards' + } + ] + } + ] + } { 'match': '(=>)' 'captures': diff --git a/spec/javascript-spec.coffee b/spec/javascript-spec.coffee index 2cdef5d1..b35a8328 100644 --- a/spec/javascript-spec.coffee +++ b/spec/javascript-spec.coffee @@ -1112,6 +1112,19 @@ describe "Javascript grammar", -> expect(tokens[11]).toEqual value: ')', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'punctuation.definition.parameters.end.bracket.round.js'] expect(tokens[13]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.js', 'storage.type.function.arrow.js'] + it "tokenizes arrow functions in object literals", -> + {tokens} = grammar.tokenizeLine('foo: param => {}') + expect(tokens[0]).toEqual value: 'foo', scopes: ['source.js', 'meta.function.arrow.json.js', 'entity.name.function.js'] + expect(tokens[1]).toEqual value: ':', scopes: ['source.js', 'meta.function.arrow.json.js', 'keyword.operator.assignment.js'] + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'meta.function.arrow.json.js', 'meta.parameters.js', 'variable.parameter.function.js'] + expect(tokens[5]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.json.js', 'storage.type.function.arrow.js'] + + {tokens} = grammar.tokenizeLine('"foo": param => {}') + expect(tokens[1]).toEqual value: 'foo', scopes: ['source.js', 'meta.function.arrow.json.js', 'string.quoted.double.js', 'entity.name.function.js'] + expect(tokens[3]).toEqual value: ':', scopes: ['source.js', 'meta.function.arrow.json.js', 'keyword.operator.assignment.js'] + expect(tokens[5]).toEqual value: 'param', scopes: ['source.js', 'meta.function.arrow.json.js', 'meta.parameters.js', 'variable.parameter.function.js'] + expect(tokens[7]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.json.js', 'storage.type.function.arrow.js'] + it "tokenizes default parameters", -> {tokens} = grammar.tokenizeLine('function multiply(a, b = 1){}') expect(tokens[7]).toEqual value: 'b', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'variable.parameter.function.js'] From adff2b27b3862a01cee1494cbe9d719fdd795efe Mon Sep 17 00:00:00 2001 From: Maxim Sokolov Date: Sun, 28 Feb 2016 22:49:38 +0300 Subject: [PATCH 21/21] :art: --- grammars/javascript.cson | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/grammars/javascript.cson b/grammars/javascript.cson index ea2680b4..9fb014ec 100644 --- a/grammars/javascript.cson +++ b/grammars/javascript.cson @@ -1215,7 +1215,7 @@ 'name': 'meta.parameters.js' 'patterns': [ { - 'match': '(\\.\\.\\.)([a-zA-Z_$][a-zA-Z_$0-9]*)' + 'match': '(\\.\\.\\.)([a-zA-Z_$][\\w$]*)' 'captures': '1': 'name': 'keyword.operator.spread.js' @@ -1226,7 +1226,7 @@ 'include': '$self' } { - 'match': '[a-zA-Z_$][a-zA-Z_$0-9]*' + 'match': '[a-zA-Z_$][\\w$]*' 'name': 'variable.parameter.function.js' } ]