From 9f6a8acba4b3538e6679b55d2bab8a9289922e8b Mon Sep 17 00:00:00 2001 From: Alhadis Date: Sun, 21 Aug 2016 00:53:35 +1000 Subject: [PATCH 01/16] Add pattern-matching for Unicode escape sequences --- grammars/javascript.cson | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/grammars/javascript.cson b/grammars/javascript.cson index 710f83d8..b9e93f81 100644 --- a/grammars/javascript.cson +++ b/grammars/javascript.cson @@ -1214,8 +1214,7 @@ 'name': 'string.quoted.single.js' 'patterns': [ { - 'match': '\\\\(x\\h{2}|[0-2][0-7]{0,2}|3[0-6][0-7]?|37[0-7]?|[4-7][0-7]?|.)' - 'name': 'constant.character.escape.js' + 'include': '#string_escapes' } { 'match': "[^']*[^\\n\\r'\\\\]$" @@ -1235,8 +1234,7 @@ 'name': 'string.quoted.double.js' 'patterns': [ { - 'match': '\\\\(x\\h{2}|[0-2][0-7]{0,2}|3[0-6][0-7]|37[0-7]?|[4-7][0-7]?|.)' - 'name': 'constant.character.escape.js' + 'include': '#string_escapes' } { 'match': '[^"]*[^\\n\\r"\\\\]$' @@ -1258,8 +1256,7 @@ 'name': 'string.quoted.template.html.js' 'patterns': [ { - 'match': '\\\\(x\\h{2}|[0-2][0-7]{0,2}|3[0-6][0-7]?|37[0-7]?|[4-7][0-7]?|.)' - 'name': 'constant.character.escape.js' + 'include': '#string_escapes' } { 'include': '#interpolated_js' @@ -1283,8 +1280,7 @@ 'name': 'string.quoted.template.graphql.js' 'patterns': [ { - 'match': '\\\\(x\\h{2}|[0-2][0-7]{0,2}|3[0-6][0-7]?|37[0-7]?|[4-7][0-7]?|.)' - 'name': 'constant.character.escape.js' + 'include': '#string_escapes' } { 'include': '#interpolated_js' @@ -1306,8 +1302,7 @@ 'name': 'string.quoted.template.js' 'patterns': [ { - 'match': '\\\\(x\\h{2}|[0-2][0-7]{0,2}|3[0-6][0-7]?|37[0-7]?|[4-7][0-7]?|.)' - 'name': 'constant.character.escape.js' + 'include': '#string_escapes' } { 'include': '#interpolated_js' @@ -1315,6 +1310,26 @@ ] } ] + 'string_escapes': + 'patterns': [ + { + 'match': '\\\\u(?![A-Fa-f0-9]{4}|\\{[A-Fa-f0-9]+\\})[^\'"]*' + 'name': 'invalid.illegal.identifier.js' + } + { + 'match': '\\\\u(?:[A-Fa-f0-9]{4}|(\\{)[A-Fa-f0-9]+(\\}))' + 'name': 'constant.character.escape.js' + 'captures': + '1': + 'name': 'punctuation.section.scope.begin.js' + '2': + 'name': 'punctuation.section.scope.end.js' + } + { + 'match': '\\\\(x\\h{2}|[0-2][0-7]{0,2}|3[0-6][0-7]?|37[0-7]?|[4-7][0-7]?|.)' + 'name': 'constant.character.escape.js' + } + ] 'function_params': 'patterns': [ { From a208ceb10f6fd154b2798e0197ccaa26e864f74b Mon Sep 17 00:00:00 2001 From: Alhadis Date: Sun, 21 Aug 2016 01:15:49 +1000 Subject: [PATCH 02/16] Add error highlighting to codepoints over U+10FFFF --- grammars/javascript.cson | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/grammars/javascript.cson b/grammars/javascript.cson index b9e93f81..20c28b4e 100644 --- a/grammars/javascript.cson +++ b/grammars/javascript.cson @@ -1317,12 +1317,20 @@ 'name': 'invalid.illegal.identifier.js' } { - 'match': '\\\\u(?:[A-Fa-f0-9]{4}|(\\{)[A-Fa-f0-9]+(\\}))' + 'match': '\\\\u(?:[A-Fa-f0-9]{4}|(\\{)([A-Fa-f0-9]+)(\\}))' 'name': 'constant.character.escape.js' 'captures': '1': 'name': 'punctuation.section.scope.begin.js' '2': + 'patterns': [ + { + # Max codepoint: \u{10FFFF} + 'match': '[A-Fa-f\\d]{7,}|(?!10)[A-Fa-f\\d]{6}' + 'name': 'invalid.illegal.identifier.js' + } + ] + '3': 'name': 'punctuation.section.scope.end.js' } { From b04283b4d847bc78c5ece6dcba65e72ff5e4c638 Mon Sep 17 00:00:00 2001 From: Alhadis Date: Tue, 23 Aug 2016 14:55:05 +1000 Subject: [PATCH 03/16] Allow whitespace in "support.constant" accessors This commit fixes a minor bug with scope-matching: window.innerWidth # Ends in support.constant.js window. innerWidth # Doesn't Both lines are nonetheless identical in JavaScript. --- grammars/javascript.cson | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/grammars/javascript.cson b/grammars/javascript.cson index 0b8a1989..2ec05e53 100644 --- a/grammars/javascript.cson +++ b/grammars/javascript.cson @@ -977,7 +977,7 @@ 'name': 'support.class.js' } { - 'match': '''(?x) (\\.) + 'match': '''(?x) (\\.) \\s* (MAX_VALUE|MIN_VALUE|NEGATIVE_INFINITY|POSITIVE_INFINITY|URLUnencoded|X[MS]LDocument|[xyz]|_content|aLinkcolor |above|all|appCodeName|appCore|appMinorVersion|appName|appVersion|arguments|arity|availHeight|availLeft|availTop |availWidth|backgroundColor|backgroundImage|below|borderBottomWidth|borderColor|borderLeftWidth|borderRightWidth @@ -1005,7 +1005,7 @@ 'name': 'support.constant.js' } { - 'match': '''(?x) (\\.) + 'match': '''(?x) (\\.) \\s* (abbr|accept|acceptCharset|accessKey|action|align|aLink|alt|anchors|applets|archive|areas|attributes|axis|background |bgColor|body|border|caption|cells|cellPadding|cellSpacing|ch|charset|checked|childNodes|chOff|cite|className|clear |code|codeBase|codeType|color|cols|colSpan|compact|content|cookie|cords|data|dateTime|declare|defaultChecked From fb9f7d8f2df8bef2a076cdce8e8a501d74d2f0da Mon Sep 17 00:00:00 2001 From: Alhadis Date: Tue, 23 Aug 2016 17:17:20 +1000 Subject: [PATCH 04/16] Update and correctly classify "support" keywords MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Added the full list of standardised web APIs from MDN: https://developer.mozilla.org/en-US/docs/Web/API * Added support for less heard-of ES6+ value types (ArrayBuffer, SIMD …) * Added "dom" scope for DOM-specific interfaces and objects * Removed highlighting for non-standard/deprecated properties/classes --- grammars/javascript.cson | 161 ++++++++++++++++++++++++++---------- spec/javascript-spec.coffee | 8 +- 2 files changed, 120 insertions(+), 49 deletions(-) diff --git a/grammars/javascript.cson b/grammars/javascript.cson index 2ec05e53..a23ac152 100644 --- a/grammars/javascript.cson +++ b/grammars/javascript.cson @@ -968,72 +968,143 @@ } { 'match': '''(?x) (? describe "built-in globals", -> it "tokenizes built-in classes", -> {tokens} = grammar.tokenizeLine('window') - expect(tokens[0]).toEqual value: 'window', scopes: ['source.js', 'support.class.js'] + expect(tokens[0]).toEqual value: 'window', scopes: ['source.js', 'support.variable.dom.js'] {tokens} = grammar.tokenizeLine('window.name') - expect(tokens[0]).toEqual value: 'window', scopes: ['source.js', 'support.class.js'] + expect(tokens[0]).toEqual value: 'window', scopes: ['source.js', 'support.variable.dom.js'] {tokens} = grammar.tokenizeLine('$window') expect(tokens[0]).toEqual value: '$window', scopes: ['source.js'] @@ -515,12 +515,12 @@ describe "Javascript grammar", -> expect(tokens[4]).toEqual value: '=', scopes: ['source.js', 'keyword.operator.assignment.js'] it "tokenizes support constants", -> - {tokens} = grammar.tokenizeLine('awesome = cool.systemLanguage;') + {tokens} = grammar.tokenizeLine('awesome = cool.EPSILON;') expect(tokens[0]).toEqual value: 'awesome ', scopes: ['source.js'] expect(tokens[1]).toEqual value: '=', scopes: ['source.js', 'keyword.operator.assignment.js'] expect(tokens[3]).toEqual value: 'cool', scopes: ['source.js', 'variable.other.object.js'] expect(tokens[4]).toEqual value: '.', scopes: ['source.js', 'meta.delimiter.property.period.js'] - expect(tokens[5]).toEqual value: 'systemLanguage', scopes: ['source.js', 'support.constant.js'] + expect(tokens[5]).toEqual value: 'EPSILON', scopes: ['source.js', 'support.constant.js'] expect(tokens[6]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js'] it "does not tokenize constants when they are object keys", -> From 2fac2bd8482282f6593086d7ff9dd54478ff3318 Mon Sep 17 00:00:00 2001 From: Alhadis Date: Tue, 23 Aug 2016 18:11:48 +1000 Subject: [PATCH 05/16] Fix bugs with matching dollar-signs in constants Fixes #409. --- grammars/javascript.cson | 6 +++++- spec/javascript-spec.coffee | 6 ++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/grammars/javascript.cson b/grammars/javascript.cson index a23ac152..0844efc1 100644 --- a/grammars/javascript.cson +++ b/grammars/javascript.cson @@ -1161,7 +1161,11 @@ 'include': '#properties' } { - 'match': '(? {tokens} = grammar.tokenizeLine('OBJ.prop') expect(tokens[0]).toEqual value: 'OBJ', scopes: ['source.js', 'constant.other.object.js'] + it "tokenises constants with dollar signs", -> + constants = ['CON$TANT', 'ABC$', 'ABC$D', '$_ALL', 'ALL_$', 'ANGULAR$$$$$$$$$$$'] + for constant in constants + {tokens} = grammar.tokenizeLine(constant) + expect(tokens[0]).toEqual value: constant, scopes: ['source.js', 'constant.other.js'] + it "tokenizes variables declared using `const` as constants", -> {tokens} = grammar.tokenizeLine('const myCoolVar = 42;') expect(tokens[0]).toEqual value: 'const', scopes: ['source.js', 'storage.modifier.js'] From de3212648440ae367f9c19ddc4c9eb8334dd2b33 Mon Sep 17 00:00:00 2001 From: Alhadis Date: Tue, 23 Aug 2016 18:19:16 +1000 Subject: [PATCH 06/16] Avoid matching "this" in "$this" Fixes #344. --- grammars/javascript.cson | 2 +- spec/javascript-spec.coffee | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/grammars/javascript.cson b/grammars/javascript.cson index 0844efc1..fb47c6ed 100644 --- a/grammars/javascript.cson +++ b/grammars/javascript.cson @@ -280,7 +280,7 @@ } { 'match': '''(?x) - (? {tokens} = grammar.tokenizeLine('this.obj.prototype = new El()') expect(tokens[0]).toEqual value: 'this', scopes: ['source.js', 'variable.language.js'] + {tokens} = grammar.tokenizeLine('$this') + expect(tokens[0].value).toEqual '$this' + + {tokens} = grammar.tokenizeLine('this$') + expect(tokens[0].value).toEqual 'this$' + it "tokenizes 'super'", -> {tokens} = grammar.tokenizeLine('super') expect(tokens[0]).toEqual value: 'super', scopes: ['source.js', 'variable.language.js'] From fb456ca9411b19aff3f417da00923df6023c29e0 Mon Sep 17 00:00:00 2001 From: Alhadis Date: Tue, 23 Aug 2016 19:09:48 +1000 Subject: [PATCH 07/16] Fix bugs with matching instantiation patterns * Fixes inconsistent scopes with full-stops (closes #280) * Fixes "new obj.ct.Class();" stopping at "new obj.ct" * Fixes "new obj.ct.Cla$$();" not matching due to dollar signs --- grammars/javascript.cson | 10 ++++++++-- spec/javascript-spec.coffee | 18 ++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/grammars/javascript.cson b/grammars/javascript.cson index fb47c6ed..b50b3198 100644 --- a/grammars/javascript.cson +++ b/grammars/javascript.cson @@ -717,13 +717,19 @@ 'name': 'meta.class.js' } { + 'match': '(new)\\s+([\\w$]+[\\w.$]*)' + 'name': 'meta.class.instance.constructor' 'captures': '1': 'name': 'keyword.operator.new.js' '2': 'name': 'entity.name.type.instance.js' - 'match': '(new)\\s+(\\$?\\w+(?:\\.\\w*)?)' - 'name': 'meta.class.instance.constructor' + 'patterns':[ + { + 'match': '\\.' + 'name': 'meta.delimiter.property.period.js' + } + ] } { # console diff --git a/spec/javascript-spec.coffee b/spec/javascript-spec.coffee index 5f415291..5fbd9e3d 100644 --- a/spec/javascript-spec.coffee +++ b/spec/javascript-spec.coffee @@ -127,6 +127,24 @@ describe "Javascript grammar", -> expect(tokens[1]).toEqual value: ' ', scopes: ['source.js', 'meta.class.instance.constructor'] expect(tokens[2]).toEqual value: '$something', scopes: ['source.js', 'meta.class.instance.constructor', 'entity.name.type.instance.js'] + {tokens} = grammar.tokenizeLine('var instance = new obj.ct.Cla$s();') + expect(tokens).toEqual [ + {value: 'var', scopes: ['source.js', 'storage.type.var.js']} + {value: ' instance ', scopes: ['source.js']} + {value: '=', scopes: ['source.js', 'keyword.operator.assignment.js']} + {value: ' ', scopes: ['source.js']} + {value: 'new', scopes: ['source.js', 'meta.class.instance.constructor', 'keyword.operator.new.js']} + {value: ' ', scopes: ['source.js', 'meta.class.instance.constructor']} + {value: 'obj', scopes: ['source.js', 'meta.class.instance.constructor', 'entity.name.type.instance.js']} + {value: '.', scopes: ['source.js', 'meta.class.instance.constructor', 'entity.name.type.instance.js', 'meta.delimiter.property.period.js']} + {value: 'ct', scopes: ['source.js', 'meta.class.instance.constructor', 'entity.name.type.instance.js']} + {value: '.', scopes: ['source.js', 'meta.class.instance.constructor', 'entity.name.type.instance.js', 'meta.delimiter.property.period.js']} + {value: 'Cla$s', scopes: ['source.js', 'meta.class.instance.constructor', 'entity.name.type.instance.js']} + {value: '(', scopes: ['source.js', 'meta.brace.round.js']} + {value: ')', scopes: ['source.js', 'meta.brace.round.js']} + {value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js']} + ] + describe "regular expressions", -> it "tokenizes regular expressions", -> {tokens} = grammar.tokenizeLine('/test/') From 50b02a4c35216ebe39eadf1afa6a619ee832a1f6 Mon Sep 17 00:00:00 2001 From: Alhadis Date: Tue, 23 Aug 2016 19:16:27 +1000 Subject: [PATCH 08/16] Allow solitary majuscules to match as constants Fixes #413. --- grammars/javascript.cson | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/grammars/javascript.cson b/grammars/javascript.cson index b50b3198..56cc3c01 100644 --- a/grammars/javascript.cson +++ b/grammars/javascript.cson @@ -1168,9 +1168,9 @@ } { 'match': '''(?x) - ((? Date: Tue, 23 Aug 2016 20:24:55 +1000 Subject: [PATCH 09/16] Add specs for 9f6a8ac and a208ceb --- spec/javascript-spec.coffee | 49 +++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/spec/javascript-spec.coffee b/spec/javascript-spec.coffee index 5fbd9e3d..c70033a0 100644 --- a/spec/javascript-spec.coffee +++ b/spec/javascript-spec.coffee @@ -54,6 +54,55 @@ describe "Javascript grammar", -> expect(lines[2][0]).toEqual value: 'line3', scopes: ['source.js', scope] expect(lines[2][1]).toEqual value: delim, scopes: ['source.js', scope, 'punctuation.definition.string.end.js'] + describe "Unicode escape sequences", -> + delimsByScope = + "string.quoted.double.js": '"' + "string.quoted.single.js": "'" + + it "tokenises 2-digit sequences", -> + for scope, quote of delimsByScope + {tokens} = grammar.tokenizeLine(quote + '\\x2011' + quote) + expect(tokens[0]).toEqual value: quote, scopes: ['source.js', scope, 'punctuation.definition.string.begin.js'] + expect(tokens[1]).toEqual value: '\\x20', scopes: ['source.js', scope, 'constant.character.escape.js'] + expect(tokens[2]).toEqual value: '11', scopes: ['source.js', scope] + expect(tokens[3]).toEqual value: quote, scopes: ['source.js', scope, 'punctuation.definition.string.end.js'] + + it "tokenises 4-digit sequences", -> + for scope, quote of delimsByScope + {tokens} = grammar.tokenizeLine(quote + '\\u2011' + quote) + expect(tokens[0]).toEqual value: quote, scopes: ['source.js', scope, 'punctuation.definition.string.begin.js'] + expect(tokens[1]).toEqual value: '\\u2011', scopes: ['source.js', scope, 'constant.character.escape.js'] + expect(tokens[2]).toEqual value: quote, scopes: ['source.js', scope, 'punctuation.definition.string.end.js'] + + it "tokenises variable-length sequences", -> + for scope, quote of delimsByScope + {tokens} = grammar.tokenizeLine(quote + '\\u{2000}' + quote) + expect(tokens[0]).toEqual value: quote, scopes: ['source.js', scope, 'punctuation.definition.string.begin.js'] + expect(tokens[1]).toEqual value: '\\u', scopes: ['source.js', scope, 'constant.character.escape.js'] + expect(tokens[2]).toEqual value: '{', scopes: ['source.js', scope, 'constant.character.escape.js', 'punctuation.section.scope.begin.js'] + expect(tokens[3]).toEqual value: '2000', scopes: ['source.js', scope, 'constant.character.escape.js'] + expect(tokens[4]).toEqual value: '}', scopes: ['source.js', scope, 'constant.character.escape.js', 'punctuation.section.scope.end.js'] + expect(tokens[5]).toEqual value: quote, scopes: ['source.js', scope, 'punctuation.definition.string.end.js'] + + it "highlights sequences with invalid syntax", -> + for invalid in ['\\u', '\\u{2000', '\\u{G}'] + {tokens} = grammar.tokenizeLine('"' + invalid + '"') + expect(tokens[1]).toEqual value: invalid, scopes: ['source.js', 'string.quoted.double.js', 'invalid.illegal.identifier.js'] + + it "highlights sequences with invalid codepoints", -> + maxCodepoint = 0x10FFFF + for codepoint in [0x5000, 0x11FFFF, 0x1000000, maxCodepoint] + pointStr = codepoint.toString(16).toUpperCase().replace(/^0x/, "") + {tokens} = grammar.tokenizeLine('"\\u{' + pointStr + '}"') + pointScopes = ['source.js', 'string.quoted.double.js', 'constant.character.escape.js'] + if codepoint > maxCodepoint then pointScopes.push 'invalid.illegal.identifier.js' + expect(tokens[0]).toEqual value: '"', scopes: ['source.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(tokens[1]).toEqual value: '\\u', scopes: ['source.js', 'string.quoted.double.js', 'constant.character.escape.js'] + expect(tokens[2]).toEqual value: '{', scopes: ['source.js', 'string.quoted.double.js', 'constant.character.escape.js', 'punctuation.section.scope.begin.js'] + expect(tokens[3]).toEqual value: pointStr, scopes: pointScopes + expect(tokens[4]).toEqual value: '}', scopes: ['source.js', 'string.quoted.double.js', 'constant.character.escape.js', 'punctuation.section.scope.end.js'] + expect(tokens[5]).toEqual value: '"', scopes: ['source.js', 'string.quoted.double.js', 'punctuation.definition.string.end.js'] + describe "keywords", -> it "tokenizes with as a keyword", -> {tokens} = grammar.tokenizeLine('with') From d95b0ea8758effc8cfaff22f53b991807064a1d9 Mon Sep 17 00:00:00 2001 From: Alhadis Date: Tue, 23 Aug 2016 20:33:37 +1000 Subject: [PATCH 10/16] Avoid matching constants without majuscules --- grammars/javascript.cson | 2 +- spec/javascript-spec.coffee | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/grammars/javascript.cson b/grammars/javascript.cson index 56cc3c01..f9a47b64 100644 --- a/grammars/javascript.cson +++ b/grammars/javascript.cson @@ -1168,7 +1168,7 @@ } { 'match': '''(?x) - ((? {tokens} = grammar.tokenizeLine(constant) expect(tokens[0]).toEqual value: constant, scopes: ['source.js', 'constant.other.js'] + it "doesn't tokenise constants without alphabetic characters", -> + for name in ['$_', '$', '_', '$_$_$_$___$___$____$'] + {tokens} = grammar.tokenizeLine(name) + expect(tokens[0]).toEqual value: name, scopes: ['source.js'] + it "tokenizes variables declared using `const` as constants", -> {tokens} = grammar.tokenizeLine('const myCoolVar = 42;') expect(tokens[0]).toEqual value: 'const', scopes: ['source.js', 'storage.modifier.js'] From 0ee586487b00ef3ea506669d819b26f06f363d76 Mon Sep 17 00:00:00 2001 From: Alhadis Date: Tue, 23 Aug 2016 21:25:01 +1000 Subject: [PATCH 11/16] Use more specific scope-names when possible --- grammars/javascript.cson | 12 ++++++------ spec/javascript-spec.coffee | 16 ++++++++++------ 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/grammars/javascript.cson b/grammars/javascript.cson index f9a47b64..98cc9a14 100644 --- a/grammars/javascript.cson +++ b/grammars/javascript.cson @@ -1443,25 +1443,25 @@ 'string_escapes': 'patterns': [ { - 'match': '\\\\u(?![A-Fa-f0-9]{4}|\\{[A-Fa-f0-9]+\\})[^\'"]*' - 'name': 'invalid.illegal.identifier.js' + 'match': '\\\\u(?![A-Fa-f0-9]{4}|{[A-Fa-f0-9]+})[^\'"]*' + 'name': 'invalid.illegal.unicode-escape.js' } { - 'match': '\\\\u(?:[A-Fa-f0-9]{4}|(\\{)([A-Fa-f0-9]+)(\\}))' + 'match': '\\\\u(?:[A-Fa-f0-9]{4}|({)([A-Fa-f0-9]+)(}))' 'name': 'constant.character.escape.js' 'captures': '1': - 'name': 'punctuation.section.scope.begin.js' + 'name': 'punctuation.definition.unicode-escape.begin.bracket.curly.js' '2': 'patterns': [ { # Max codepoint: \u{10FFFF} 'match': '[A-Fa-f\\d]{7,}|(?!10)[A-Fa-f\\d]{6}' - 'name': 'invalid.illegal.identifier.js' + 'name': 'invalid.illegal.unicode-escape.js' } ] '3': - 'name': 'punctuation.section.scope.end.js' + 'name': 'punctuation.definition.unicode-escape.end.bracket.curly.js' } { 'match': '\\\\(x\\h{2}|[0-2][0-7]{0,2}|3[0-6][0-7]?|37[0-7]?|[4-7][0-7]?|.)' diff --git a/spec/javascript-spec.coffee b/spec/javascript-spec.coffee index 36667549..5e9151c1 100644 --- a/spec/javascript-spec.coffee +++ b/spec/javascript-spec.coffee @@ -55,6 +55,10 @@ describe "Javascript grammar", -> expect(lines[2][1]).toEqual value: delim, scopes: ['source.js', scope, 'punctuation.definition.string.end.js'] describe "Unicode escape sequences", -> + bracketScopes = [ + 'punctuation.definition.unicode-escape.begin.bracket.curly.js', + 'punctuation.definition.unicode-escape.end.bracket.curly.js' + ] delimsByScope = "string.quoted.double.js": '"' "string.quoted.single.js": "'" @@ -79,15 +83,15 @@ describe "Javascript grammar", -> {tokens} = grammar.tokenizeLine(quote + '\\u{2000}' + quote) expect(tokens[0]).toEqual value: quote, scopes: ['source.js', scope, 'punctuation.definition.string.begin.js'] expect(tokens[1]).toEqual value: '\\u', scopes: ['source.js', scope, 'constant.character.escape.js'] - expect(tokens[2]).toEqual value: '{', scopes: ['source.js', scope, 'constant.character.escape.js', 'punctuation.section.scope.begin.js'] + expect(tokens[2]).toEqual value: '{', scopes: ['source.js', scope, 'constant.character.escape.js', bracketScopes[0]] expect(tokens[3]).toEqual value: '2000', scopes: ['source.js', scope, 'constant.character.escape.js'] - expect(tokens[4]).toEqual value: '}', scopes: ['source.js', scope, 'constant.character.escape.js', 'punctuation.section.scope.end.js'] + expect(tokens[4]).toEqual value: '}', scopes: ['source.js', scope, 'constant.character.escape.js', bracketScopes[1]] expect(tokens[5]).toEqual value: quote, scopes: ['source.js', scope, 'punctuation.definition.string.end.js'] it "highlights sequences with invalid syntax", -> for invalid in ['\\u', '\\u{2000', '\\u{G}'] {tokens} = grammar.tokenizeLine('"' + invalid + '"') - expect(tokens[1]).toEqual value: invalid, scopes: ['source.js', 'string.quoted.double.js', 'invalid.illegal.identifier.js'] + expect(tokens[1]).toEqual value: invalid, scopes: ['source.js', 'string.quoted.double.js', 'invalid.illegal.unicode-escape.js'] it "highlights sequences with invalid codepoints", -> maxCodepoint = 0x10FFFF @@ -95,12 +99,12 @@ describe "Javascript grammar", -> pointStr = codepoint.toString(16).toUpperCase().replace(/^0x/, "") {tokens} = grammar.tokenizeLine('"\\u{' + pointStr + '}"') pointScopes = ['source.js', 'string.quoted.double.js', 'constant.character.escape.js'] - if codepoint > maxCodepoint then pointScopes.push 'invalid.illegal.identifier.js' + if codepoint > maxCodepoint then pointScopes.push 'invalid.illegal.unicode-escape.js' expect(tokens[0]).toEqual value: '"', scopes: ['source.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] expect(tokens[1]).toEqual value: '\\u', scopes: ['source.js', 'string.quoted.double.js', 'constant.character.escape.js'] - expect(tokens[2]).toEqual value: '{', scopes: ['source.js', 'string.quoted.double.js', 'constant.character.escape.js', 'punctuation.section.scope.begin.js'] + expect(tokens[2]).toEqual value: '{', scopes: ['source.js', 'string.quoted.double.js', 'constant.character.escape.js', bracketScopes[0]] expect(tokens[3]).toEqual value: pointStr, scopes: pointScopes - expect(tokens[4]).toEqual value: '}', scopes: ['source.js', 'string.quoted.double.js', 'constant.character.escape.js', 'punctuation.section.scope.end.js'] + expect(tokens[4]).toEqual value: '}', scopes: ['source.js', 'string.quoted.double.js', 'constant.character.escape.js', bracketScopes[1]] expect(tokens[5]).toEqual value: '"', scopes: ['source.js', 'string.quoted.double.js', 'punctuation.definition.string.end.js'] describe "keywords", -> From fc8560ee6f03b8de6386984989ab883c2caf0d42 Mon Sep 17 00:00:00 2001 From: Alhadis Date: Tue, 23 Aug 2016 22:16:14 +1000 Subject: [PATCH 12/16] Fix failing specs --- grammars/javascript.cson | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/grammars/javascript.cson b/grammars/javascript.cson index f76b30e7..59d79cc4 100644 --- a/grammars/javascript.cson +++ b/grammars/javascript.cson @@ -279,7 +279,7 @@ ] } { - 'match': '(? Date: Wed, 24 Aug 2016 03:34:09 +1000 Subject: [PATCH 13/16] Add missing space and scope-name --- grammars/javascript.cson | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/grammars/javascript.cson b/grammars/javascript.cson index 59d79cc4..3195438f 100644 --- a/grammars/javascript.cson +++ b/grammars/javascript.cson @@ -706,13 +706,13 @@ } { 'match': '(new)\\s+([\\w$]+[\\w.$]*)' - 'name': 'meta.class.instance.constructor' + 'name': 'meta.class.instance.constructor.js' 'captures': '1': 'name': 'keyword.operator.new.js' '2': 'name': 'entity.name.type.instance.js' - 'patterns':[ + 'patterns': [ { 'match': '\\.' 'name': 'meta.delimiter.property.period.js' From 70a1a68af70cedf62e2cd0ddcab978cf00b187c9 Mon Sep 17 00:00:00 2001 From: Alhadis Date: Wed, 24 Aug 2016 03:37:06 +1000 Subject: [PATCH 14/16] Remove whitespace in specs --- spec/javascript-spec.coffee | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/spec/javascript-spec.coffee b/spec/javascript-spec.coffee index d9db9d11..34e783c7 100644 --- a/spec/javascript-spec.coffee +++ b/spec/javascript-spec.coffee @@ -66,27 +66,27 @@ describe "Javascript grammar", -> it "tokenises 2-digit sequences", -> for scope, quote of delimsByScope {tokens} = grammar.tokenizeLine(quote + '\\x2011' + quote) - expect(tokens[0]).toEqual value: quote, scopes: ['source.js', scope, 'punctuation.definition.string.begin.js'] + expect(tokens[0]).toEqual value: quote, scopes: ['source.js', scope, 'punctuation.definition.string.begin.js'] expect(tokens[1]).toEqual value: '\\x20', scopes: ['source.js', scope, 'constant.character.escape.js'] - expect(tokens[2]).toEqual value: '11', scopes: ['source.js', scope] - expect(tokens[3]).toEqual value: quote, scopes: ['source.js', scope, 'punctuation.definition.string.end.js'] + expect(tokens[2]).toEqual value: '11', scopes: ['source.js', scope] + expect(tokens[3]).toEqual value: quote, scopes: ['source.js', scope, 'punctuation.definition.string.end.js'] it "tokenises 4-digit sequences", -> for scope, quote of delimsByScope {tokens} = grammar.tokenizeLine(quote + '\\u2011' + quote) - expect(tokens[0]).toEqual value: quote, scopes: ['source.js', scope, 'punctuation.definition.string.begin.js'] + expect(tokens[0]).toEqual value: quote, scopes: ['source.js', scope, 'punctuation.definition.string.begin.js'] expect(tokens[1]).toEqual value: '\\u2011', scopes: ['source.js', scope, 'constant.character.escape.js'] - expect(tokens[2]).toEqual value: quote, scopes: ['source.js', scope, 'punctuation.definition.string.end.js'] + expect(tokens[2]).toEqual value: quote, scopes: ['source.js', scope, 'punctuation.definition.string.end.js'] it "tokenises variable-length sequences", -> for scope, quote of delimsByScope {tokens} = grammar.tokenizeLine(quote + '\\u{2000}' + quote) - expect(tokens[0]).toEqual value: quote, scopes: ['source.js', scope, 'punctuation.definition.string.begin.js'] - expect(tokens[1]).toEqual value: '\\u', scopes: ['source.js', scope, 'constant.character.escape.js'] - expect(tokens[2]).toEqual value: '{', scopes: ['source.js', scope, 'constant.character.escape.js', bracketScopes[0]] - expect(tokens[3]).toEqual value: '2000', scopes: ['source.js', scope, 'constant.character.escape.js'] - expect(tokens[4]).toEqual value: '}', scopes: ['source.js', scope, 'constant.character.escape.js', bracketScopes[1]] - expect(tokens[5]).toEqual value: quote, scopes: ['source.js', scope, 'punctuation.definition.string.end.js'] + expect(tokens[0]).toEqual value: quote, scopes: ['source.js', scope, 'punctuation.definition.string.begin.js'] + expect(tokens[1]).toEqual value: '\\u', scopes: ['source.js', scope, 'constant.character.escape.js'] + expect(tokens[2]).toEqual value: '{', scopes: ['source.js', scope, 'constant.character.escape.js', bracketScopes[0]] + expect(tokens[3]).toEqual value: '2000', scopes: ['source.js', scope, 'constant.character.escape.js'] + expect(tokens[4]).toEqual value: '}', scopes: ['source.js', scope, 'constant.character.escape.js', bracketScopes[1]] + expect(tokens[5]).toEqual value: quote, scopes: ['source.js', scope, 'punctuation.definition.string.end.js'] it "highlights sequences with invalid syntax", -> for invalid in ['\\u', '\\u{2000', '\\u{G}'] @@ -100,12 +100,12 @@ describe "Javascript grammar", -> {tokens} = grammar.tokenizeLine('"\\u{' + pointStr + '}"') pointScopes = ['source.js', 'string.quoted.double.js', 'constant.character.escape.js'] if codepoint > maxCodepoint then pointScopes.push 'invalid.illegal.unicode-escape.js' - expect(tokens[0]).toEqual value: '"', scopes: ['source.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] - expect(tokens[1]).toEqual value: '\\u', scopes: ['source.js', 'string.quoted.double.js', 'constant.character.escape.js'] - expect(tokens[2]).toEqual value: '{', scopes: ['source.js', 'string.quoted.double.js', 'constant.character.escape.js', bracketScopes[0]] + expect(tokens[0]).toEqual value: '"', scopes: ['source.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(tokens[1]).toEqual value: '\\u', scopes: ['source.js', 'string.quoted.double.js', 'constant.character.escape.js'] + expect(tokens[2]).toEqual value: '{', scopes: ['source.js', 'string.quoted.double.js', 'constant.character.escape.js', bracketScopes[0]] expect(tokens[3]).toEqual value: pointStr, scopes: pointScopes - expect(tokens[4]).toEqual value: '}', scopes: ['source.js', 'string.quoted.double.js', 'constant.character.escape.js', bracketScopes[1]] - expect(tokens[5]).toEqual value: '"', scopes: ['source.js', 'string.quoted.double.js', 'punctuation.definition.string.end.js'] + expect(tokens[4]).toEqual value: '}', scopes: ['source.js', 'string.quoted.double.js', 'constant.character.escape.js', bracketScopes[1]] + expect(tokens[5]).toEqual value: '"', scopes: ['source.js', 'string.quoted.double.js', 'punctuation.definition.string.end.js'] describe "keywords", -> keywords = ['await', 'break', 'catch', 'continue', 'do'] From 33fcb714e36a831b8cd343da645d40daec1a557b Mon Sep 17 00:00:00 2001 From: Alhadis Date: Wed, 24 Aug 2016 05:19:04 +1000 Subject: [PATCH 15/16] Update specs to accommodate 3b082ef's changes --- spec/javascript-spec.coffee | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/spec/javascript-spec.coffee b/spec/javascript-spec.coffee index 0906b0af..e8903f35 100644 --- a/spec/javascript-spec.coffee +++ b/spec/javascript-spec.coffee @@ -141,19 +141,19 @@ describe "Javascript grammar", -> describe "instantiation", -> it "tokenizes the new keyword and instance entities", -> {tokens} = grammar.tokenizeLine('new something') - expect(tokens[0]).toEqual value: 'new', scopes: ['source.js', 'meta.class.instance.constructor', 'keyword.operator.new.js'] - expect(tokens[1]).toEqual value: ' ', scopes: ['source.js', 'meta.class.instance.constructor'] - expect(tokens[2]).toEqual value: 'something', scopes: ['source.js', 'meta.class.instance.constructor', 'entity.name.type.instance.js'] + expect(tokens[0]).toEqual value: 'new', scopes: ['source.js', 'meta.class.instance.constructor.js', 'keyword.operator.new.js'] + expect(tokens[1]).toEqual value: ' ', scopes: ['source.js', 'meta.class.instance.constructor.js'] + expect(tokens[2]).toEqual value: 'something', scopes: ['source.js', 'meta.class.instance.constructor.js', 'entity.name.type.instance.js'] {tokens} = grammar.tokenizeLine('new Something') - expect(tokens[0]).toEqual value: 'new', scopes: ['source.js', 'meta.class.instance.constructor', 'keyword.operator.new.js'] - expect(tokens[1]).toEqual value: ' ', scopes: ['source.js', 'meta.class.instance.constructor'] - expect(tokens[2]).toEqual value: 'Something', scopes: ['source.js', 'meta.class.instance.constructor', 'entity.name.type.instance.js'] + expect(tokens[0]).toEqual value: 'new', scopes: ['source.js', 'meta.class.instance.constructor.js', 'keyword.operator.new.js'] + expect(tokens[1]).toEqual value: ' ', scopes: ['source.js', 'meta.class.instance.constructor.js'] + expect(tokens[2]).toEqual value: 'Something', scopes: ['source.js', 'meta.class.instance.constructor.js', 'entity.name.type.instance.js'] {tokens} = grammar.tokenizeLine('new $something') - expect(tokens[0]).toEqual value: 'new', scopes: ['source.js', 'meta.class.instance.constructor', 'keyword.operator.new.js'] - expect(tokens[1]).toEqual value: ' ', scopes: ['source.js', 'meta.class.instance.constructor'] - expect(tokens[2]).toEqual value: '$something', scopes: ['source.js', 'meta.class.instance.constructor', 'entity.name.type.instance.js'] + expect(tokens[0]).toEqual value: 'new', scopes: ['source.js', 'meta.class.instance.constructor.js', 'keyword.operator.new.js'] + expect(tokens[1]).toEqual value: ' ', scopes: ['source.js', 'meta.class.instance.constructor.js'] + expect(tokens[2]).toEqual value: '$something', scopes: ['source.js', 'meta.class.instance.constructor.js', 'entity.name.type.instance.js'] {tokens} = grammar.tokenizeLine('var instance = new obj.ct.Cla$s();') expect(tokens).toEqual [ @@ -161,13 +161,13 @@ describe "Javascript grammar", -> {value: ' instance ', scopes: ['source.js']} {value: '=', scopes: ['source.js', 'keyword.operator.assignment.js']} {value: ' ', scopes: ['source.js']} - {value: 'new', scopes: ['source.js', 'meta.class.instance.constructor', 'keyword.operator.new.js']} - {value: ' ', scopes: ['source.js', 'meta.class.instance.constructor']} - {value: 'obj', scopes: ['source.js', 'meta.class.instance.constructor', 'entity.name.type.instance.js']} - {value: '.', scopes: ['source.js', 'meta.class.instance.constructor', 'entity.name.type.instance.js', 'meta.delimiter.property.period.js']} - {value: 'ct', scopes: ['source.js', 'meta.class.instance.constructor', 'entity.name.type.instance.js']} - {value: '.', scopes: ['source.js', 'meta.class.instance.constructor', 'entity.name.type.instance.js', 'meta.delimiter.property.period.js']} - {value: 'Cla$s', scopes: ['source.js', 'meta.class.instance.constructor', 'entity.name.type.instance.js']} + {value: 'new', scopes: ['source.js', 'meta.class.instance.constructor.js', 'keyword.operator.new.js']} + {value: ' ', scopes: ['source.js', 'meta.class.instance.constructor.js']} + {value: 'obj', scopes: ['source.js', 'meta.class.instance.constructor.js', 'entity.name.type.instance.js']} + {value: '.', scopes: ['source.js', 'meta.class.instance.constructor.js', 'entity.name.type.instance.js', 'meta.delimiter.property.period.js']} + {value: 'ct', scopes: ['source.js', 'meta.class.instance.constructor.js', 'entity.name.type.instance.js']} + {value: '.', scopes: ['source.js', 'meta.class.instance.constructor.js', 'entity.name.type.instance.js', 'meta.delimiter.property.period.js']} + {value: 'Cla$s', scopes: ['source.js', 'meta.class.instance.constructor.js', 'entity.name.type.instance.js']} {value: '(', scopes: ['source.js', 'meta.brace.round.js']} {value: ')', scopes: ['source.js', 'meta.brace.round.js']} {value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js']} From f2871d41f9494479a4d4a93f7ef530485ab599e2 Mon Sep 17 00:00:00 2001 From: Alhadis Date: Wed, 24 Aug 2016 15:07:30 +1000 Subject: [PATCH 16/16] Conform to formatting of existing specs --- spec/javascript-spec.coffee | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/spec/javascript-spec.coffee b/spec/javascript-spec.coffee index e8903f35..e681c9e6 100644 --- a/spec/javascript-spec.coffee +++ b/spec/javascript-spec.coffee @@ -156,22 +156,20 @@ describe "Javascript grammar", -> expect(tokens[2]).toEqual value: '$something', scopes: ['source.js', 'meta.class.instance.constructor.js', 'entity.name.type.instance.js'] {tokens} = grammar.tokenizeLine('var instance = new obj.ct.Cla$s();') - expect(tokens).toEqual [ - {value: 'var', scopes: ['source.js', 'storage.type.var.js']} - {value: ' instance ', scopes: ['source.js']} - {value: '=', scopes: ['source.js', 'keyword.operator.assignment.js']} - {value: ' ', scopes: ['source.js']} - {value: 'new', scopes: ['source.js', 'meta.class.instance.constructor.js', 'keyword.operator.new.js']} - {value: ' ', scopes: ['source.js', 'meta.class.instance.constructor.js']} - {value: 'obj', scopes: ['source.js', 'meta.class.instance.constructor.js', 'entity.name.type.instance.js']} - {value: '.', scopes: ['source.js', 'meta.class.instance.constructor.js', 'entity.name.type.instance.js', 'meta.delimiter.property.period.js']} - {value: 'ct', scopes: ['source.js', 'meta.class.instance.constructor.js', 'entity.name.type.instance.js']} - {value: '.', scopes: ['source.js', 'meta.class.instance.constructor.js', 'entity.name.type.instance.js', 'meta.delimiter.property.period.js']} - {value: 'Cla$s', scopes: ['source.js', 'meta.class.instance.constructor.js', 'entity.name.type.instance.js']} - {value: '(', scopes: ['source.js', 'meta.brace.round.js']} - {value: ')', scopes: ['source.js', 'meta.brace.round.js']} - {value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js']} - ] + expect(tokens[0]).toEqual value: 'var', scopes: ['source.js', 'storage.type.var.js'] + expect(tokens[1]).toEqual value: ' instance ', scopes: ['source.js'] + expect(tokens[2]).toEqual value: '=', scopes: ['source.js', 'keyword.operator.assignment.js'] + expect(tokens[3]).toEqual value: ' ', scopes: ['source.js'] + expect(tokens[4]).toEqual value: 'new', scopes: ['source.js', 'meta.class.instance.constructor.js', 'keyword.operator.new.js'] + expect(tokens[5]).toEqual value: ' ', scopes: ['source.js', 'meta.class.instance.constructor.js'] + expect(tokens[6]).toEqual value: 'obj', scopes: ['source.js', 'meta.class.instance.constructor.js', 'entity.name.type.instance.js'] + expect(tokens[7]).toEqual value: '.', scopes: ['source.js', 'meta.class.instance.constructor.js', 'entity.name.type.instance.js', 'meta.delimiter.property.period.js'] + expect(tokens[8]).toEqual value: 'ct', scopes: ['source.js', 'meta.class.instance.constructor.js', 'entity.name.type.instance.js'] + expect(tokens[9]).toEqual value: '.', scopes: ['source.js', 'meta.class.instance.constructor.js', 'entity.name.type.instance.js', 'meta.delimiter.property.period.js'] + expect(tokens[10]).toEqual value: 'Cla$s', scopes: ['source.js', 'meta.class.instance.constructor.js', 'entity.name.type.instance.js'] + expect(tokens[11]).toEqual value: '(', scopes: ['source.js', 'meta.brace.round.js'] + expect(tokens[12]).toEqual value: ')', scopes: ['source.js', 'meta.brace.round.js'] + expect(tokens[13]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js'] describe "regular expressions", -> it "tokenizes regular expressions", -> @@ -1221,10 +1219,10 @@ describe "Javascript grammar", -> expect(tokens[0]).toEqual value: 'this', scopes: ['source.js', 'variable.language.js'] {tokens} = grammar.tokenizeLine('$this') - expect(tokens[0].value).toEqual '$this' + expect(tokens[0]).toEqual value: '$this', scopes: ['source.js'] {tokens} = grammar.tokenizeLine('this$') - expect(tokens[0].value).toEqual 'this$' + expect(tokens[0]).toEqual value: 'this$', scopes: ['source.js'] it "tokenizes 'super'", -> {tokens} = grammar.tokenizeLine('super')