Skip to content

Commit ff1e08f

Browse files
sandersnsandersn
authored
Rename scanner token methods (#53078)
1 parent 3236ac9 commit ff1e08f

File tree

12 files changed

+172
-141
lines changed

12 files changed

+172
-141
lines changed

‎src/compiler/parser.ts

Lines changed: 52 additions & 52 deletions
Large diffs are not rendered by default.

‎src/compiler/scanner.ts

Lines changed: 55 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -40,9 +40,15 @@ export function tokenIsIdentifierOrKeywordOrGreaterThan(token: SyntaxKind): bool
4040
}
4141

4242
export interface Scanner {
43+
/** @deprecated use {@link getTokenFullStart} */
4344
getStartPos(): number;
4445
getToken(): SyntaxKind;
46+
getTokenFullStart(): number;
47+
getTokenStart(): number;
48+
getTokenEnd(): number;
49+
/** @deprecated use {@link getTokenEnd} */
4550
getTextPos(): number;
51+
/** @deprecated use {@link getTokenStart} */
4652
getTokenPos(): number;
4753
getTokenText(): string;
4854
getTokenValue(): string;
@@ -86,7 +92,9 @@ export interface Scanner {
8692
setOnError(onError: ErrorCallback | undefined): void;
8793
setScriptTarget(scriptTarget: ScriptTarget): void;
8894
setLanguageVariant(variant: LanguageVariant): void;
95+
/** @deprecated use {@link resetTokenState} */
8996
setTextPos(textPos: number): void;
97+
resetTokenState(pos: number): void;
9098
/** @internal */
9199
setInJSDocType(inType: boolean): void;
92100
// Invokes the provided callback then unconditionally restores the scanner to the state it
@@ -974,10 +982,10 @@ export function createScanner(languageVersion: ScriptTarget,
974982
var end: number;
975983

976984
// Start position of whitespace before current token
977-
var startPos: number;
985+
var fullStartPos: number;
978986

979987
// Start position of text of current token
980-
var tokenPos: number;
988+
var tokenStart: number;
981989

982990
var token: SyntaxKind;
983991
var tokenValue!: string;
@@ -989,11 +997,14 @@ export function createScanner(languageVersion: ScriptTarget,
989997
setText(text, start, length);
990998

991999
var scanner: Scanner = {
992-
getStartPos: () => startPos,
1000+
getTokenFullStart: () => fullStartPos,
1001+
getStartPos: () => fullStartPos,
1002+
getTokenEnd: () => pos,
9931003
getTextPos: () => pos,
9941004
getToken: () => token,
995-
getTokenPos: () => tokenPos,
996-
getTokenText: () => text.substring(tokenPos, pos),
1005+
getTokenStart: () => tokenStart,
1006+
getTokenPos: () => tokenStart,
1007+
getTokenText: () => text.substring(tokenStart, pos),
9971008
getTokenValue: () => tokenValue,
9981009
hasUnicodeEscape: () => (tokenFlags & TokenFlags.UnicodeEscape) !== 0,
9991010
hasExtendedUnicodeEscape: () => (tokenFlags & TokenFlags.ExtendedUnicodeEscape) !== 0,
@@ -1027,7 +1038,8 @@ export function createScanner(languageVersion: ScriptTarget,
10271038
setScriptTarget,
10281039
setLanguageVariant,
10291040
setOnError,
1030-
setTextPos,
1041+
resetTokenState,
1042+
setTextPos: resetTokenState,
10311043
setInJSDocType,
10321044
tryScan,
10331045
lookAhead,
@@ -1039,7 +1051,7 @@ export function createScanner(languageVersion: ScriptTarget,
10391051
Object.defineProperty(scanner, "__debugShowCurrentPositionInText", {
10401052
get: () => {
10411053
const text = scanner.getText();
1042-
return text.slice(0, scanner.getStartPos()) + "║" + text.slice(scanner.getStartPos());
1054+
return text.slice(0, scanner.getTokenFullStart()) + "║" + text.slice(scanner.getTokenFullStart());
10431055
},
10441056
});
10451057
}
@@ -1638,11 +1650,11 @@ export function createScanner(languageVersion: ScriptTarget,
16381650
}
16391651

16401652
function scan(): SyntaxKind {
1641-
startPos = pos;
1653+
fullStartPos = pos;
16421654
tokenFlags = TokenFlags.None;
16431655
let asteriskSeen = false;
16441656
while (true) {
1645-
tokenPos = pos;
1657+
tokenStart = pos;
16461658
if (pos >= end) {
16471659
return token = SyntaxKind.EndOfFileToken;
16481660
}
@@ -1810,9 +1822,9 @@ export function createScanner(languageVersion: ScriptTarget,
18101822

18111823
commentDirectives = appendIfCommentDirective(
18121824
commentDirectives,
1813-
text.slice(tokenPos, pos),
1825+
text.slice(tokenStart, pos),
18141826
commentDirectiveRegExSingleLine,
1815-
tokenPos,
1827+
tokenStart,
18161828
);
18171829

18181830
if (skipTrivia) {
@@ -1830,7 +1842,7 @@ export function createScanner(languageVersion: ScriptTarget,
18301842
}
18311843

18321844
let commentClosed = false;
1833-
let lastLineStart = tokenPos;
1845+
let lastLineStart = tokenStart;
18341846
while (pos < end) {
18351847
const ch = text.charCodeAt(pos);
18361848

@@ -2139,7 +2151,7 @@ export function createScanner(languageVersion: ScriptTarget,
21392151

21402152
function reScanInvalidIdentifier(): SyntaxKind {
21412153
Debug.assert(token === SyntaxKind.Unknown, "'reScanInvalidIdentifier' should only be called when the current token is 'SyntaxKind.Unknown'.");
2142-
pos = tokenPos = startPos;
2154+
pos = tokenStart = fullStartPos;
21432155
tokenFlags = 0;
21442156
const ch = codePointAt(text, pos);
21452157
const identifierKind = scanIdentifier(ch, ScriptTarget.ESNext);
@@ -2155,7 +2167,7 @@ export function createScanner(languageVersion: ScriptTarget,
21552167
if (isIdentifierStart(ch, languageVersion)) {
21562168
pos += charSize(ch);
21572169
while (pos < end && isIdentifierPart(ch = codePointAt(text, pos), languageVersion)) pos += charSize(ch);
2158-
tokenValue = text.substring(tokenPos, pos);
2170+
tokenValue = text.substring(tokenStart, pos);
21592171
if (ch === CharacterCodes.backslash) {
21602172
tokenValue += scanIdentifierParts();
21612173
}
@@ -2188,13 +2200,13 @@ export function createScanner(languageVersion: ScriptTarget,
21882200

21892201
function reScanAsteriskEqualsToken(): SyntaxKind {
21902202
Debug.assert(token === SyntaxKind.AsteriskEqualsToken, "'reScanAsteriskEqualsToken' should only be called on a '*='");
2191-
pos = tokenPos + 1;
2203+
pos = tokenStart + 1;
21922204
return token = SyntaxKind.EqualsToken;
21932205
}
21942206

21952207
function reScanSlashToken(): SyntaxKind {
21962208
if (token === SyntaxKind.SlashToken || token === SyntaxKind.SlashEqualsToken) {
2197-
let p = tokenPos + 1;
2209+
let p = tokenStart + 1;
21982210
let inEscape = false;
21992211
let inCharacterClass = false;
22002212
while (true) {
@@ -2240,7 +2252,7 @@ export function createScanner(languageVersion: ScriptTarget,
22402252
p++;
22412253
}
22422254
pos = p;
2243-
tokenValue = text.substring(tokenPos, pos);
2255+
tokenValue = text.substring(tokenStart, pos);
22442256
token = SyntaxKind.RegularExpressionLiteral;
22452257
}
22462258
return token;
@@ -2288,44 +2300,44 @@ export function createScanner(languageVersion: ScriptTarget,
22882300
*/
22892301
function reScanTemplateToken(isTaggedTemplate: boolean): SyntaxKind {
22902302
Debug.assert(token === SyntaxKind.CloseBraceToken, "'reScanTemplateToken' should only be called on a '}'");
2291-
pos = tokenPos;
2303+
pos = tokenStart;
22922304
return token = scanTemplateAndSetTokenValue(isTaggedTemplate);
22932305
}
22942306

22952307
function reScanTemplateHeadOrNoSubstitutionTemplate(): SyntaxKind {
2296-
pos = tokenPos;
2308+
pos = tokenStart;
22972309
return token = scanTemplateAndSetTokenValue(/* isTaggedTemplate */ true);
22982310
}
22992311

23002312
function reScanJsxToken(allowMultilineJsxText = true): JsxTokenSyntaxKind {
2301-
pos = tokenPos = startPos;
2313+
pos = tokenStart = fullStartPos;
23022314
return token = scanJsxToken(allowMultilineJsxText);
23032315
}
23042316

23052317
function reScanLessThanToken(): SyntaxKind {
23062318
if (token === SyntaxKind.LessThanLessThanToken) {
2307-
pos = tokenPos + 1;
2319+
pos = tokenStart + 1;
23082320
return token = SyntaxKind.LessThanToken;
23092321
}
23102322
return token;
23112323
}
23122324

23132325
function reScanHashToken(): SyntaxKind {
23142326
if (token === SyntaxKind.PrivateIdentifier) {
2315-
pos = tokenPos + 1;
2327+
pos = tokenStart + 1;
23162328
return token = SyntaxKind.HashToken;
23172329
}
23182330
return token;
23192331
}
23202332

23212333
function reScanQuestionToken(): SyntaxKind {
23222334
Debug.assert(token === SyntaxKind.QuestionQuestionToken, "'reScanQuestionToken' should only be called on a '??'");
2323-
pos = tokenPos + 1;
2335+
pos = tokenStart + 1;
23242336
return token = SyntaxKind.QuestionToken;
23252337
}
23262338

23272339
function scanJsxToken(allowMultilineJsxText = true): JsxTokenSyntaxKind {
2328-
startPos = tokenPos = pos;
2340+
fullStartPos = tokenStart = pos;
23292341

23302342
if (pos >= end) {
23312343
return token = SyntaxKind.EndOfFileToken;
@@ -2392,7 +2404,7 @@ export function createScanner(languageVersion: ScriptTarget,
23922404
pos++;
23932405
}
23942406

2395-
tokenValue = text.substring(startPos, pos);
2407+
tokenValue = text.substring(fullStartPos, pos);
23962408

23972409
return firstNonWhitespace === -1 ? SyntaxKind.JsxTextAllWhiteSpaces : SyntaxKind.JsxText;
23982410
}
@@ -2437,7 +2449,7 @@ export function createScanner(languageVersion: ScriptTarget,
24372449
}
24382450

24392451
function scanJsxAttributeValue(): SyntaxKind {
2440-
startPos = pos;
2452+
fullStartPos = pos;
24412453

24422454
switch (text.charCodeAt(pos)) {
24432455
case CharacterCodes.doubleQuote:
@@ -2451,12 +2463,12 @@ export function createScanner(languageVersion: ScriptTarget,
24512463
}
24522464

24532465
function reScanJsxAttributeValue(): SyntaxKind {
2454-
pos = tokenPos = startPos;
2466+
pos = tokenStart = fullStartPos;
24552467
return scanJsxAttributeValue();
24562468
}
24572469

24582470
function scanJsDocToken(): JSDocSyntaxKind {
2459-
startPos = tokenPos = pos;
2471+
fullStartPos = tokenStart = pos;
24602472
tokenFlags = TokenFlags.None;
24612473
if (pos >= end) {
24622474
return token = SyntaxKind.EndOfFileToken;
@@ -2531,7 +2543,7 @@ export function createScanner(languageVersion: ScriptTarget,
25312543
if (isIdentifierStart(ch, languageVersion)) {
25322544
let char = ch;
25332545
while (pos < end && isIdentifierPart(char = codePointAt(text, pos), languageVersion) || text.charCodeAt(pos) === CharacterCodes.minus) pos += charSize(char);
2534-
tokenValue = text.substring(tokenPos, pos);
2546+
tokenValue = text.substring(tokenStart, pos);
25352547
if (char === CharacterCodes.backslash) {
25362548
tokenValue += scanIdentifierParts();
25372549
}
@@ -2544,8 +2556,8 @@ export function createScanner(languageVersion: ScriptTarget,
25442556

25452557
function speculationHelper<T>(callback: () => T, isLookahead: boolean): T {
25462558
const savePos = pos;
2547-
const saveStartPos = startPos;
2548-
const saveTokenPos = tokenPos;
2559+
const saveStartPos = fullStartPos;
2560+
const saveTokenPos = tokenStart;
25492561
const saveToken = token;
25502562
const saveTokenValue = tokenValue;
25512563
const saveTokenFlags = tokenFlags;
@@ -2555,8 +2567,8 @@ export function createScanner(languageVersion: ScriptTarget,
25552567
// then unconditionally restore us to where we were.
25562568
if (!result || isLookahead) {
25572569
pos = savePos;
2558-
startPos = saveStartPos;
2559-
tokenPos = saveTokenPos;
2570+
fullStartPos = saveStartPos;
2571+
tokenStart = saveTokenPos;
25602572
token = saveToken;
25612573
tokenValue = saveTokenValue;
25622574
tokenFlags = saveTokenFlags;
@@ -2567,8 +2579,8 @@ export function createScanner(languageVersion: ScriptTarget,
25672579
function scanRange<T>(start: number, length: number, callback: () => T): T {
25682580
const saveEnd = end;
25692581
const savePos = pos;
2570-
const saveStartPos = startPos;
2571-
const saveTokenPos = tokenPos;
2582+
const saveStartPos = fullStartPos;
2583+
const saveTokenPos = tokenStart;
25722584
const saveToken = token;
25732585
const saveTokenValue = tokenValue;
25742586
const saveTokenFlags = tokenFlags;
@@ -2579,8 +2591,8 @@ export function createScanner(languageVersion: ScriptTarget,
25792591

25802592
end = saveEnd;
25812593
pos = savePos;
2582-
startPos = saveStartPos;
2583-
tokenPos = saveTokenPos;
2594+
fullStartPos = saveStartPos;
2595+
tokenStart = saveTokenPos;
25842596
token = saveToken;
25852597
tokenValue = saveTokenValue;
25862598
tokenFlags = saveTokenFlags;
@@ -2608,7 +2620,7 @@ export function createScanner(languageVersion: ScriptTarget,
26082620
function setText(newText: string | undefined, start: number | undefined, length: number | undefined) {
26092621
text = newText || "";
26102622
end = length === undefined ? text.length : start! + length;
2611-
setTextPos(start || 0);
2623+
resetTokenState(start || 0);
26122624
}
26132625

26142626
function setOnError(errorCallback: ErrorCallback | undefined) {
@@ -2623,11 +2635,11 @@ export function createScanner(languageVersion: ScriptTarget,
26232635
languageVariant = variant;
26242636
}
26252637

2626-
function setTextPos(textPos: number) {
2627-
Debug.assert(textPos >= 0);
2628-
pos = textPos;
2629-
startPos = textPos;
2630-
tokenPos = textPos;
2638+
function resetTokenState(position: number) {
2639+
Debug.assert(position >= 0);
2640+
pos = position;
2641+
fullStartPos = position;
2642+
tokenStart = position;
26312643
token = SyntaxKind.Unknown;
26322644
tokenValue = undefined!;
26332645
tokenFlags = TokenFlags.None;

‎src/compiler/utilities.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2150,8 +2150,8 @@ export function createDiagnosticForRange(sourceFile: SourceFile, range: TextRang
21502150
export function getSpanOfTokenAtPosition(sourceFile: SourceFile, pos: number): TextSpan {
21512151
const scanner = createScanner(sourceFile.languageVersion, /*skipTrivia*/ true, sourceFile.languageVariant, sourceFile.text, /*onError:*/ undefined, pos);
21522152
scanner.scan();
2153-
const start = scanner.getTokenPos();
2154-
return createTextSpanFromBounds(start, scanner.getTextPos());
2153+
const start = scanner.getTokenStart();
2154+
return createTextSpanFromBounds(start, scanner.getTokenEnd());
21552155
}
21562156

21572157
/** @internal */
@@ -9558,7 +9558,7 @@ export function isValidBigIntString(s: string, roundTripOnly: boolean): boolean
95589558
// * a bigint can be scanned, and that when it is scanned, it is
95599559
// * the full length of the input string (so the scanner is one character beyond the augmented input length)
95609560
// * it does not contain a numeric seperator (the `BigInt` constructor does not accept a numeric seperator in its input)
9561-
return success && result === SyntaxKind.BigIntLiteral && scanner.getTextPos() === (s.length + 1) && !(flags & TokenFlags.ContainsSeparator)
9561+
return success && result === SyntaxKind.BigIntLiteral && scanner.getTokenEnd() === (s.length + 1) && !(flags & TokenFlags.ContainsSeparator)
95629562
&& (!roundTripOnly || s === pseudoBigIntToString({ negative, base10Value: parsePseudoBigInt(scanner.getTokenValue()) }));
95639563
}
95649564

‎src/harness/fourslashImpl.ts

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4897,11 +4897,11 @@ function codeFence(code: string, lang?: string) {
48974897
function getRangeOfIdentifierTouchingPosition(content: string, position: number): ts.TextRange | undefined {
48984898
const scanner = ts.createScanner(ts.ScriptTarget.Latest, /*skipTrivia*/ true, ts.LanguageVariant.Standard, content);
48994899
while (scanner.scan() !== ts.SyntaxKind.EndOfFileToken) {
4900-
const tokenStart = scanner.getStartPos();
4901-
if (scanner.getToken() === ts.SyntaxKind.Identifier && tokenStart <= position && scanner.getTextPos() >= position) {
4902-
return { pos: tokenStart, end: scanner.getTextPos() };
4900+
const tokenFullStart = scanner.getTokenFullStart();
4901+
if (scanner.getToken() === ts.SyntaxKind.Identifier && tokenFullStart <= position && scanner.getTokenEnd() >= position) {
4902+
return { pos: tokenFullStart, end: scanner.getTokenEnd() };
49034903
}
4904-
if (tokenStart > position) {
4904+
if (tokenFullStart > position) {
49054905
break;
49064906
}
49074907
}

0 commit comments

Comments
 (0)