Skip to content

Commit fc52230

Browse files
dcodeIOdcodeIO
authored
fix: Maintain onNewLine state on subsequent lookahead (#2613)
1 parent 9971708 commit fc52230

File tree

4 files changed

+68
-38
lines changed

4 files changed

+68
-38
lines changed

‎src/parser.ts‎

Lines changed: 20 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -290,12 +290,12 @@ export class Parser extends DiagnosticEmitter {
290290
tn.next();
291291
let abstractStart = tn.tokenPos;
292292
let abstractEnd = tn.pos;
293-
let next = tn.peek(true);
294-
if (tn.nextTokenOnNewLine) {
293+
if (tn.peekOnNewLine()) {
295294
tn.reset(state);
296295
statement = this.parseStatement(tn, true);
297296
break;
298297
}
298+
let next = tn.peek();
299299
if (next != Token.Class) {
300300
if (next == Token.Interface) {
301301
this.error(
@@ -322,7 +322,7 @@ export class Parser extends DiagnosticEmitter {
322322
case Token.Namespace: {
323323
let state = tn.mark();
324324
tn.next();
325-
if (tn.peek(false, IdentifierHandling.Prefer) == Token.Identifier) {
325+
if (tn.peek(IdentifierHandling.Prefer) == Token.Identifier) {
326326
tn.discard(state);
327327
statement = this.parseNamespace(tn, flags, decorators, startPos);
328328
decorators = null;
@@ -345,7 +345,7 @@ export class Parser extends DiagnosticEmitter {
345345
case Token.Type: { // also identifier
346346
let state = tn.mark();
347347
tn.next();
348-
if (tn.peek(false, IdentifierHandling.Prefer) == Token.Identifier) {
348+
if (tn.peek(IdentifierHandling.Prefer) == Token.Identifier) {
349349
tn.discard(state);
350350
statement = this.parseTypeDeclaration(tn, flags, decorators, startPos);
351351
decorators = null;
@@ -358,7 +358,7 @@ export class Parser extends DiagnosticEmitter {
358358
case Token.Module: { // also identifier
359359
let state = tn.mark();
360360
tn.next();
361-
if (tn.peek(true) == Token.StringLiteral && !tn.nextTokenOnNewLine) {
361+
if (tn.peek() == Token.StringLiteral && !tn.peekOnNewLine()) {
362362
tn.discard(state);
363363
statement = this.parseModuleDeclaration(tn, flags);
364364
} else {
@@ -1113,10 +1113,11 @@ export class Parser extends DiagnosticEmitter {
11131113

11141114
let startPos = tn.tokenPos;
11151115
let expr: Expression | null = null;
1116+
let nextToken = tn.peek();
11161117
if (
1117-
tn.peek(true) != Token.Semicolon &&
1118-
tn.nextToken != Token.CloseBrace &&
1119-
!tn.nextTokenOnNewLine
1118+
nextToken != Token.Semicolon &&
1119+
nextToken != Token.CloseBrace &&
1120+
!tn.peekOnNewLine()
11201121
) {
11211122
if (!(expr = this.parseExpression(tn))) return null;
11221123
}
@@ -2042,7 +2043,7 @@ export class Parser extends DiagnosticEmitter {
20422043
let setEnd = 0;
20432044
if (!isInterface) {
20442045
if (tn.skip(Token.Get)) {
2045-
if (tn.peek(true, IdentifierHandling.Prefer) == Token.Identifier && !tn.nextTokenOnNewLine) {
2046+
if (tn.peek(IdentifierHandling.Prefer) == Token.Identifier && !tn.peekOnNewLine()) {
20462047
flags |= CommonFlags.Get;
20472048
isGetter = true;
20482049
getStart = tn.tokenPos;
@@ -2058,7 +2059,7 @@ export class Parser extends DiagnosticEmitter {
20582059
tn.reset(state);
20592060
}
20602061
} else if (tn.skip(Token.Set)) {
2061-
if (tn.peek(true, IdentifierHandling.Prefer) == Token.Identifier && !tn.nextTokenOnNewLine) {
2062+
if (tn.peek(IdentifierHandling.Prefer) == Token.Identifier && !tn.peekOnNewLine()) {
20622063
flags |= CommonFlags.Set;
20632064
isSetter = true;
20642065
setStart = tn.tokenPos;
@@ -2966,7 +2967,7 @@ export class Parser extends DiagnosticEmitter {
29662967
break;
29672968
}
29682969
case Token.Type: { // also identifier
2969-
if (tn.peek(false, IdentifierHandling.Prefer) == Token.Identifier) {
2970+
if (tn.peek(IdentifierHandling.Prefer) == Token.Identifier) {
29702971
statement = this.parseTypeDeclaration(tn, CommonFlags.None, null, tn.tokenPos);
29712972
break;
29722973
}
@@ -3020,7 +3021,7 @@ export class Parser extends DiagnosticEmitter {
30203021
// at 'break': Identifier? ';'?
30213022

30223023
let identifier: IdentifierExpression | null = null;
3023-
if (tn.peek(true) == Token.Identifier && !tn.nextTokenOnNewLine) {
3024+
if (tn.peek() == Token.Identifier && !tn.peekOnNewLine()) {
30243025
tn.next(IdentifierHandling.Prefer);
30253026
identifier = Node.createIdentifierExpression(tn.readIdentifier(), tn.range());
30263027
}
@@ -3036,7 +3037,7 @@ export class Parser extends DiagnosticEmitter {
30363037
// at 'continue': Identifier? ';'?
30373038

30383039
let identifier: IdentifierExpression | null = null;
3039-
if (tn.peek(true) == Token.Identifier && !tn.nextTokenOnNewLine) {
3040+
if (tn.peek() == Token.Identifier && !tn.peekOnNewLine()) {
30403041
tn.next(IdentifierHandling.Prefer);
30413042
identifier = Node.createIdentifierExpression(tn.readIdentifier(), tn.range());
30423043
}
@@ -3948,7 +3949,7 @@ export class Parser extends DiagnosticEmitter {
39483949
if (tn.skip(Token.TemplateLiteral)) {
39493950
return this.parseTemplateLiteral(tn, identifier);
39503951
}
3951-
if (tn.peek(true) == Token.Equals_GreaterThan && !tn.nextTokenOnNewLine) {
3952+
if (tn.peek() == Token.Equals_GreaterThan && !tn.peekOnNewLine()) {
39523953
return this.parseFunctionExpressionCommon(
39533954
tn,
39543955
Node.createEmptyIdentifierExpression(tn.range(startPos)),
@@ -4405,8 +4406,8 @@ export class Parser extends DiagnosticEmitter {
44054406
tn: Tokenizer
44064407
): void {
44074408
// see: https://tc39.es/ecma262/#sec-automatic-semicolon-insertion
4408-
let token = tn.peek(true);
4409-
if (tn.nextTokenOnNewLine || token == Token.EndOfFile || token == Token.CloseBrace) return;
4409+
let nextToken = tn.peek();
4410+
if (nextToken == Token.EndOfFile || nextToken == Token.CloseBrace || tn.peekOnNewLine()) return;
44104411
this.error(
44114412
DiagnosticCode.Unexpected_token,
44124413
tn.range(tn.nextTokenPos)
@@ -4415,18 +4416,17 @@ export class Parser extends DiagnosticEmitter {
44154416

44164417
/** Skips over a statement on errors in an attempt to reduce unnecessary diagnostic noise. */
44174418
skipStatement(tn: Tokenizer): void {
4418-
tn.peek(true);
4419-
if (tn.nextTokenOnNewLine) tn.next(); // if reset() to the previous line
4419+
if (tn.peekOnNewLine()) tn.next(); // if reset() to the previous line
44204420
do {
4421-
let nextToken = tn.peek(true);
4421+
let nextToken = tn.peek();
44224422
if (
44234423
nextToken == Token.EndOfFile || // next step should handle this
44244424
nextToken == Token.Semicolon // end of the statement for sure
44254425
) {
44264426
tn.next();
44274427
break;
44284428
}
4429-
if (tn.nextTokenOnNewLine) break; // end of the statement maybe
4429+
if (tn.peekOnNewLine()) break; // end of the statement maybe
44304430
switch (tn.next()) {
44314431
case Token.Identifier: {
44324432
tn.readIdentifier();

‎src/tokenizer.ts‎

Lines changed: 37 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -449,6 +449,13 @@ export function operatorTokenToString(token: Token): string {
449449
/** Handler for intercepting comments while tokenizing. */
450450
export type CommentHandler = (kind: CommentKind, text: string, range: Range) => void;
451451

452+
/** Whether a token begins on a new line, if known. */
453+
enum OnNewLine {
454+
No,
455+
Yes,
456+
Unknown
457+
}
458+
452459
/** Tokenizes a source to individual {@link Token}s. */
453460
export class Tokenizer extends DiagnosticEmitter {
454461

@@ -461,7 +468,7 @@ export class Tokenizer extends DiagnosticEmitter {
461468

462469
nextToken: Token = -1;
463470
nextTokenPos: i32 = 0;
464-
nextTokenOnNewLine: bool = false;
471+
nextTokenOnNewLine: OnNewLine = OnNewLine.Unknown;
465472

466473
onComment: CommentHandler | null = null;
467474

@@ -504,7 +511,7 @@ export class Tokenizer extends DiagnosticEmitter {
504511
}
505512

506513
next(identifierHandling: IdentifierHandling = IdentifierHandling.Default): Token {
507-
this.nextToken = -1;
514+
this.clearNextToken();
508515
let token: Token;
509516
do token = this.unsafeNext(identifierHandling);
510517
while (token == Token.Invalid);
@@ -959,34 +966,41 @@ export class Tokenizer extends DiagnosticEmitter {
959966
}
960967

961968
peek(
962-
checkOnNewLine: bool = false,
963969
identifierHandling: IdentifierHandling = IdentifierHandling.Default,
964970
maxCompoundLength: i32 = i32.MAX_VALUE
965971
): Token {
966-
let text = this.source.text;
967-
if (this.nextToken < 0) {
972+
let nextToken = this.nextToken;
973+
if (nextToken < 0) {
968974
let posBefore = this.pos;
969975
let tokenBefore = this.token;
970976
let tokenPosBefore = this.tokenPos;
971-
let nextToken: Token;
972977
do nextToken = this.unsafeNext(identifierHandling, maxCompoundLength);
973978
while (nextToken == Token.Invalid);
974979
this.nextToken = nextToken;
975980
this.nextTokenPos = this.tokenPos;
976-
if (checkOnNewLine) {
977-
this.nextTokenOnNewLine = false;
978-
for (let pos = posBefore, end = this.nextTokenPos; pos < end; ++pos) {
979-
if (isLineBreak(text.charCodeAt(pos))) {
980-
this.nextTokenOnNewLine = true;
981-
break;
982-
}
983-
}
984-
}
981+
this.nextTokenOnNewLine = OnNewLine.Unknown;
985982
this.pos = posBefore;
986983
this.token = tokenBefore;
987984
this.tokenPos = tokenPosBefore;
988985
}
989-
return this.nextToken;
986+
return nextToken;
987+
}
988+
989+
peekOnNewLine(): bool {
990+
switch (this.nextTokenOnNewLine) {
991+
case OnNewLine.No: return false;
992+
case OnNewLine.Yes: return true;
993+
}
994+
this.peek();
995+
let text = this.source.text;
996+
for (let pos = this.pos, end = this.nextTokenPos; pos < end; ++pos) {
997+
if (isLineBreak(text.charCodeAt(pos))) {
998+
this.nextTokenOnNewLine = OnNewLine.Yes;
999+
return true;
1000+
}
1001+
}
1002+
this.nextTokenOnNewLine = OnNewLine.No;
1003+
return false;
9901004
}
9911005

9921006
skipIdentifier(identifierHandling: IdentifierHandling = IdentifierHandling.Prefer): bool {
@@ -1006,7 +1020,7 @@ export class Tokenizer extends DiagnosticEmitter {
10061020
while (nextToken == Token.Invalid);
10071021
if (nextToken == token) {
10081022
this.token = token;
1009-
this.nextToken = -1;
1023+
this.clearNextToken();
10101024
return true;
10111025
} else {
10121026
this.pos = posBefore;
@@ -1037,7 +1051,13 @@ export class Tokenizer extends DiagnosticEmitter {
10371051
this.pos = state.pos;
10381052
this.token = state.token;
10391053
this.tokenPos = state.tokenPos;
1054+
this.clearNextToken();
1055+
}
1056+
1057+
clearNextToken(): void {
10401058
this.nextToken = -1;
1059+
this.nextTokenPos = 0;
1060+
this.nextTokenOnNewLine = OnNewLine.Unknown;
10411061
}
10421062

10431063
range(start: i32 = -1, end: i32 = -1): Range {

‎tests/parser/asi.ts‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,4 +16,10 @@ function successCloseBrace(): i32 {
1616

1717
function successCloseParen(): i32 {
1818
return ( 123 )
19-
}
19+
}
20+
21+
function successAfterLet(): i32 {
22+
// multiple tn.peeks
23+
let a = 0
24+
return a
25+
}

‎tests/parser/asi.ts.fixture.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,10 @@ function successCloseBrace(): i32 {
1717
function successCloseParen(): i32 {
1818
return (123);
1919
}
20+
function successAfterLet(): i32 {
21+
let a = 0;
22+
return a;
23+
}
2024
// ERROR 1012: "Unexpected token." in asi.ts(2,13+0)
2125
// ERROR 1012: "Unexpected token." in asi.ts(7,14+0)
2226
// ERROR 1012: "Unexpected token." in asi.ts(11,13+0)

0 commit comments

Comments
 (0)