Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit fc52230

Browse files
authored
fix: Maintain onNewLine state on subsequent lookahead (#2613)
1 parent 9971708 commit fc52230

File tree

4 files changed

+68
-38
lines changed

4 files changed

+68
-38
lines changed

‎src/parser.ts‎

Lines changed: 20 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -290,12 +290,12 @@ export class Parser extends DiagnosticEmitter {
290290
tn.next();
291291
let abstractStart = tn.tokenPos;
292292
let abstractEnd = tn.pos;
293-
let next = tn.peek(true);
294-
if (tn.nextTokenOnNewLine) {
293+
if (tn.peekOnNewLine()) {
295294
tn.reset(state);
296295
statement = this.parseStatement(tn, true);
297296
break;
298297
}
298+
let next = tn.peek();
299299
if (next != Token.Class) {
300300
if (next == Token.Interface) {
301301
this.error(
@@ -322,7 +322,7 @@ export class Parser extends DiagnosticEmitter {
322322
case Token.Namespace: {
323323
let state = tn.mark();
324324
tn.next();
325-
if (tn.peek(false,IdentifierHandling.Prefer) == Token.Identifier) {
325+
if (tn.peek(IdentifierHandling.Prefer) == Token.Identifier) {
326326
tn.discard(state);
327327
statement = this.parseNamespace(tn, flags, decorators, startPos);
328328
decorators = null;
@@ -345,7 +345,7 @@ export class Parser extends DiagnosticEmitter {
345345
case Token.Type: { // also identifier
346346
let state = tn.mark();
347347
tn.next();
348-
if (tn.peek(false,IdentifierHandling.Prefer) == Token.Identifier) {
348+
if (tn.peek(IdentifierHandling.Prefer) == Token.Identifier) {
349349
tn.discard(state);
350350
statement = this.parseTypeDeclaration(tn, flags, decorators, startPos);
351351
decorators = null;
@@ -358,7 +358,7 @@ export class Parser extends DiagnosticEmitter {
358358
case Token.Module: { // also identifier
359359
let state = tn.mark();
360360
tn.next();
361-
if (tn.peek(true) == Token.StringLiteral && !tn.nextTokenOnNewLine) {
361+
if (tn.peek() == Token.StringLiteral && !tn.peekOnNewLine()) {
362362
tn.discard(state);
363363
statement = this.parseModuleDeclaration(tn, flags);
364364
} else {
@@ -1113,10 +1113,11 @@ export class Parser extends DiagnosticEmitter {
11131113

11141114
let startPos = tn.tokenPos;
11151115
let expr: Expression | null = null;
1116+
let nextToken = tn.peek();
11161117
if (
1117-
tn.peek(true) != Token.Semicolon &&
1118-
tn.nextToken != Token.CloseBrace &&
1119-
!tn.nextTokenOnNewLine
1118+
nextToken != Token.Semicolon &&
1119+
nextToken != Token.CloseBrace &&
1120+
!tn.peekOnNewLine()
11201121
) {
11211122
if (!(expr = this.parseExpression(tn))) return null;
11221123
}
@@ -2042,7 +2043,7 @@ export class Parser extends DiagnosticEmitter {
20422043
let setEnd = 0;
20432044
if (!isInterface) {
20442045
if (tn.skip(Token.Get)) {
2045-
if (tn.peek(true,IdentifierHandling.Prefer) == Token.Identifier && !tn.nextTokenOnNewLine) {
2046+
if (tn.peek(IdentifierHandling.Prefer) == Token.Identifier && !tn.peekOnNewLine()) {
20462047
flags |= CommonFlags.Get;
20472048
isGetter = true;
20482049
getStart = tn.tokenPos;
@@ -2058,7 +2059,7 @@ export class Parser extends DiagnosticEmitter {
20582059
tn.reset(state);
20592060
}
20602061
} else if (tn.skip(Token.Set)) {
2061-
if (tn.peek(true,IdentifierHandling.Prefer) == Token.Identifier && !tn.nextTokenOnNewLine) {
2062+
if (tn.peek(IdentifierHandling.Prefer) == Token.Identifier && !tn.peekOnNewLine()) {
20622063
flags |= CommonFlags.Set;
20632064
isSetter = true;
20642065
setStart = tn.tokenPos;
@@ -2966,7 +2967,7 @@ export class Parser extends DiagnosticEmitter {
29662967
break;
29672968
}
29682969
case Token.Type: { // also identifier
2969-
if (tn.peek(false,IdentifierHandling.Prefer) == Token.Identifier) {
2970+
if (tn.peek(IdentifierHandling.Prefer) == Token.Identifier) {
29702971
statement = this.parseTypeDeclaration(tn, CommonFlags.None, null, tn.tokenPos);
29712972
break;
29722973
}
@@ -3020,7 +3021,7 @@ export class Parser extends DiagnosticEmitter {
30203021
// at 'break': Identifier? ';'?
30213022

30223023
let identifier: IdentifierExpression | null = null;
3023-
if (tn.peek(true) == Token.Identifier && !tn.nextTokenOnNewLine) {
3024+
if (tn.peek() == Token.Identifier && !tn.peekOnNewLine()) {
30243025
tn.next(IdentifierHandling.Prefer);
30253026
identifier = Node.createIdentifierExpression(tn.readIdentifier(), tn.range());
30263027
}
@@ -3036,7 +3037,7 @@ export class Parser extends DiagnosticEmitter {
30363037
// at 'continue': Identifier? ';'?
30373038

30383039
let identifier: IdentifierExpression | null = null;
3039-
if (tn.peek(true) == Token.Identifier && !tn.nextTokenOnNewLine) {
3040+
if (tn.peek() == Token.Identifier && !tn.peekOnNewLine()) {
30403041
tn.next(IdentifierHandling.Prefer);
30413042
identifier = Node.createIdentifierExpression(tn.readIdentifier(), tn.range());
30423043
}
@@ -3948,7 +3949,7 @@ export class Parser extends DiagnosticEmitter {
39483949
if (tn.skip(Token.TemplateLiteral)) {
39493950
return this.parseTemplateLiteral(tn, identifier);
39503951
}
3951-
if (tn.peek(true) == Token.Equals_GreaterThan && !tn.nextTokenOnNewLine) {
3952+
if (tn.peek() == Token.Equals_GreaterThan && !tn.peekOnNewLine()) {
39523953
return this.parseFunctionExpressionCommon(
39533954
tn,
39543955
Node.createEmptyIdentifierExpression(tn.range(startPos)),
@@ -4405,8 +4406,8 @@ export class Parser extends DiagnosticEmitter {
44054406
tn: Tokenizer
44064407
): void {
44074408
// see: https://tc39.es/ecma262/#sec-automatic-semicolon-insertion
4408-
let token = tn.peek(true);
4409-
if (tn.nextTokenOnNewLine||token== Token.EndOfFile || token == Token.CloseBrace) return;
4409+
let nextToken = tn.peek();
4410+
if (nextToken== Token.EndOfFile || nextToken == Token.CloseBrace||tn.peekOnNewLine()) return;
44104411
this.error(
44114412
DiagnosticCode.Unexpected_token,
44124413
tn.range(tn.nextTokenPos)
@@ -4415,18 +4416,17 @@ export class Parser extends DiagnosticEmitter {
44154416

44164417
/** Skips over a statement on errors in an attempt to reduce unnecessary diagnostic noise. */
44174418
skipStatement(tn: Tokenizer): void {
4418-
tn.peek(true);
4419-
if (tn.nextTokenOnNewLine) tn.next(); // if reset() to the previous line
4419+
if (tn.peekOnNewLine()) tn.next(); // if reset() to the previous line
44204420
do {
4421-
let nextToken = tn.peek(true);
4421+
let nextToken = tn.peek();
44224422
if (
44234423
nextToken == Token.EndOfFile || // next step should handle this
44244424
nextToken == Token.Semicolon // end of the statement for sure
44254425
) {
44264426
tn.next();
44274427
break;
44284428
}
4429-
if (tn.nextTokenOnNewLine) break; // end of the statement maybe
4429+
if (tn.peekOnNewLine()) break; // end of the statement maybe
44304430
switch (tn.next()) {
44314431
case Token.Identifier: {
44324432
tn.readIdentifier();

‎src/tokenizer.ts‎

Lines changed: 37 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -449,6 +449,13 @@ export function operatorTokenToString(token: Token): string {
449449
/** Handler for intercepting comments while tokenizing. */
450450
export type CommentHandler = (kind: CommentKind, text: string, range: Range) => void;
451451

452+
/** Whether a token begins on a new line, if known. */
453+
enum OnNewLine {
454+
No,
455+
Yes,
456+
Unknown
457+
}
458+
452459
/** Tokenizes a source to individual {@link Token}s. */
453460
export class Tokenizer extends DiagnosticEmitter {
454461

@@ -461,7 +468,7 @@ export class Tokenizer extends DiagnosticEmitter {
461468

462469
nextToken: Token = -1;
463470
nextTokenPos: i32 = 0;
464-
nextTokenOnNewLine: bool = false;
471+
nextTokenOnNewLine: OnNewLine = OnNewLine.Unknown;
465472

466473
onComment: CommentHandler | null = null;
467474

@@ -504,7 +511,7 @@ export class Tokenizer extends DiagnosticEmitter {
504511
}
505512

506513
next(identifierHandling: IdentifierHandling = IdentifierHandling.Default): Token {
507-
this.nextToken=-1;
514+
this.clearNextToken();
508515
let token: Token;
509516
do token = this.unsafeNext(identifierHandling);
510517
while (token == Token.Invalid);
@@ -959,34 +966,41 @@ export class Tokenizer extends DiagnosticEmitter {
959966
}
960967

961968
peek(
962-
checkOnNewLine: bool = false,
963969
identifierHandling: IdentifierHandling = IdentifierHandling.Default,
964970
maxCompoundLength: i32 = i32.MAX_VALUE
965971
): Token {
966-
let text = this.source.text;
967-
if (this.nextToken < 0) {
972+
let nextToken = this.nextToken;
973+
if (nextToken < 0) {
968974
let posBefore = this.pos;
969975
let tokenBefore = this.token;
970976
let tokenPosBefore = this.tokenPos;
971-
let nextToken: Token;
972977
do nextToken = this.unsafeNext(identifierHandling, maxCompoundLength);
973978
while (nextToken == Token.Invalid);
974979
this.nextToken = nextToken;
975980
this.nextTokenPos = this.tokenPos;
976-
if (checkOnNewLine) {
977-
this.nextTokenOnNewLine = false;
978-
for (let pos = posBefore, end = this.nextTokenPos; pos < end; ++pos) {
979-
if (isLineBreak(text.charCodeAt(pos))) {
980-
this.nextTokenOnNewLine = true;
981-
break;
982-
}
983-
}
984-
}
981+
this.nextTokenOnNewLine = OnNewLine.Unknown;
985982
this.pos = posBefore;
986983
this.token = tokenBefore;
987984
this.tokenPos = tokenPosBefore;
988985
}
989-
return this.nextToken;
986+
return nextToken;
987+
}
988+
989+
peekOnNewLine(): bool {
990+
switch (this.nextTokenOnNewLine) {
991+
case OnNewLine.No: return false;
992+
case OnNewLine.Yes: return true;
993+
}
994+
this.peek();
995+
let text = this.source.text;
996+
for (let pos = this.pos, end = this.nextTokenPos; pos < end; ++pos) {
997+
if (isLineBreak(text.charCodeAt(pos))) {
998+
this.nextTokenOnNewLine = OnNewLine.Yes;
999+
return true;
1000+
}
1001+
}
1002+
this.nextTokenOnNewLine = OnNewLine.No;
1003+
return false;
9901004
}
9911005

9921006
skipIdentifier(identifierHandling: IdentifierHandling = IdentifierHandling.Prefer): bool {
@@ -1006,7 +1020,7 @@ export class Tokenizer extends DiagnosticEmitter {
10061020
while (nextToken == Token.Invalid);
10071021
if (nextToken == token) {
10081022
this.token = token;
1009-
this.nextToken=-1;
1023+
this.clearNextToken();
10101024
return true;
10111025
} else {
10121026
this.pos = posBefore;
@@ -1037,7 +1051,13 @@ export class Tokenizer extends DiagnosticEmitter {
10371051
this.pos = state.pos;
10381052
this.token = state.token;
10391053
this.tokenPos = state.tokenPos;
1054+
this.clearNextToken();
1055+
}
1056+
1057+
clearNextToken(): void {
10401058
this.nextToken = -1;
1059+
this.nextTokenPos = 0;
1060+
this.nextTokenOnNewLine = OnNewLine.Unknown;
10411061
}
10421062

10431063
range(start: i32 = -1, end: i32 = -1): Range {

‎tests/parser/asi.ts‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,4 +16,10 @@ function successCloseBrace(): i32 {
1616

1717
function successCloseParen(): i32 {
1818
return ( 123 )
19-
}
19+
}
20+
21+
function successAfterLet(): i32 {
22+
// multiple tn.peeks
23+
let a = 0
24+
return a
25+
}

‎tests/parser/asi.ts.fixture.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,10 @@ function successCloseBrace(): i32 {
1717
function successCloseParen(): i32 {
1818
return (123);
1919
}
20+
function successAfterLet(): i32 {
21+
let a = 0;
22+
return a;
23+
}
2024
// ERROR 1012: "Unexpected token." in asi.ts(2,13+0)
2125
// ERROR 1012: "Unexpected token." in asi.ts(7,14+0)
2226
// ERROR 1012: "Unexpected token." in asi.ts(11,13+0)

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /