Skip to content

Commit 834db54

Browse files
alangpierce1Lighty
authored andcommitted
Port babel-parser changes from 2022-01-10 to 2022-02-26 (alangpierce#716)
Instructions: https://github.com/alangpierce/sucrase/wiki/Porting-changes-from-Babel's-parser 69246b6212 [babel 8] fix properties name for function-like TS nodes (#13709) 🚫 AST only. bc0d1ef860 v7.16.8 🚫 Release only. 478a9709ab Improve errors location tracking (#14130) 🚫 Change to error messaging details that were removed in Sucrase. 4a737547e5 v7.16.10 🚫 Release only. e5d29f6e40 fix: incorrect conciseBody lookahead (#14194) ✅ Bug doesn't appear in Sucrase, but I added a test anyway. 5266605528 v7.16.12 🚫 Release only. 5861002b33 Reinterpret << when parsing TS type arguments (#14145) ✅ Ported following a very similar implementation. 6b427ced22 Fuzz test location-related parser options (#14201) 🚫 Babel-internal change testing location logic not present in Sucrase. 96a8251def Add parser support for the "regexp unicode sets" proposal (#14086) ✅ No implementation necessary, but I added a quick test. f893b333a8 Add the `decoratorsAutoAccessors` parser plugin (#13681) ✅ Ported basic parsing for accessor properties. de5c7b1bce Parse destructuring private fields (#13931) ✅ Seems to be already working, and I added a test. 🚫 d50c18dbc2 fix: usePrivateName usage Only affects Babel scope code, not relevant to Sucrase. 🚫 b092bd0cb5 remove invalid test output Only affects Babel tests. df27d542ef proposal-pipe: Add support for `^^` and `@@` topics (#13973) 🚫 Sucrase is holding off on pipeline topic parsing until the details are finalized. Tracked in alangpierce#674. 97a8bcb9cc Expose `.index` on Position to internally track nodes location (#14174) 🚫 Only affects class not used in Sucrase. 38c23cded4 v7.17.0 🚫 Release only. 19ede090eb parser: Update `babel-parser.d.ts` for 7v.17.0 features (#14266) 🚫 Not relevant to Sucrase. f52c70c357 v7.17.3 🚫 Release only. a53c2fa4a2 fix(ts): skip func-type param start on parsing (#14293) 🚫 Bug not present in Sucrase. 5749c16dc2 [tsx] raise error on single arrow type argument without comma (#14135) 🚫 Only affects error handling.
1 parent d20bfc3 commit 834db54

11 files changed

+589
-330
lines changed

generator/generateReadWordTree.ts

+1
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ const KEYWORDS = [
4242

4343
const CONTEXTUAL_KEYWORDS = [
4444
"abstract",
45+
"accessor",
4546
"as",
4647
"asserts",
4748
"async",

generator/generateTokenTypes.ts

+5-1
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,8 @@ const types = {
9797
lessThan: new BinopTokenType("<", 7),
9898
greaterThan: new BinopTokenType(">", 7),
9999
relationalOrEqual: new BinopTokenType("<=/>=", 7),
100-
bitShift: new BinopTokenType("<</>>", 8),
100+
bitShiftL: new BinopTokenType("<<", 8),
101+
bitShiftR: new BinopTokenType(">>/>>>", 8),
101102
plus: new TokenType("+", {binop: 9, prefix}),
102103
minus: new TokenType("-", {binop: 9, prefix}),
103104
modulo: new BinopTokenType("%", 10),
@@ -173,6 +174,9 @@ const types = {
173174

174175
export default function generateTokenTypes(): string {
175176
let code = '// Generated file, do not edit! Run "yarn generate" to re-generate this file.\n';
177+
// formatTokenType is trivial and used for debugging purposes, so we shouldn't
178+
// need full test coverage.
179+
code += "/* istanbul ignore file */\n";
176180
code += generateTokenTypeEnum();
177181
code += generateFormatTokenType();
178182
return code;

src/parser/plugins/typescript.ts

+28-12
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import {
22
eat,
3+
finishToken,
34
IdentifierRole,
45
lookaheadType,
56
lookaheadTypeAndKeyword,
@@ -1095,6 +1096,25 @@ function tsTryParseGenericAsyncArrowFunction(): boolean {
10951096
return true;
10961097
}
10971098

1099+
/**
1100+
* If necessary, hack the tokenizer state so that this bitshift was actually a
1101+
* less-than token, then keep parsing. This should only be used in situations
1102+
* where we restore from snapshot on error (which reverts this change) or
1103+
* where bitshift would be illegal anyway (e.g. in a class "extends" clause).
1104+
*
1105+
* This hack is useful to handle situations like foo<<T>() => void>() where
1106+
* there can legitimately be two open-angle-brackets in a row in TS. This
1107+
* situation is very obscure and (as of this writing) is handled by Babel but
1108+
* not TypeScript itself, so it may be fine in the future to remove this case.
1109+
*/
1110+
function tsParseTypeArgumentsWithPossibleBitshift(): void {
1111+
if (state.type === tt.bitShiftL) {
1112+
state.pos -= 1;
1113+
finishToken(tt.lessThan);
1114+
}
1115+
tsParseTypeArguments();
1116+
}
1117+
10981118
function tsParseTypeArguments(): void {
10991119
const oldIsType = pushTypeContext(0);
11001120
expect(tt.lessThan);
@@ -1165,7 +1185,7 @@ export function tsParseSubscript(
11651185
return;
11661186
}
11671187

1168-
if (match(tt.lessThan)) {
1188+
if (match(tt.lessThan) || match(tt.bitShiftL)) {
11691189
// There are number of things we are going to "maybe" parse, like type arguments on
11701190
// tagged template expressions. If any of them fail, walk it back and continue.
11711191
const snapshot = state.snapshot();
@@ -1178,7 +1198,7 @@ export function tsParseSubscript(
11781198
return;
11791199
}
11801200
}
1181-
tsParseTypeArguments();
1201+
tsParseTypeArgumentsWithPossibleBitshift();
11821202
if (!noCalls && eat(tt.parenL)) {
11831203
// With f<T>(), the subscriptStartIndex marker is on the ( token.
11841204
state.tokens[state.tokens.length - 1].subscriptStartIndex = startTokenIndex;
@@ -1210,15 +1230,11 @@ export function tsParseSubscript(
12101230
}
12111231

12121232
export function tsStartParseNewArguments(): void {
1213-
if (match(tt.lessThan)) {
1233+
if (match(tt.lessThan) || match(tt.bitShiftL)) {
12141234
// 99% certain this is `new C<T>();`. But may be `new C < T;`, which is also legal.
12151235
const snapshot = state.snapshot();
12161236

1217-
state.type = tt.typeParameterStart;
1218-
tsParseTypeArguments();
1219-
if (!match(tt.parenL)) {
1220-
unexpected();
1221-
}
1237+
tsParseTypeArgumentsWithPossibleBitshift();
12221238

12231239
if (state.error) {
12241240
state.restoreFromSnapshot(snapshot);
@@ -1431,8 +1447,8 @@ export function tsParseExportDeclaration(): void {
14311447
}
14321448

14331449
export function tsAfterParseClassSuper(hasSuper: boolean): void {
1434-
if (hasSuper && match(tt.lessThan)) {
1435-
tsParseTypeArguments();
1450+
if (hasSuper && (match(tt.lessThan) || match(tt.bitShiftL))) {
1451+
tsParseTypeArgumentsWithPossibleBitshift();
14361452
}
14371453
if (eatContextual(ContextualKeyword._implements)) {
14381454
state.tokens[state.tokens.length - 1].type = tt._implements;
@@ -1553,8 +1569,8 @@ export function tsParseAssignableListItemTypes(): void {
15531569
}
15541570

15551571
export function tsParseMaybeDecoratorArguments(): void {
1556-
if (match(tt.lessThan)) {
1557-
tsParseTypeArguments();
1572+
if (match(tt.lessThan) || match(tt.bitShiftL)) {
1573+
tsParseTypeArgumentsWithPossibleBitshift();
15581574
}
15591575
baseParseMaybeDecoratorArguments();
15601576
}

src/parser/tokenizer/index.ts

+33-11
Original file line numberDiff line numberDiff line change
@@ -482,15 +482,36 @@ function readToken_plus_min(code: number): void {
482482
}
483483
}
484484

485-
// '<>'
486-
function readToken_lt_gt(code: number): void {
485+
function readToken_lt(): void {
486+
const nextChar = input.charCodeAt(state.pos + 1);
487+
488+
if (nextChar === charCodes.lessThan) {
489+
if (input.charCodeAt(state.pos + 2) === charCodes.equalsTo) {
490+
finishOp(tt.assign, 3);
491+
return;
492+
}
493+
// This still might be two instances of <, e.g. the TS type argument
494+
// expression f<<T>() => void>() , but parse as left shift for now and we'll
495+
// retokenize if necessary. We can't use isType for this case because we
496+
// don't know yet if we're in a type.
497+
finishOp(tt.bitShiftL, 2);
498+
return;
499+
}
500+
501+
if (nextChar === charCodes.equalsTo) {
502+
// <=
503+
finishOp(tt.relationalOrEqual, 2);
504+
} else {
505+
finishOp(tt.lessThan, 1);
506+
}
507+
}
508+
509+
function readToken_gt(): void {
510+
const code = charCodes.greaterThan;
487511
const nextChar = input.charCodeAt(state.pos + 1);
488512

489513
if (nextChar === code) {
490-
const size =
491-
code === charCodes.greaterThan && input.charCodeAt(state.pos + 2) === charCodes.greaterThan
492-
? 3
493-
: 2;
514+
const size = input.charCodeAt(state.pos + 2) === charCodes.greaterThan ? 3 : 2;
494515
if (input.charCodeAt(state.pos + size) === charCodes.equalsTo) {
495516
finishOp(tt.assign, size + 1);
496517
return;
@@ -500,15 +521,13 @@ function readToken_lt_gt(code: number): void {
500521
finishOp(tt.greaterThan, 1);
501522
return;
502523
}
503-
finishOp(tt.bitShift, size);
524+
finishOp(tt.bitShiftR, size);
504525
return;
505526
}
506527

507528
if (nextChar === charCodes.equalsTo) {
508-
// <= | >=
529+
// >=
509530
finishOp(tt.relationalOrEqual, 2);
510-
} else if (code === charCodes.lessThan) {
511-
finishOp(tt.lessThan, 1);
512531
} else {
513532
finishOp(tt.greaterThan, 1);
514533
}
@@ -695,8 +714,11 @@ export function getTokenFromCode(code: number): void {
695714
return;
696715

697716
case charCodes.lessThan:
717+
readToken_lt();
718+
return;
719+
698720
case charCodes.greaterThan:
699-
readToken_lt_gt(code);
721+
readToken_gt();
700722
return;
701723

702724
case charCodes.equalsTo:

src/parser/tokenizer/keywords.ts

+1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
export enum ContextualKeyword {
22
NONE,
33
_abstract,
4+
_accessor,
45
_as,
56
_asserts,
67
_async,

0 commit comments

Comments
 (0)