Skip to content

Commit

Permalink
add support for Lift Template Literal Restriction (microsoft#23801)
Browse files Browse the repository at this point in the history 
* add support for Lift Template Literal Restriction

* rename file and improve comment and tests

* fix NoSubstitutionTemplateLiteral support

* extract tagged template and add more test

* avoid useless parameter

* fix incorrect return node if cannot transform

* accept baseline

* correctly baseline

* accept baseline

* fix merge break

* fix merge break

* inline rescan template head or no subsititution template

* update scan error

* add comment and fix lint

* refactor and fix lint

* avoid blank

* fix merge conflict

* fix again

* fix again

* use multiple target

* fix space lint

Co-authored-by: Nathan Shively-Sanders <[email protected]>
  • Loading branch information
@Kingwl @sandersn
Kingwl and sandersn committed Feb 5, 2020
1 parent 2047118 commit 70399e1
Show file tree
Hide file tree
Showing 31 changed files with 1,647 additions and 125 deletions.
12 changes: 11 additions & 1 deletion src/compiler/binder.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4124,8 +4124,18 @@ namespace ts {
case SyntaxKind.TemplateHead:
case SyntaxKind.TemplateMiddle:
case SyntaxKind.TemplateTail:
case SyntaxKind.TemplateExpression:
if ((<NoSubstitutionTemplateLiteral | TemplateHead | TemplateMiddle | TemplateTail>node).templateFlags) {
transformFlags |= TransformFlags.AssertES2018;
break;
}
// falls through
case SyntaxKind.TaggedTemplateExpression:
if (hasInvalidEscape((<TaggedTemplateExpression>node).template)) {
transformFlags |= TransformFlags.AssertES2018;
break;
}
// falls through
case SyntaxKind.TemplateExpression:
case SyntaxKind.ShorthandPropertyAssignment:
case SyntaxKind.StaticKeyword:
case SyntaxKind.MetaProperty:
Expand Down
2 changes: 1 addition & 1 deletion src/compiler/factoryPublic.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1486,7 +1486,7 @@ namespace ts {

let token = rawTextScanner.scan();
if (token === SyntaxKind.CloseBracketToken) {
token = rawTextScanner.reScanTemplateToken();
token = rawTextScanner.reScanTemplateToken(/* isTaggedTemplate */ false);
}

if (rawTextScanner.isUnterminated()) {
Expand Down
33 changes: 22 additions & 11 deletions src/compiler/parser.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1137,8 +1137,12 @@ namespace ts {
return currentToken = scanner.reScanSlashToken();
}

function reScanTemplateToken(): SyntaxKind {
return currentToken = scanner.reScanTemplateToken();
function reScanTemplateToken(isTaggedTemplate: boolean): SyntaxKind {
return currentToken = scanner.reScanTemplateToken(isTaggedTemplate);
}

function reScanTemplateHeadOrNoSubstitutionTemplate(): SyntaxKind {
return currentToken = scanner.reScanTemplateHeadOrNoSubstitutionTemplate();
}

function reScanLessThanToken(): SyntaxKind {
Expand Down Expand Up @@ -2329,17 +2333,17 @@ namespace ts {
return allowIdentifierNames ? parseIdentifierName() : parseIdentifier();
}

function parseTemplateExpression(): TemplateExpression {
function parseTemplateExpression(isTaggedTemplate: boolean): TemplateExpression {
const template = <TemplateExpression>createNode(SyntaxKind.TemplateExpression);

template.head = parseTemplateHead();
template.head = parseTemplateHead(isTaggedTemplate);
Debug.assert(template.head.kind === SyntaxKind.TemplateHead, "Template head has wrong token kind");

const list = [];
const listPos = getNodePos();

do {
list.push(parseTemplateSpan());
list.push(parseTemplateSpan(isTaggedTemplate));
}
while (last(list).literal.kind === SyntaxKind.TemplateMiddle);

Expand All @@ -2348,13 +2352,13 @@ namespace ts {
return finishNode(template);
}

function parseTemplateSpan(): TemplateSpan {
function parseTemplateSpan(isTaggedTemplate: boolean): TemplateSpan {
const span = <TemplateSpan>createNode(SyntaxKind.TemplateSpan);
span.expression = allowInAnd(parseExpression);

let literal: TemplateMiddle | TemplateTail;
if (token() === SyntaxKind.CloseBraceToken) {
reScanTemplateToken();
reScanTemplateToken(isTaggedTemplate);
literal = parseTemplateMiddleOrTemplateTail();
}
else {
Expand All @@ -2369,7 +2373,10 @@ namespace ts {
return <LiteralExpression>parseLiteralLikeNode(token());
}

function parseTemplateHead(): TemplateHead {
function parseTemplateHead(isTaggedTemplate: boolean): TemplateHead {
if (isTaggedTemplate) {
reScanTemplateHeadOrNoSubstitutionTemplate();
}
const fragment = parseLiteralLikeNode(token());
Debug.assert(fragment.kind === SyntaxKind.TemplateHead, "Template head has wrong token kind");
return <TemplateHead>fragment;
Expand Down Expand Up @@ -2413,6 +2420,10 @@ namespace ts {
(<NumericLiteral>node).numericLiteralFlags = scanner.getTokenFlags() & TokenFlags.NumericLiteralFlags;
}

if (isTemplateLiteralKind(node.kind)) {
(<TemplateHead | TemplateMiddle | TemplateTail | NoSubstitutionTemplateLiteral>node).templateFlags = scanner.getTokenFlags() & TokenFlags.ContainsInvalidEscape;
}

nextToken();
finishNode(node);

Expand Down Expand Up @@ -4772,8 +4783,8 @@ namespace ts {
tagExpression.questionDotToken = questionDotToken;
tagExpression.typeArguments = typeArguments;
tagExpression.template = token() === SyntaxKind.NoSubstitutionTemplateLiteral
? <NoSubstitutionTemplateLiteral>parseLiteralNode()
: parseTemplateExpression();
? (reScanTemplateHeadOrNoSubstitutionTemplate(), <NoSubstitutionTemplateLiteral>parseLiteralNode())
: parseTemplateExpression(/*isTaggedTemplate*/ true);
if (questionDotToken || tag.flags & NodeFlags.OptionalChain) {
tagExpression.flags |= NodeFlags.OptionalChain;
}
Expand Down Expand Up @@ -4945,7 +4956,7 @@ namespace ts {
}
break;
case SyntaxKind.TemplateHead:
return parseTemplateExpression();
return parseTemplateExpression(/* isTaggedTemplate */ false);
}

return parseIdentifier(Diagnostics.Expression_expected);
Expand Down
80 changes: 72 additions & 8 deletions src/compiler/scanner.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,8 @@ namespace ts {
getTokenFlags(): TokenFlags;
reScanGreaterToken(): SyntaxKind;
reScanSlashToken(): SyntaxKind;
reScanTemplateToken(): SyntaxKind;
reScanTemplateToken(isTaggedTemplate: boolean): SyntaxKind;
reScanTemplateHeadOrNoSubstitutionTemplate(): SyntaxKind;
scanJsxIdentifier(): SyntaxKind;
scanJsxAttributeValue(): SyntaxKind;
reScanJsxAttributeValue(): SyntaxKind;
Expand Down Expand Up @@ -468,6 +469,14 @@ namespace ts {
return ch >= CharacterCodes._0 && ch <= CharacterCodes._9;
}

function isHexDigit(ch: number): boolean {
return isDigit(ch) || ch >= CharacterCodes.A && ch <= CharacterCodes.F || ch >= CharacterCodes.a && ch <= CharacterCodes.f;
}

function isCodePoint(code: number): boolean {
return code <= 0x10FFFF;
}

/* @internal */
export function isOctalDigit(ch: number): boolean {
return ch >= CharacterCodes._0 && ch <= CharacterCodes._7;
Expand Down Expand Up @@ -901,6 +910,7 @@ namespace ts {
reScanGreaterToken,
reScanSlashToken,
reScanTemplateToken,
reScanTemplateHeadOrNoSubstitutionTemplate,
scanJsxIdentifier,
scanJsxAttributeValue,
reScanJsxAttributeValue,
Expand Down Expand Up @@ -1164,7 +1174,7 @@ namespace ts {
* Sets the current 'tokenValue' and returns a NoSubstitutionTemplateLiteral or
* a literal component of a TemplateExpression.
*/
function scanTemplateAndSetTokenValue(): SyntaxKind {
function scanTemplateAndSetTokenValue(isTaggedTemplate: boolean): SyntaxKind {
const startedWithBacktick = text.charCodeAt(pos) === CharacterCodes.backtick;

pos++;
Expand Down Expand Up @@ -1202,7 +1212,7 @@ namespace ts {
// Escape character
if (currChar === CharacterCodes.backslash) {
contents += text.substring(start, pos);
contents += scanEscapeSequence();
contents += scanEscapeSequence(isTaggedTemplate);
start = pos;
continue;
}
Expand Down Expand Up @@ -1231,7 +1241,8 @@ namespace ts {
return resultingToken;
}

function scanEscapeSequence(): string {
function scanEscapeSequence(isTaggedTemplate?: boolean): string {
const start = pos;
pos++;
if (pos >= end) {
error(Diagnostics.Unexpected_end_of_text);
Expand All @@ -1241,6 +1252,12 @@ namespace ts {
pos++;
switch (ch) {
case CharacterCodes._0:
// '\01'
if (isTaggedTemplate && pos < end && isDigit(text.charCodeAt(pos))) {
pos++;
tokenFlags |= TokenFlags.ContainsInvalidEscape;
return text.substring(start, pos);
}
return "\0";
case CharacterCodes.b:
return "\b";
Expand All @@ -1259,10 +1276,41 @@ namespace ts {
case CharacterCodes.doubleQuote:
return "\"";
case CharacterCodes.u:
if (isTaggedTemplate) {
// '\u' or '\u0' or '\u00' or '\u000'
for (let escapePos = pos; escapePos < pos + 4; escapePos++) {
if (escapePos < end && !isHexDigit(text.charCodeAt(escapePos)) && text.charCodeAt(escapePos) !== CharacterCodes.openBrace) {
pos = escapePos;
tokenFlags |= TokenFlags.ContainsInvalidEscape;
return text.substring(start, pos);
}
}
}
// '\u{DDDDDDDD}'
if (pos < end && text.charCodeAt(pos) === CharacterCodes.openBrace) {
tokenFlags |= TokenFlags.ExtendedUnicodeEscape;
pos++;

// '\u{'
if (isTaggedTemplate && !isHexDigit(text.charCodeAt(pos))) {
tokenFlags |= TokenFlags.ContainsInvalidEscape;
return text.substring(start, pos);
}

if (isTaggedTemplate) {
const savePos = pos;
const escapedValueString = scanMinimumNumberOfHexDigits(1, /*canHaveSeparators*/ false);
const escapedValue = escapedValueString ? parseInt(escapedValueString, 16) : -1;

// '\u{Not Code Point' or '\u{CodePoint'
if (!isCodePoint(escapedValue) || text.charCodeAt(pos) !== CharacterCodes.closeBrace) {
tokenFlags |= TokenFlags.ContainsInvalidEscape;
return text.substring(start, pos);
}
else {
pos = savePos;
}
}
tokenFlags |= TokenFlags.ExtendedUnicodeEscape;
return scanExtendedUnicodeEscape();
}

Expand All @@ -1271,6 +1319,17 @@ namespace ts {
return scanHexadecimalEscape(/*numDigits*/ 4);

case CharacterCodes.x:
if (isTaggedTemplate) {
if (!isHexDigit(text.charCodeAt(pos))) {
tokenFlags |= TokenFlags.ContainsInvalidEscape;
return text.substring(start, pos);
}
else if (!isHexDigit(text.charCodeAt(pos + 1))) {
pos++;
tokenFlags |= TokenFlags.ContainsInvalidEscape;
return text.substring(start, pos);
}
}
// '\xDD'
return scanHexadecimalEscape(/*numDigits*/ 2);

Expand Down Expand Up @@ -1561,7 +1620,7 @@ namespace ts {
tokenValue = scanString();
return token = SyntaxKind.StringLiteral;
case CharacterCodes.backtick:
return token = scanTemplateAndSetTokenValue();
return token = scanTemplateAndSetTokenValue(/* isTaggedTemplate */ false);
case CharacterCodes.percent:
if (text.charCodeAt(pos + 1) === CharacterCodes.equals) {
return pos += 2, token = SyntaxKind.PercentEqualsToken;
Expand Down Expand Up @@ -2019,10 +2078,15 @@ namespace ts {
/**
* Unconditionally back up and scan a template expression portion.
*/
function reScanTemplateToken(): SyntaxKind {
function reScanTemplateToken(isTaggedTemplate: boolean): SyntaxKind {
Debug.assert(token === SyntaxKind.CloseBraceToken, "'reScanTemplateToken' should only be called on a '}'");
pos = tokenPos;
return token = scanTemplateAndSetTokenValue();
return token = scanTemplateAndSetTokenValue(isTaggedTemplate);
}

function reScanTemplateHeadOrNoSubstitutionTemplate(): SyntaxKind {
pos = tokenPos;
return token = scanTemplateAndSetTokenValue(/* isTaggedTemplate */ true);
}

function reScanJsxToken(): JsxTokenSyntaxKind {
Expand Down
Loading

0 comments on commit 70399e1

Please sign in to comment.