From 9276c09a94ced68460c2112577ea6df0bf437df4 Mon Sep 17 00:00:00 2001 From: AlekseyTs Date: Mon, 21 Mar 2022 13:45:16 -0700 Subject: [PATCH] Implement classification for UTF8 String Literals (#60265) --- .../CSharp/Portable/CSharpExtensions.cs | 2 +- .../SyntacticClassifierTests.cs | 337 +++++++++++++++++- .../Classification/ClassificationHelpers.cs | 5 +- 3 files changed, 338 insertions(+), 6 deletions(-) diff --git a/src/Compilers/CSharp/Portable/CSharpExtensions.cs b/src/Compilers/CSharp/Portable/CSharpExtensions.cs index d2ae4cdd4d06f..2518175c5c544 100644 --- a/src/Compilers/CSharp/Portable/CSharpExtensions.cs +++ b/src/Compilers/CSharp/Portable/CSharpExtensions.cs @@ -232,7 +232,7 @@ public static bool IsReservedKeyword(this SyntaxToken token) public static bool IsVerbatimStringLiteral(this SyntaxToken token) { - return token.IsKind(SyntaxKind.StringLiteralToken) && token.Text.Length > 0 && token.Text[0] == '@'; + return token.Kind() is (SyntaxKind.StringLiteralToken or SyntaxKind.UTF8StringLiteralToken) && token.Text.Length > 0 && token.Text[0] == '@'; } public static bool IsVerbatimIdentifier(this SyntaxToken token) diff --git a/src/EditorFeatures/CSharpTest/Classification/SyntacticClassifierTests.cs b/src/EditorFeatures/CSharpTest/Classification/SyntacticClassifierTests.cs index fd5fcbb83f2a5..fb627ba3d85b0 100644 --- a/src/EditorFeatures/CSharpTest/Classification/SyntacticClassifierTests.cs +++ b/src/EditorFeatures/CSharpTest/Classification/SyntacticClassifierTests.cs @@ -349,6 +349,24 @@ await TestInMethodAsync(@"@""goo""", Verbatim(@"@""goo""")); } + [Theory] + [CombinatorialData] + public async Task VerbatimStringLiteralsUTF8_01(TestHost testHost) + { + await TestInMethodAsync(@"@""goo""u8", + testHost, + Verbatim(@"@""goo""u8")); + } + + [Theory] + [CombinatorialData] + public async Task VerbatimStringLiteralsUTF8_02(TestHost testHost) + { + await TestInMethodAsync(@"@""goo""U8", + testHost, + Verbatim(@"@""goo""U8")); + } + /// /// Should show up as soon as we get the @\" typed out /// @@ -366,7 +384,7 @@ await TestAsync(@"@""", /// [Theory] [CombinatorialData] - public async Task VerbatimStringLiteral3(TestHost testHost) + public async Task VerbatimStringLiterals3(TestHost testHost) { await TestAsync(@"goo @""", testHost, @@ -379,7 +397,7 @@ await TestAsync(@"goo @""", /// [Theory] [CombinatorialData] - public async Task VerbatimStringLiteral4(TestHost testHost) + public async Task VerbatimStringLiterals4(TestHost testHost) { var code = @" @@ -395,7 +413,7 @@ await TestAsync(code, [Theory] [CombinatorialData] - public async Task VerbatimStringLiteral5(TestHost testHost) + public async Task VerbatimStringLiterals5(TestHost testHost) { var code = @" @@ -412,10 +430,48 @@ await TestInMethodAsync(code, Local("stuff")); } + [Theory] + [CombinatorialData] + public async Task VerbatimStringLiteralsUTF8_03(TestHost testHost) + { + var code = @" + +@"" goo bar +and +on a new line ""u8 +more stuff"; + await TestInMethodAsync(code, + testHost, + Verbatim(@"@"" goo bar +and +on a new line ""u8"), + Identifier("more"), + Local("stuff")); + } + + [Theory] + [CombinatorialData] + public async Task VerbatimStringLiteralsUTF8_04(TestHost testHost) + { + var code = @" + +@"" goo bar +and +on a new line ""U8 +more stuff"; + await TestInMethodAsync(code, + testHost, + Verbatim(@"@"" goo bar +and +on a new line ""U8"), + Identifier("more"), + Local("stuff")); + } + [Theory] [WorkItem(44423, "https://github.com/dotnet/roslyn/issues/44423")] [CombinatorialData] - public async Task VerbatimStringLiteral6(bool script, TestHost testHost) + public async Task VerbatimStringLiterals6(bool script, TestHost testHost) { var code = @"string s = @""""""/*"";"; @@ -433,6 +489,46 @@ await TestAsync( Punctuation.Semicolon); } + [Theory] + [CombinatorialData] + public async Task VerbatimStringLiteralsUTF8_05(bool script, TestHost testHost) + { + var code = @"string s = @""""""/*""u8;"; + + var parseOptions = script ? Options.Script : null; + + await TestAsync( + code, + code, + testHost, + parseOptions, + Keyword("string"), + script ? Field("s") : Local("s"), + Operators.Equals, + Verbatim(@"@""""""/*""u8"), + Punctuation.Semicolon); + } + + [Theory] + [CombinatorialData] + public async Task VerbatimStringLiteralsUTF8_06(bool script, TestHost testHost) + { + var code = @"string s = @""""""/*""u8;"; + + var parseOptions = script ? Options.Script : null; + + await TestAsync( + code, + code, + testHost, + parseOptions, + Keyword("string"), + script ? Field("s") : Local("s"), + Operators.Equals, + Verbatim(@"@""""""/*""u8"), + Punctuation.Semicolon); + } + [Theory] [CombinatorialData] public async Task StringLiteral1(TestHost testHost) @@ -442,6 +538,24 @@ await TestAsync(@"""goo""", String(@"""goo""")); } + [Theory] + [CombinatorialData] + public async Task StringLiteralUTF8_01(TestHost testHost) + { + await TestAsync(@"""goo""u8", + testHost, + String(@"""goo""u8")); + } + + [Theory] + [CombinatorialData] + public async Task StringLiteralUTF8_02(TestHost testHost) + { + await TestAsync(@"""goo""U8", + testHost, + String(@"""goo""U8")); + } + [Theory] [CombinatorialData] public async Task StringLiteral2(TestHost testHost) @@ -451,6 +565,24 @@ await TestAsync(@"""""", String(@"""""")); } + [Theory] + [CombinatorialData] + public async Task StringLiteralUTF8_03(TestHost testHost) + { + await TestAsync(@"""""u8", + testHost, + String(@"""""u8")); + } + + [Theory] + [CombinatorialData] + public async Task StringLiteralUTF8_04(TestHost testHost) + { + await TestAsync(@"""""U8", + testHost, + String(@"""""U8")); + } + [Theory] [CombinatorialData] public async Task CharacterLiteral1(TestHost testHost) @@ -5745,6 +5877,203 @@ await TestAsync(code, Punctuation.CloseCurly); } + [Theory] + [CombinatorialData] + public async Task TestRawStringLiteralUTF8_01(TestHost testHost) + { + var code = @" +class C +{ + public static void M(int x) + { + var s = """"""Hello world""""""u8; + } +}"; + + await TestAsync(code, + testHost, + Keyword("class"), + Class("C"), + Punctuation.OpenCurly, + Keyword("public"), + Keyword("static"), + Keyword("void"), + Method("M"), + Static("M"), + Punctuation.OpenParen, + Keyword("int"), + Parameter("x"), + Punctuation.CloseParen, + Punctuation.OpenCurly, + Keyword("var"), + Local("s"), + Operators.Equals, + String("\"\"\"Hello world\"\"\"u8"), + Punctuation.Semicolon, + Punctuation.CloseCurly, + Punctuation.CloseCurly); + } + + [Theory] + [CombinatorialData] + public async Task TestRawStringLiteralUTF8_02(TestHost testHost) + { + var code = @" +class C +{ + public static void M(int x) + { + var s = """"""Hello world""""""U8; + } +}"; + + await TestAsync(code, + testHost, + Keyword("class"), + Class("C"), + Punctuation.OpenCurly, + Keyword("public"), + Keyword("static"), + Keyword("void"), + Method("M"), + Static("M"), + Punctuation.OpenParen, + Keyword("int"), + Parameter("x"), + Punctuation.CloseParen, + Punctuation.OpenCurly, + Keyword("var"), + Local("s"), + Operators.Equals, + String("\"\"\"Hello world\"\"\"U8"), + Punctuation.Semicolon, + Punctuation.CloseCurly, + Punctuation.CloseCurly); + } + + [Theory] + [CombinatorialData] + public async Task TestRawStringLiteralMultiline(TestHost testHost) + { + var code = @" +class C +{ + public static void M(int x) + { + var s = """""" + Hello world + """"""; + } +}"; + + await TestAsync(code, + testHost, + Keyword("class"), + Class("C"), + Punctuation.OpenCurly, + Keyword("public"), + Keyword("static"), + Keyword("void"), + Method("M"), + Static("M"), + Punctuation.OpenParen, + Keyword("int"), + Parameter("x"), + Punctuation.CloseParen, + Punctuation.OpenCurly, + Keyword("var"), + Local("s"), + Operators.Equals, + String(@""""""" + Hello world + """""""), + Punctuation.Semicolon, + Punctuation.CloseCurly, + Punctuation.CloseCurly); + } + + [Theory] + [CombinatorialData] + public async Task TestRawStringLiteralMultilineUTF8_01(TestHost testHost) + { + var code = @" +class C +{ + public static void M(int x) + { + var s = """""" + Hello world + """"""u8; + } +}"; + + await TestAsync(code, + testHost, + Keyword("class"), + Class("C"), + Punctuation.OpenCurly, + Keyword("public"), + Keyword("static"), + Keyword("void"), + Method("M"), + Static("M"), + Punctuation.OpenParen, + Keyword("int"), + Parameter("x"), + Punctuation.CloseParen, + Punctuation.OpenCurly, + Keyword("var"), + Local("s"), + Operators.Equals, + String(@""""""" + Hello world + """"""u8"), + Punctuation.Semicolon, + Punctuation.CloseCurly, + Punctuation.CloseCurly); + } + + [Theory] + [CombinatorialData] + public async Task TestRawStringLiteralMultilineUTF8_02(TestHost testHost) + { + var code = @" +class C +{ + public static void M(int x) + { + var s = """""" + Hello world + """"""U8; + } +}"; + + await TestAsync(code, + testHost, + Keyword("class"), + Class("C"), + Punctuation.OpenCurly, + Keyword("public"), + Keyword("static"), + Keyword("void"), + Method("M"), + Static("M"), + Punctuation.OpenParen, + Keyword("int"), + Parameter("x"), + Punctuation.CloseParen, + Punctuation.OpenCurly, + Keyword("var"), + Local("s"), + Operators.Equals, + String(@""""""" + Hello world + """"""U8"), + Punctuation.Semicolon, + Punctuation.CloseCurly, + Punctuation.CloseCurly); + } + [Theory] [CombinatorialData] public async Task TestRawStringLiteralInterpolation1(TestHost testHost) diff --git a/src/Workspaces/CSharp/Portable/Classification/ClassificationHelpers.cs b/src/Workspaces/CSharp/Portable/Classification/ClassificationHelpers.cs index 4df12c5b2b8e8..6581f6b1470f9 100644 --- a/src/Workspaces/CSharp/Portable/Classification/ClassificationHelpers.cs +++ b/src/Workspaces/CSharp/Portable/Classification/ClassificationHelpers.cs @@ -141,6 +141,7 @@ private static bool IsControlStatementKind(SyntaxKind kind) private static bool IsStringToken(SyntaxToken token) { return token.IsKind(SyntaxKind.StringLiteralToken) + || token.IsKind(SyntaxKind.UTF8StringLiteralToken) || token.IsKind(SyntaxKind.CharacterLiteralToken) || token.IsKind(SyntaxKind.InterpolatedStringStartToken) || token.IsKind(SyntaxKind.InterpolatedVerbatimStringStartToken) @@ -150,7 +151,9 @@ private static bool IsStringToken(SyntaxToken token) || token.IsKind(SyntaxKind.InterpolatedSingleLineRawStringStartToken) || token.IsKind(SyntaxKind.InterpolatedMultiLineRawStringStartToken) || token.IsKind(SyntaxKind.SingleLineRawStringLiteralToken) - || token.IsKind(SyntaxKind.MultiLineRawStringLiteralToken); + || token.IsKind(SyntaxKind.UTF8SingleLineRawStringLiteralToken) + || token.IsKind(SyntaxKind.MultiLineRawStringLiteralToken) + || token.IsKind(SyntaxKind.UTF8MultiLineRawStringLiteralToken); } private static bool IsVerbatimStringToken(SyntaxToken token)