From 27a0ea973a02a4dd01ffa63d4cba427106c3755f Mon Sep 17 00:00:00 2001 From: AlekseyTs Date: Thu, 20 Jan 2022 21:02:39 -0800 Subject: [PATCH 1/5] Support `u8` type suffix for UTF8 string literals. Proposal - https://github.com/dotnet/csharplang/blob/main/proposals/utf8-string-literals.md --- .../Portable/Binder/Binder_Expressions.cs | 15 + .../Binder/EarlyWellKnownAttributeBinder.cs | 1 + .../Semantics/Conversions/Conversion.cs | 2 +- .../Semantics/Conversions/ConversionsBase.cs | 8 +- .../CSharp/Portable/BoundTree/BoundNodes.xml | 6 + .../Portable/FlowAnalysis/AbstractFlowPass.cs | 5 + .../Portable/FlowAnalysis/NullableWalker.cs | 8 + .../Generated/BoundNodes.xml.Generated.cs | 72 ++ .../Portable/Generated/CSharp.Generated.g4 | 5 + .../Syntax.xml.Internal.Generated.cs | 4 + .../Syntax.xml.Main.Generated.cs | 3 + .../Syntax.xml.Syntax.Generated.cs | 1 + .../LocalRewriter/LocalRewriter_Conversion.cs | 70 +- .../Operations/CSharpOperationFactory.cs | 1 + .../CSharp/Portable/Parser/LanguageParser.cs | 4 + src/Compilers/CSharp/Portable/Parser/Lexer.cs | 1 + .../Portable/Parser/Lexer_StringLiteral.cs | 18 +- .../CSharp/Portable/PublicAPI.Unshipped.txt | 2 + .../CSharp/Portable/Syntax/Syntax.xml | 2 + .../Portable/Syntax/SyntaxEquivalence.cs | 1 + .../CSharp/Portable/Syntax/SyntaxKind.cs | 3 + .../CSharp/Portable/Syntax/SyntaxKindFacts.cs | 3 + .../Semantics/Utf8StringsLiteralsTests.cs | 861 +++++++++++++++++- .../Parsing/LineSpanDirectiveParsingTests.cs | 44 + .../Test/Syntax/Parsing/ParsingTests.cs | 1 + .../Parsing/UTF8StringLiteralsParsingTests.cs | 300 ++++++ src/Compilers/Test/Core/TestResource.resx | 1 + 27 files changed, 1402 insertions(+), 40 deletions(-) create mode 100644 src/Compilers/CSharp/Test/Syntax/Parsing/UTF8StringLiteralsParsingTests.cs diff --git a/src/Compilers/CSharp/Portable/Binder/Binder_Expressions.cs b/src/Compilers/CSharp/Portable/Binder/Binder_Expressions.cs index a11f9e07bb228..5c0d62b05f355 100644 --- a/src/Compilers/CSharp/Portable/Binder/Binder_Expressions.cs +++ b/src/Compilers/CSharp/Portable/Binder/Binder_Expressions.cs @@ -673,6 +673,9 @@ private BoundExpression BindExpressionInternal(ExpressionSyntax node, BindingDia case SyntaxKind.NullLiteralExpression: return BindLiteralConstant((LiteralExpressionSyntax)node, diagnostics); + case SyntaxKind.UTF8StringLiteralExpression: + return BindUTF8StringLiteral((LiteralExpressionSyntax)node, diagnostics); + case SyntaxKind.DefaultLiteralExpression: return new BoundDefaultLiteral(node); @@ -5897,6 +5900,18 @@ private BoundLiteral BindLiteralConstant(LiteralExpressionSyntax node, BindingDi return new BoundLiteral(node, cv, type); } + private BoundUTF8String BindUTF8StringLiteral(LiteralExpressionSyntax node, BindingDiagnosticBag diagnostics) + { + Debug.Assert(node.Kind() == SyntaxKind.UTF8StringLiteralExpression); + + CheckFeatureAvailability(node, MessageID.IDS_FeatureUTF8StringLiterals, diagnostics); + + var value = (string)node.Token.Value; + var type = ArrayTypeSymbol.CreateSZArray(Compilation.Assembly, TypeWithAnnotations.Create(GetSpecialType(SpecialType.System_Byte, diagnostics, node))); + + return new BoundUTF8String(node, value, type); + } + private BoundExpression BindCheckedExpression(CheckedExpressionSyntax node, BindingDiagnosticBag diagnostics) { // the binder is not cached since we only cache statement level binders diff --git a/src/Compilers/CSharp/Portable/Binder/EarlyWellKnownAttributeBinder.cs b/src/Compilers/CSharp/Portable/Binder/EarlyWellKnownAttributeBinder.cs index 149a790972594..95a75ba13e072 100644 --- a/src/Compilers/CSharp/Portable/Binder/EarlyWellKnownAttributeBinder.cs +++ b/src/Compilers/CSharp/Portable/Binder/EarlyWellKnownAttributeBinder.cs @@ -82,6 +82,7 @@ internal static bool CanBeValidAttributeArgument(ExpressionSyntax node, Binder t // Literals (including the null literal). case SyntaxKind.NumericLiteralExpression: case SyntaxKind.StringLiteralExpression: + case SyntaxKind.UTF8StringLiteralExpression: case SyntaxKind.CharacterLiteralExpression: case SyntaxKind.TrueLiteralExpression: case SyntaxKind.FalseLiteralExpression: diff --git a/src/Compilers/CSharp/Portable/Binder/Semantics/Conversions/Conversion.cs b/src/Compilers/CSharp/Portable/Binder/Semantics/Conversions/Conversion.cs index c5b7b052f1a85..bf206a18343f2 100644 --- a/src/Compilers/CSharp/Portable/Binder/Semantics/Conversions/Conversion.cs +++ b/src/Compilers/CSharp/Portable/Binder/Semantics/Conversions/Conversion.cs @@ -828,7 +828,7 @@ public bool IsConstantExpression } /// - /// Returns true if the conversion is an implicit Utf8 strings literal conversion. + /// Returns true if the conversion is an implicit Utf8 string literal conversion. /// public bool IsUtf8StringLiteral { diff --git a/src/Compilers/CSharp/Portable/Binder/Semantics/Conversions/ConversionsBase.cs b/src/Compilers/CSharp/Portable/Binder/Semantics/Conversions/ConversionsBase.cs index c0228dff0686e..1a218e82bb5e1 100644 --- a/src/Compilers/CSharp/Portable/Binder/Semantics/Conversions/ConversionsBase.cs +++ b/src/Compilers/CSharp/Portable/Binder/Semantics/Conversions/ConversionsBase.cs @@ -1110,10 +1110,10 @@ private Conversion ClassifyImplicitUtf8StringLiteralConversion(BoundExpression s if (constantValue?.IsString == true && // PROTOTYPE(UTF8StringLiterals) : confirm if we actually want it to work with 'null' constant value. sourceExpression.Type?.SpecialType == SpecialType.System_String && (destination is ArrayTypeSymbol { IsSZArray: true, ElementType.SpecialType: SpecialType.System_Byte } || // byte[] - ((destinationOriginalDefinition.Equals(compilation.GetWellKnownType(WellKnownType.System_Span_T), TypeCompareKind.AllIgnoreOptions) || // Span + (destinationOriginalDefinition.TypeKind == TypeKind.Struct && destinationOriginalDefinition.IsRefLikeType && + (destinationOriginalDefinition.Equals(compilation.GetWellKnownType(WellKnownType.System_Span_T), TypeCompareKind.AllIgnoreOptions) || // Span destinationOriginalDefinition.Equals(compilation.GetWellKnownType(WellKnownType.System_ReadOnlySpan_T), TypeCompareKind.AllIgnoreOptions)) && // ReadOnlySpan - destinationOriginalDefinition.TypeKind == TypeKind.Struct && destinationOriginalDefinition.IsRefLikeType && - ((NamedTypeSymbol)destination).TypeArgumentsWithAnnotationsNoUseSiteDiagnostics.Single().SpecialType == SpecialType.System_Byte))) // T is byte + ((NamedTypeSymbol)destination).TypeArgumentsWithAnnotationsNoUseSiteDiagnostics.Single().SpecialType == SpecialType.System_Byte))) // T is byte { return Conversion.ImplicitUtf8StringLiteral; } @@ -1638,7 +1638,7 @@ internal Conversion GetCallerLineNumberConversion(TypeSymbol destination, ref Co TypeSymbol expectedAttributeType = corLibrary.GetSpecialType(SpecialType.System_Int32); BoundLiteral intMaxValueLiteral = new BoundLiteral(syntaxNode, ConstantValue.Create(int.MaxValue), expectedAttributeType); - // Below is a dupliucation of relevant parts of ClassifyStandardImplicitConversion method. + // Below is a duplication of relevant parts of ClassifyStandardImplicitConversion method. // It needs a compilation instance, but we don't have it and the relevant parts actually do not depend on // a compilation. if (HasImplicitEnumerationConversion(intMaxValueLiteral, destination)) diff --git a/src/Compilers/CSharp/Portable/BoundTree/BoundNodes.xml b/src/Compilers/CSharp/Portable/BoundTree/BoundNodes.xml index 819613f95226e..2392b58d7b13b 100644 --- a/src/Compilers/CSharp/Portable/BoundTree/BoundNodes.xml +++ b/src/Compilers/CSharp/Portable/BoundTree/BoundNodes.xml @@ -1288,6 +1288,12 @@ + + + + + + diff --git a/src/Compilers/CSharp/Portable/FlowAnalysis/AbstractFlowPass.cs b/src/Compilers/CSharp/Portable/FlowAnalysis/AbstractFlowPass.cs index 9515734437ab1..60fb502caead8 100644 --- a/src/Compilers/CSharp/Portable/FlowAnalysis/AbstractFlowPass.cs +++ b/src/Compilers/CSharp/Portable/FlowAnalysis/AbstractFlowPass.cs @@ -1543,6 +1543,11 @@ public override BoundNode VisitLiteral(BoundLiteral node) return null; } + public override BoundNode VisitUTF8String(BoundUTF8String node) + { + return null; + } + protected void SplitIfBooleanConstant(BoundExpression node) { if (node.ConstantValue is { IsBoolean: true, BooleanValue: bool booleanValue }) diff --git a/src/Compilers/CSharp/Portable/FlowAnalysis/NullableWalker.cs b/src/Compilers/CSharp/Portable/FlowAnalysis/NullableWalker.cs index 2e1098cd19211..8c0a8807275b5 100644 --- a/src/Compilers/CSharp/Portable/FlowAnalysis/NullableWalker.cs +++ b/src/Compilers/CSharp/Portable/FlowAnalysis/NullableWalker.cs @@ -10357,6 +10357,14 @@ private TypeWithState InferResultNullabilityOfBinaryLogicalOperator(BoundExpress return result; } + public override BoundNode? VisitUTF8String(BoundUTF8String node) + { + Debug.Assert(!IsConditionalState); + var result = base.VisitUTF8String(node); + SetNotNullResult(node); + return result; + } + public override BoundNode? VisitPreviousSubmissionReference(BoundPreviousSubmissionReference node) { var result = base.VisitPreviousSubmissionReference(node); diff --git a/src/Compilers/CSharp/Portable/Generated/BoundNodes.xml.Generated.cs b/src/Compilers/CSharp/Portable/Generated/BoundNodes.xml.Generated.cs index 27917beaaab63..635303a0282b4 100644 --- a/src/Compilers/CSharp/Portable/Generated/BoundNodes.xml.Generated.cs +++ b/src/Compilers/CSharp/Portable/Generated/BoundNodes.xml.Generated.cs @@ -124,6 +124,7 @@ internal enum BoundKind : byte TryStatement, CatchBlock, Literal, + UTF8String, ThisReference, PreviousSubmissionReference, HostObjectMemberReference, @@ -4244,6 +4245,47 @@ public BoundLiteral Update(ConstantValue? constantValueOpt, TypeSymbol? type) } } + internal sealed partial class BoundUTF8String : BoundExpression + { + public BoundUTF8String(SyntaxNode syntax, string value, TypeSymbol type, bool hasErrors) + : base(BoundKind.UTF8String, syntax, type, hasErrors) + { + + RoslynDebug.Assert(value is object, "Field 'value' cannot be null (make the type nullable in BoundNodes.xml to remove this check)"); + RoslynDebug.Assert(type is object, "Field 'type' cannot be null (make the type nullable in BoundNodes.xml to remove this check)"); + + this.Value = value; + } + + public BoundUTF8String(SyntaxNode syntax, string value, TypeSymbol type) + : base(BoundKind.UTF8String, syntax, type) + { + + RoslynDebug.Assert(value is object, "Field 'value' cannot be null (make the type nullable in BoundNodes.xml to remove this check)"); + RoslynDebug.Assert(type is object, "Field 'type' cannot be null (make the type nullable in BoundNodes.xml to remove this check)"); + + this.Value = value; + } + + + public new TypeSymbol Type => base.Type!; + + public string Value { get; } + [DebuggerStepThrough] + public override BoundNode? Accept(BoundTreeVisitor visitor) => visitor.VisitUTF8String(this); + + public BoundUTF8String Update(string value, TypeSymbol type) + { + if (value != this.Value || !TypeSymbol.Equals(type, this.Type, TypeCompareKind.ConsiderEverything)) + { + var result = new BoundUTF8String(this.Syntax, value, type, this.HasErrors); + result.CopyAttributes(this); + return result; + } + return this; + } + } + internal sealed partial class BoundThisReference : BoundExpression { public BoundThisReference(SyntaxNode syntax, TypeSymbol type, bool hasErrors) @@ -8980,6 +9022,8 @@ internal R VisitInternal(BoundNode node, A arg) return VisitCatchBlock((BoundCatchBlock)node, arg); case BoundKind.Literal: return VisitLiteral((BoundLiteral)node, arg); + case BoundKind.UTF8String: + return VisitUTF8String((BoundUTF8String)node, arg); case BoundKind.ThisReference: return VisitThisReference((BoundThisReference)node, arg); case BoundKind.PreviousSubmissionReference: @@ -9320,6 +9364,7 @@ internal abstract partial class BoundTreeVisitor public virtual R VisitTryStatement(BoundTryStatement node, A arg) => this.DefaultVisit(node, arg); public virtual R VisitCatchBlock(BoundCatchBlock node, A arg) => this.DefaultVisit(node, arg); public virtual R VisitLiteral(BoundLiteral node, A arg) => this.DefaultVisit(node, arg); + public virtual R VisitUTF8String(BoundUTF8String node, A arg) => this.DefaultVisit(node, arg); public virtual R VisitThisReference(BoundThisReference node, A arg) => this.DefaultVisit(node, arg); public virtual R VisitPreviousSubmissionReference(BoundPreviousSubmissionReference node, A arg) => this.DefaultVisit(node, arg); public virtual R VisitHostObjectMemberReference(BoundHostObjectMemberReference node, A arg) => this.DefaultVisit(node, arg); @@ -9542,6 +9587,7 @@ internal abstract partial class BoundTreeVisitor public virtual BoundNode? VisitTryStatement(BoundTryStatement node) => this.DefaultVisit(node); public virtual BoundNode? VisitCatchBlock(BoundCatchBlock node) => this.DefaultVisit(node); public virtual BoundNode? VisitLiteral(BoundLiteral node) => this.DefaultVisit(node); + public virtual BoundNode? VisitUTF8String(BoundUTF8String node) => this.DefaultVisit(node); public virtual BoundNode? VisitThisReference(BoundThisReference node) => this.DefaultVisit(node); public virtual BoundNode? VisitPreviousSubmissionReference(BoundPreviousSubmissionReference node) => this.DefaultVisit(node); public virtual BoundNode? VisitHostObjectMemberReference(BoundHostObjectMemberReference node) => this.DefaultVisit(node); @@ -10090,6 +10136,7 @@ internal abstract partial class BoundTreeWalker : BoundTreeVisitor return null; } public override BoundNode? VisitLiteral(BoundLiteral node) => null; + public override BoundNode? VisitUTF8String(BoundUTF8String node) => null; public override BoundNode? VisitThisReference(BoundThisReference node) => null; public override BoundNode? VisitPreviousSubmissionReference(BoundPreviousSubmissionReference node) => null; public override BoundNode? VisitHostObjectMemberReference(BoundHostObjectMemberReference node) => null; @@ -11241,6 +11288,11 @@ internal abstract partial class BoundTreeRewriter : BoundTreeVisitor TypeSymbol? type = this.VisitType(node.Type); return node.Update(node.ConstantValueOpt, type); } + public override BoundNode? VisitUTF8String(BoundUTF8String node) + { + TypeSymbol? type = this.VisitType(node.Type); + return node.Update(node.Value, type); + } public override BoundNode? VisitThisReference(BoundThisReference node) { TypeSymbol? type = this.VisitType(node.Type); @@ -13130,6 +13182,18 @@ public NullabilityRewriter(ImmutableDictionary new TreeDumperNode("uTF8String", null, new TreeDumperNode[] + { + new TreeDumperNode("value", node.Value, null), + new TreeDumperNode("type", node.Type, null), + new TreeDumperNode("isSuppressed", node.IsSuppressed, null), + new TreeDumperNode("hasErrors", node.HasErrors, null) + } + ); public override TreeDumperNode VisitThisReference(BoundThisReference node, object? arg) => new TreeDumperNode("thisReference", null, new TreeDumperNode[] { new TreeDumperNode("type", node.Type, null), diff --git a/src/Compilers/CSharp/Portable/Generated/CSharp.Generated.g4 b/src/Compilers/CSharp/Portable/Generated/CSharp.Generated.g4 index 7f8ef199e869d..05e136115808d 100644 --- a/src/Compilers/CSharp/Portable/Generated/CSharp.Generated.g4 +++ b/src/Compilers/CSharp/Portable/Generated/CSharp.Generated.g4 @@ -920,6 +920,7 @@ literal_expression | character_literal_token | numeric_literal_token | string_literal_token + | utf_8_string_literal_token ; make_ref_expression @@ -1363,6 +1364,10 @@ syntax_token : /* see lexical specification */ ; +utf_8_string_literal_token + : /* see lexical specification */ + ; + xml_text_literal_token : /* see lexical specification */ ; diff --git a/src/Compilers/CSharp/Portable/Generated/CSharpSyntaxGenerator/CSharpSyntaxGenerator.SourceGenerator/Syntax.xml.Internal.Generated.cs b/src/Compilers/CSharp/Portable/Generated/CSharpSyntaxGenerator/CSharpSyntaxGenerator.SourceGenerator/Syntax.xml.Internal.Generated.cs index 248a299cfe280..84b670d2c5647 100644 --- a/src/Compilers/CSharp/Portable/Generated/CSharpSyntaxGenerator/CSharpSyntaxGenerator.SourceGenerator/Syntax.xml.Internal.Generated.cs +++ b/src/Compilers/CSharp/Portable/Generated/CSharpSyntaxGenerator/CSharpSyntaxGenerator.SourceGenerator/Syntax.xml.Internal.Generated.cs @@ -36270,6 +36270,7 @@ public LiteralExpressionSyntax LiteralExpression(SyntaxKind kind, SyntaxToken to case SyntaxKind.ArgListExpression: case SyntaxKind.NumericLiteralExpression: case SyntaxKind.StringLiteralExpression: + case SyntaxKind.UTF8StringLiteralExpression: case SyntaxKind.CharacterLiteralExpression: case SyntaxKind.TrueLiteralExpression: case SyntaxKind.FalseLiteralExpression: @@ -36284,6 +36285,7 @@ public LiteralExpressionSyntax LiteralExpression(SyntaxKind kind, SyntaxToken to case SyntaxKind.ArgListKeyword: case SyntaxKind.NumericLiteralToken: case SyntaxKind.StringLiteralToken: + case SyntaxKind.UTF8StringLiteralToken: case SyntaxKind.CharacterLiteralToken: case SyntaxKind.TrueKeyword: case SyntaxKind.FalseKeyword: @@ -41284,6 +41286,7 @@ public static LiteralExpressionSyntax LiteralExpression(SyntaxKind kind, SyntaxT case SyntaxKind.ArgListExpression: case SyntaxKind.NumericLiteralExpression: case SyntaxKind.StringLiteralExpression: + case SyntaxKind.UTF8StringLiteralExpression: case SyntaxKind.CharacterLiteralExpression: case SyntaxKind.TrueLiteralExpression: case SyntaxKind.FalseLiteralExpression: @@ -41298,6 +41301,7 @@ public static LiteralExpressionSyntax LiteralExpression(SyntaxKind kind, SyntaxT case SyntaxKind.ArgListKeyword: case SyntaxKind.NumericLiteralToken: case SyntaxKind.StringLiteralToken: + case SyntaxKind.UTF8StringLiteralToken: case SyntaxKind.CharacterLiteralToken: case SyntaxKind.TrueKeyword: case SyntaxKind.FalseKeyword: diff --git a/src/Compilers/CSharp/Portable/Generated/CSharpSyntaxGenerator/CSharpSyntaxGenerator.SourceGenerator/Syntax.xml.Main.Generated.cs b/src/Compilers/CSharp/Portable/Generated/CSharpSyntaxGenerator/CSharpSyntaxGenerator.SourceGenerator/Syntax.xml.Main.Generated.cs index ce862c9412c74..e5be80b79d87a 100644 --- a/src/Compilers/CSharp/Portable/Generated/CSharpSyntaxGenerator/CSharpSyntaxGenerator.SourceGenerator/Syntax.xml.Main.Generated.cs +++ b/src/Compilers/CSharp/Portable/Generated/CSharpSyntaxGenerator/CSharpSyntaxGenerator.SourceGenerator/Syntax.xml.Main.Generated.cs @@ -2836,6 +2836,7 @@ public static LiteralExpressionSyntax LiteralExpression(SyntaxKind kind, SyntaxT case SyntaxKind.ArgListExpression: case SyntaxKind.NumericLiteralExpression: case SyntaxKind.StringLiteralExpression: + case SyntaxKind.UTF8StringLiteralExpression: case SyntaxKind.CharacterLiteralExpression: case SyntaxKind.TrueLiteralExpression: case SyntaxKind.FalseLiteralExpression: @@ -2848,6 +2849,7 @@ public static LiteralExpressionSyntax LiteralExpression(SyntaxKind kind, SyntaxT case SyntaxKind.ArgListKeyword: case SyntaxKind.NumericLiteralToken: case SyntaxKind.StringLiteralToken: + case SyntaxKind.UTF8StringLiteralToken: case SyntaxKind.CharacterLiteralToken: case SyntaxKind.TrueKeyword: case SyntaxKind.FalseKeyword: @@ -2868,6 +2870,7 @@ private static SyntaxKind GetLiteralExpressionTokenKind(SyntaxKind kind) SyntaxKind.ArgListExpression => SyntaxKind.ArgListKeyword, SyntaxKind.NumericLiteralExpression => SyntaxKind.NumericLiteralToken, SyntaxKind.StringLiteralExpression => SyntaxKind.StringLiteralToken, + SyntaxKind.UTF8StringLiteralExpression => SyntaxKind.UTF8StringLiteralToken, SyntaxKind.CharacterLiteralExpression => SyntaxKind.CharacterLiteralToken, SyntaxKind.TrueLiteralExpression => SyntaxKind.TrueKeyword, SyntaxKind.FalseLiteralExpression => SyntaxKind.FalseKeyword, diff --git a/src/Compilers/CSharp/Portable/Generated/CSharpSyntaxGenerator/CSharpSyntaxGenerator.SourceGenerator/Syntax.xml.Syntax.Generated.cs b/src/Compilers/CSharp/Portable/Generated/CSharpSyntaxGenerator/CSharpSyntaxGenerator.SourceGenerator/Syntax.xml.Syntax.Generated.cs index a3893c47708be..94f56ff4c61ee 100644 --- a/src/Compilers/CSharp/Portable/Generated/CSharpSyntaxGenerator/CSharpSyntaxGenerator.SourceGenerator/Syntax.xml.Syntax.Generated.cs +++ b/src/Compilers/CSharp/Portable/Generated/CSharpSyntaxGenerator/CSharpSyntaxGenerator.SourceGenerator/Syntax.xml.Syntax.Generated.cs @@ -1947,6 +1947,7 @@ public BaseExpressionSyntax Update(SyntaxToken token) /// /// /// + /// /// /// /// diff --git a/src/Compilers/CSharp/Portable/Lowering/LocalRewriter/LocalRewriter_Conversion.cs b/src/Compilers/CSharp/Portable/Lowering/LocalRewriter/LocalRewriter_Conversion.cs index 76f9eb9b9a9b8..2bc0ddaff38da 100644 --- a/src/Compilers/CSharp/Portable/Lowering/LocalRewriter/LocalRewriter_Conversion.cs +++ b/src/Compilers/CSharp/Portable/Lowering/LocalRewriter/LocalRewriter_Conversion.cs @@ -81,29 +81,6 @@ private BoundNode RewriteUtf8StringLiteralConversion(BoundConversion node) string? value = node.Operand.ConstantValue?.StringValue; Debug.Assert(value != null); // PROTOTYPE(UTF8StringLiterals) : Adjust if we actually want it to work with 'null' value. - var utf8 = new System.Text.UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true); - byte[] bytes; - - try - { - bytes = utf8.GetBytes(value); - } - catch (Exception ex) - { - _diagnostics.Add( - ErrorCode.ERR_CannotBeConvertedToUTF8, - node.Operand.Syntax.Location, - ex.Message); - - return BadExpression(node.Syntax, node.Type, ImmutableArray.Empty); - } - - var builder = ArrayBuilder.GetInstance(bytes.Length); - foreach (byte b in bytes) - { - builder.Add(_factory.Literal(b)); - } - ArrayTypeSymbol byteArray; if (node.Type is ArrayTypeSymbol array) @@ -124,11 +101,7 @@ private BoundNode RewriteUtf8StringLiteralConversion(BoundConversion node) byteArray = ArrayTypeSymbol.CreateSZArray(_compilation.Assembly, TypeWithAnnotations.Create(byteType)); } - var utf8Bytes = new BoundArrayCreation( - node.Syntax, - ImmutableArray.Create(_factory.Literal(builder.Count)), - new BoundArrayInitialization(node.Syntax, builder.ToImmutableAndFree()), - byteArray); + BoundExpression utf8Bytes = CreateUTF8ByteRepresentation(node.Syntax, node.Operand.Syntax, value, byteArray); if ((object)node.Type == byteArray) { @@ -155,6 +128,47 @@ private BoundNode RewriteUtf8StringLiteralConversion(BoundConversion node) return new BoundObjectCreationExpression(node.Syntax, ctor.AsMember((NamedTypeSymbol)node.Type), utf8Bytes); } + private BoundExpression CreateUTF8ByteRepresentation(SyntaxNode resultSyntax, SyntaxNode valueSyntax, string value, ArrayTypeSymbol byteArray) + { + Debug.Assert(byteArray.IsSZArray); + Debug.Assert(byteArray.ElementType.SpecialType == SpecialType.System_Byte); + + var utf8 = new System.Text.UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true); + byte[] bytes; + + try + { + bytes = utf8.GetBytes(value); + } + catch (Exception ex) + { + _diagnostics.Add( + ErrorCode.ERR_CannotBeConvertedToUTF8, + valueSyntax.Location, + ex.Message); + + return BadExpression(resultSyntax, byteArray, ImmutableArray.Empty); + } + + var builder = ArrayBuilder.GetInstance(bytes.Length); + foreach (byte b in bytes) + { + builder.Add(_factory.Literal(b)); + } + + var utf8Bytes = new BoundArrayCreation( + resultSyntax, + ImmutableArray.Create(_factory.Literal(builder.Count)), + new BoundArrayInitialization(resultSyntax, builder.ToImmutableAndFree()), + byteArray); + return utf8Bytes; + } + + public override BoundNode VisitUTF8String(BoundUTF8String node) + { + return CreateUTF8ByteRepresentation(node.Syntax, node.Syntax, node.Value, (ArrayTypeSymbol)node.Type); + } + private static bool IsFloatingPointExpressionOfUnknownPrecision(BoundExpression rewrittenNode) { if (rewrittenNode == null) diff --git a/src/Compilers/CSharp/Portable/Operations/CSharpOperationFactory.cs b/src/Compilers/CSharp/Portable/Operations/CSharpOperationFactory.cs index 6d4c4b8241256..d7c0ea902a818 100644 --- a/src/Compilers/CSharp/Portable/Operations/CSharpOperationFactory.cs +++ b/src/Compilers/CSharp/Portable/Operations/CSharpOperationFactory.cs @@ -306,6 +306,7 @@ public CSharpOperationFactory(SemanticModel semanticModel) case BoundKind.StackAllocArrayCreation: case BoundKind.TypeExpression: case BoundKind.TypeOrValueExpression: + case BoundKind.UTF8String: // PROTOTYPE(UTF8StringLiterals) : add special node? ConstantValue? constantValue = (boundNode as BoundExpression)?.ConstantValue; bool isImplicit = boundNode.WasCompilerGenerated; diff --git a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs index bfe6488af215c..ef5560dfb1528 100644 --- a/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs +++ b/src/Compilers/CSharp/Portable/Parser/LanguageParser.cs @@ -10161,6 +10161,7 @@ private bool IsPossibleExpression(bool allowBinaryExpressions, bool allowAssignm case SyntaxKind.OpenParenToken: case SyntaxKind.NumericLiteralToken: case SyntaxKind.StringLiteralToken: + case SyntaxKind.UTF8StringLiteralToken: case SyntaxKind.InterpolatedStringStartToken: case SyntaxKind.InterpolatedStringToken: case SyntaxKind.CharacterLiteralToken: @@ -10382,6 +10383,7 @@ private static Precedence GetPrecedence(SyntaxKind op) case SyntaxKind.SimpleMemberAccessExpression: case SyntaxKind.StackAllocArrayCreationExpression: case SyntaxKind.StringLiteralExpression: + case SyntaxKind.UTF8StringLiteralExpression: case SyntaxKind.SuppressNullableWarningExpression: case SyntaxKind.ThisExpression: case SyntaxKind.TrueLiteralExpression: @@ -10445,6 +10447,7 @@ private bool IsAwaitExpression() case SyntaxKind.TrueKeyword: case SyntaxKind.FalseKeyword: case SyntaxKind.StringLiteralToken: + case SyntaxKind.UTF8StringLiteralToken: // PROTOTYPE(UTF8StringLiterals) : add test coverage for this code path. case SyntaxKind.InterpolatedStringStartToken: case SyntaxKind.InterpolatedStringToken: case SyntaxKind.NumericLiteralToken: @@ -10880,6 +10883,7 @@ private ExpressionSyntax ParseTermWithoutPostfix(Precedence precedence) case SyntaxKind.NullKeyword: case SyntaxKind.NumericLiteralToken: case SyntaxKind.StringLiteralToken: + case SyntaxKind.UTF8StringLiteralToken: case SyntaxKind.CharacterLiteralToken: return _syntaxFactory.LiteralExpression(SyntaxFacts.GetLiteralExpression(tk), this.EatToken()); case SyntaxKind.InterpolatedStringStartToken: diff --git a/src/Compilers/CSharp/Portable/Parser/Lexer.cs b/src/Compilers/CSharp/Portable/Parser/Lexer.cs index 74ef7e7134761..3c12118e0638f 100644 --- a/src/Compilers/CSharp/Portable/Parser/Lexer.cs +++ b/src/Compilers/CSharp/Portable/Parser/Lexer.cs @@ -386,6 +386,7 @@ private SyntaxToken Create(ref TokenInfo info, SyntaxListBuilder leading, Syntax token = SyntaxFactory.Literal(leadingNode, info.Text, info.Kind, info.Text, trailingNode); break; case SyntaxKind.StringLiteralToken: + case SyntaxKind.UTF8StringLiteralToken: token = SyntaxFactory.Literal(leadingNode, info.Text, info.Kind, info.StringValue, trailingNode); break; case SyntaxKind.CharacterLiteralToken: diff --git a/src/Compilers/CSharp/Portable/Parser/Lexer_StringLiteral.cs b/src/Compilers/CSharp/Portable/Parser/Lexer_StringLiteral.cs index a8e476ce3ab12..000e5ab1cd380 100644 --- a/src/Compilers/CSharp/Portable/Parser/Lexer_StringLiteral.cs +++ b/src/Compilers/CSharp/Portable/Parser/Lexer_StringLiteral.cs @@ -55,9 +55,9 @@ private void ScanStringLiteral(ref TokenInfo info, bool inDirective) } } - info.Text = TextWindow.GetText(intern: true); if (quoteCharacter == '\'') { + info.Text = TextWindow.GetText(intern: true); info.Kind = SyntaxKind.CharacterLiteralToken; if (_builder.Length != 1) { @@ -78,6 +78,15 @@ private void ScanStringLiteral(ref TokenInfo info, bool inDirective) else { info.Kind = SyntaxKind.StringLiteralToken; + + if (!inDirective && TextWindow.PeekChar() == 'u' && TextWindow.PeekChar(1) == '8') + { + info.Kind = SyntaxKind.UTF8StringLiteralToken; + TextWindow.AdvanceChar(2); + } + + info.Text = TextWindow.GetText(intern: true); + if (_builder.Length > 0) { info.StringValue = TextWindow.Intern(_builder); @@ -183,6 +192,13 @@ private void ScanVerbatimStringLiteral(ref TokenInfo info) } info.Kind = SyntaxKind.StringLiteralToken; + + if (TextWindow.PeekChar() == 'u' && TextWindow.PeekChar(1) == '8') + { + info.Kind = SyntaxKind.UTF8StringLiteralToken; + TextWindow.AdvanceChar(2); + } + info.Text = TextWindow.GetText(intern: false); info.StringValue = _builder.ToString(); } diff --git a/src/Compilers/CSharp/Portable/PublicAPI.Unshipped.txt b/src/Compilers/CSharp/Portable/PublicAPI.Unshipped.txt index 8cea827bd6408..e099506ffa048 100644 --- a/src/Compilers/CSharp/Portable/PublicAPI.Unshipped.txt +++ b/src/Compilers/CSharp/Portable/PublicAPI.Unshipped.txt @@ -49,6 +49,8 @@ Microsoft.CodeAnalysis.CSharp.Syntax.SlicePatternSyntax.WithDotDotToken(Microsof Microsoft.CodeAnalysis.CSharp.Syntax.SlicePatternSyntax.WithPattern(Microsoft.CodeAnalysis.CSharp.Syntax.PatternSyntax? pattern) -> Microsoft.CodeAnalysis.CSharp.Syntax.SlicePatternSyntax! Microsoft.CodeAnalysis.CSharp.SyntaxKind.ListPattern = 9035 -> Microsoft.CodeAnalysis.CSharp.SyntaxKind Microsoft.CodeAnalysis.CSharp.SyntaxKind.SlicePattern = 9034 -> Microsoft.CodeAnalysis.CSharp.SyntaxKind +Microsoft.CodeAnalysis.CSharp.SyntaxKind.UTF8StringLiteralExpression = 8756 -> Microsoft.CodeAnalysis.CSharp.SyntaxKind +Microsoft.CodeAnalysis.CSharp.SyntaxKind.UTF8StringLiteralToken = 8518 -> Microsoft.CodeAnalysis.CSharp.SyntaxKind override Microsoft.CodeAnalysis.CSharp.CSharpSyntaxRewriter.VisitListPattern(Microsoft.CodeAnalysis.CSharp.Syntax.ListPatternSyntax! node) -> Microsoft.CodeAnalysis.SyntaxNode? override Microsoft.CodeAnalysis.CSharp.CSharpSyntaxRewriter.VisitSlicePattern(Microsoft.CodeAnalysis.CSharp.Syntax.SlicePatternSyntax! node) -> Microsoft.CodeAnalysis.SyntaxNode? override Microsoft.CodeAnalysis.CSharp.Syntax.LineDirectiveTriviaSyntax.File.get -> Microsoft.CodeAnalysis.SyntaxToken diff --git a/src/Compilers/CSharp/Portable/Syntax/Syntax.xml b/src/Compilers/CSharp/Portable/Syntax/Syntax.xml index 4aebdf0ee086d..7355614e7d48e 100644 --- a/src/Compilers/CSharp/Portable/Syntax/Syntax.xml +++ b/src/Compilers/CSharp/Portable/Syntax/Syntax.xml @@ -865,6 +865,7 @@ + @@ -874,6 +875,7 @@ + diff --git a/src/Compilers/CSharp/Portable/Syntax/SyntaxEquivalence.cs b/src/Compilers/CSharp/Portable/Syntax/SyntaxEquivalence.cs index 0899313520fb4..032f700094cce 100644 --- a/src/Compilers/CSharp/Portable/Syntax/SyntaxEquivalence.cs +++ b/src/Compilers/CSharp/Portable/Syntax/SyntaxEquivalence.cs @@ -82,6 +82,7 @@ private static bool AreTokensEquivalent(GreenNode? before, GreenNode? after, Fun case SyntaxKind.NumericLiteralToken: case SyntaxKind.CharacterLiteralToken: case SyntaxKind.StringLiteralToken: + case SyntaxKind.UTF8StringLiteralToken: case SyntaxKind.InterpolatedStringTextToken: if (((Green.SyntaxToken)before).Text != ((Green.SyntaxToken)after).Text) { diff --git a/src/Compilers/CSharp/Portable/Syntax/SyntaxKind.cs b/src/Compilers/CSharp/Portable/Syntax/SyntaxKind.cs index ecff495816ec2..20f3f58e822ac 100644 --- a/src/Compilers/CSharp/Portable/Syntax/SyntaxKind.cs +++ b/src/Compilers/CSharp/Portable/Syntax/SyntaxKind.cs @@ -489,6 +489,8 @@ public enum SyntaxKind : ushort // This only exists in transient form during parsing. InterpolatedStringTextToken = 8517, // literal text that is part of an interpolated string + UTF8StringLiteralToken = 8518, + // trivia EndOfLineTrivia = 8539, WhitespaceTrivia = 8540, @@ -655,6 +657,7 @@ public enum SyntaxKind : ushort FalseLiteralExpression = 8753, NullLiteralExpression = 8754, DefaultLiteralExpression = 8755, + UTF8StringLiteralExpression = 8756, // primary function expressions TypeOfExpression = 8760, diff --git a/src/Compilers/CSharp/Portable/Syntax/SyntaxKindFacts.cs b/src/Compilers/CSharp/Portable/Syntax/SyntaxKindFacts.cs index 2b39e146d4f58..85866f18eaa9d 100644 --- a/src/Compilers/CSharp/Portable/Syntax/SyntaxKindFacts.cs +++ b/src/Compilers/CSharp/Portable/Syntax/SyntaxKindFacts.cs @@ -181,6 +181,7 @@ internal static bool IsLiteral(SyntaxKind kind) { case SyntaxKind.IdentifierToken: case SyntaxKind.StringLiteralToken: + case SyntaxKind.UTF8StringLiteralToken: case SyntaxKind.CharacterLiteralToken: case SyntaxKind.NumericLiteralToken: case SyntaxKind.XmlTextLiteralToken: @@ -534,6 +535,8 @@ public static SyntaxKind GetLiteralExpression(SyntaxKind token) { case SyntaxKind.StringLiteralToken: return SyntaxKind.StringLiteralExpression; + case SyntaxKind.UTF8StringLiteralToken: + return SyntaxKind.UTF8StringLiteralExpression; case SyntaxKind.CharacterLiteralToken: return SyntaxKind.CharacterLiteralExpression; case SyntaxKind.NumericLiteralToken: diff --git a/src/Compilers/CSharp/Test/Semantic/Semantics/Utf8StringsLiteralsTests.cs b/src/Compilers/CSharp/Test/Semantic/Semantics/Utf8StringsLiteralsTests.cs index db78dd7b45bd0..a91a0f522e1ee 100644 --- a/src/Compilers/CSharp/Test/Semantic/Semantics/Utf8StringsLiteralsTests.cs +++ b/src/Compilers/CSharp/Test/Semantic/Semantics/Utf8StringsLiteralsTests.cs @@ -949,6 +949,75 @@ .maxstack 2 "); } + [ConditionalFact(typeof(CoreClrOnly))] + public void ConstantExpressions_02() + { + var source = @" +using System; +class C +{ + const string second = ""\uDE00""; // low surrogate + + static void Main() + { + System.Console.WriteLine(); + Helpers.Print(Test1()); + Helpers.Print(Test2()); + Helpers.Print(Test3()); + } + + static byte[] Test1() => $""\uD83D{second}""; + static Span Test2() => $""\uD83D{second}""; + static ReadOnlySpan Test3() => $""\uD83D{second}""; +} +"; + var comp = CreateCompilation(source + HelpersSource, targetFramework: TargetFramework.NetCoreApp, options: TestOptions.DebugExe); + + var verifier = CompileAndVerify(comp, expectedOutput: @" +{ 0xF0 0x9F 0x98 0x80 } +{ 0xF0 0x9F 0x98 0x80 } +{ 0xF0 0x9F 0x98 0x80 } +").VerifyDiagnostics(); + + verifier.VerifyIL("C.Test1()", @" +{ + // Code size 18 (0x12) + .maxstack 3 + IL_0000: ldc.i4.4 + IL_0001: newarr ""byte"" + IL_0006: dup + IL_0007: ldtoken ""int .F0443A342C5EF54783A111B51BA56C938E474C32324D90C3A60C9C8E3A37E2D9"" + IL_000c: call ""void System.Runtime.CompilerServices.RuntimeHelpers.InitializeArray(System.Array, System.RuntimeFieldHandle)"" + IL_0011: ret +} +"); + + verifier.VerifyIL("C.Test2()", @" +{ + // Code size 23 (0x17) + .maxstack 3 + IL_0000: ldc.i4.4 + IL_0001: newarr ""byte"" + IL_0006: dup + IL_0007: ldtoken ""int .F0443A342C5EF54783A111B51BA56C938E474C32324D90C3A60C9C8E3A37E2D9"" + IL_000c: call ""void System.Runtime.CompilerServices.RuntimeHelpers.InitializeArray(System.Array, System.RuntimeFieldHandle)"" + IL_0011: newobj ""System.Span..ctor(byte[])"" + IL_0016: ret +} +"); + + verifier.VerifyIL("C.Test3()", @" +{ + // Code size 12 (0xc) + .maxstack 2 + IL_0000: ldsflda ""int .F0443A342C5EF54783A111B51BA56C938E474C32324D90C3A60C9C8E3A37E2D9"" + IL_0005: ldc.i4.4 + IL_0006: newobj ""System.ReadOnlySpan..ctor(void*, int)"" + IL_000b: ret +} +"); + } + [ConditionalFact(typeof(CoreClrOnly))] public void UserDefinedImplicitConversions_01() { @@ -1254,7 +1323,7 @@ public static implicit operator C3(ReadOnlySpan x) } [ConditionalFact(typeof(CoreClrOnly))] - public void UserDefinedExplictConversions_01() + public void UserDefinedExplicitConversions_01() { var source = @" using System; @@ -1398,7 +1467,7 @@ public static explicit operator C3(ReadOnlySpan x) } [ConditionalFact(typeof(CoreClrOnly))] - public void UserDefinedExplictConversions_02() + public void UserDefinedExplicitConversions_02() { var source = @" using System; @@ -1649,7 +1718,8 @@ class C { static void Main() { - System.Console.WriteLine(Test(""s"")); + System.Console.Write(Test(""s"")); + System.Console.Write(Test(""s""u8)); } static string Test(ReadOnlySpan a) => ""ReadOnlySpan""; @@ -1658,7 +1728,7 @@ static void Main() "; var comp = CreateCompilation(source, targetFramework: TargetFramework.NetCoreApp, options: TestOptions.DebugExe); - CompileAndVerify(comp, expectedOutput: @"ReadOnlySpan").VerifyDiagnostics(); + CompileAndVerify(comp, expectedOutput: @"ReadOnlySpanarray").VerifyDiagnostics(); } [ConditionalFact(typeof(CoreClrOnly))] @@ -1670,7 +1740,8 @@ class C { static void Main() { - System.Console.WriteLine(Test(""s"")); + System.Console.Write(Test(""s"")); + System.Console.Write(Test(""s""u8)); } static string Test(byte[] a) => ""array""; @@ -1679,7 +1750,7 @@ static void Main() "; var comp = CreateCompilation(source, targetFramework: TargetFramework.NetCoreApp, options: TestOptions.DebugExe); - CompileAndVerify(comp, expectedOutput: @"ReadOnlySpan").VerifyDiagnostics(); + CompileAndVerify(comp, expectedOutput: @"ReadOnlySpanarray").VerifyDiagnostics(); } [ConditionalFact(typeof(CoreClrOnly))] @@ -1739,6 +1810,150 @@ static void Main() CompileAndVerify(comp, expectedOutput: @"ReadOnlySpan").VerifyDiagnostics(); } + [ConditionalFact(typeof(CoreClrOnly))] + public void OverloadResolution_05() + { + var source = @" +using System; + +class C +{ + static void Main() + { + Test(""s""); + } + + static void Test(C1 a) {} +} + +class C1 +{ + public static implicit operator C1(string x) + { + System.Console.WriteLine(""string""); + return new C1(); + } + + public static implicit operator C1(ReadOnlySpan x) + { + System.Console.WriteLine(""ReadOnlySpan""); + return new C1(); + } +} +"; + var comp = CreateCompilation(source, targetFramework: TargetFramework.NetCoreApp, options: TestOptions.DebugExe); + + CompileAndVerify(comp, expectedOutput: @"string").VerifyDiagnostics(); + } + + [ConditionalFact(typeof(CoreClrOnly))] + public void OverloadResolution_06() + { + var source = @" +using System; +class C +{ + static void Main() + { + System.Console.WriteLine(Test(""s"")); + } + + static string Test(ReadOnlySpan a) => ""ReadOnlySpan""; + static string Test(byte[] a) => ""array""; +} +"; + var comp = CreateCompilation(source, targetFramework: TargetFramework.NetCoreApp, options: TestOptions.DebugExe); + CompileAndVerify(comp, expectedOutput: @"array").VerifyDiagnostics(); + } + + [ConditionalFact(typeof(CoreClrOnly))] + public void OverloadResolution_07() + { + var source = @" +using System; +class C +{ + static void Main() + { + System.Console.WriteLine(Test(""s"")); + } + + static string Test(Span a) => ""Span""; + static string Test(byte[] a) => ""array""; +} +"; + var comp = CreateCompilation(source, targetFramework: TargetFramework.NetCoreApp, options: TestOptions.DebugExe); + CompileAndVerify(comp, expectedOutput: @"array").VerifyDiagnostics(); + } + + [ConditionalFact(typeof(CoreClrOnly))] + public void OverloadResolution_08() + { + var source = @" +using System; +class C +{ + static void Main() + { + System.Console.WriteLine(Test(""s"")); + } + + static string Test(ReadOnlySpan a) => ""ReadOnlySpan""; + static string Test(Span a) => ""Span""; +} +"; + var comp = CreateCompilation(source, targetFramework: TargetFramework.NetCoreApp, options: TestOptions.DebugExe); + CompileAndVerify(comp, expectedOutput: @"Span").VerifyDiagnostics(); + } + + [ConditionalFact(typeof(CoreClrOnly))] + public void OverloadResolution_09() + { + var source = @" +using System; +class C +{ + static void Main() + { + System.Console.WriteLine(Test(""s"")); + } + + static string Test(ReadOnlySpan a) => ""ReadOnlySpan""; + static string Test(byte[] a) => ""array""; + static string Test(Span a) => ""Span""; +} +"; + var comp = CreateCompilation(source, targetFramework: TargetFramework.NetCoreApp, options: TestOptions.DebugExe); + CompileAndVerify(comp, expectedOutput: @"array").VerifyDiagnostics(); + } + + [ConditionalFact(typeof(CoreClrOnly))] + public void OverloadResolution_10() + { + var source = @" +using System; + +class Program +{ + static void Main() + { + var p = new Program(); + Console.WriteLine(p.M("""")); + } + + public string M(byte[] b) => ""byte[]""; +} + +static class E +{ + public static string M(this object o, string s) => ""string""; +} +"; + var comp = CreateCompilation(source, targetFramework: TargetFramework.NetCoreApp, options: TestOptions.DebugExe); + CompileAndVerify(comp, expectedOutput: @"byte[]"). // The behavior has changed + VerifyDiagnostics(); + } + [Fact] public void NullableAnalysis_01() { @@ -1786,5 +2001,639 @@ static void Main() Diagnostic(ErrorCode.WRN_NullReferenceReceiver, "x").WithLocation(9, 13) ); } + + [Fact] + public void NullableAnalysis_03() + { + var source = @" +#nullable enable + +class C +{ + static void Main() + { + _ = ""hello""u8.Length; + } +} +"; + var comp = CreateCompilation(source, targetFramework: TargetFramework.NetCoreApp, options: TestOptions.DebugExe); + comp.VerifyDiagnostics(); + } + + [ConditionalFact(typeof(CoreClrOnly))] + public void UTF8StringLiteral_01() + { + var source = @" +using System; +class C +{ + static void Main() + { + System.Console.WriteLine(); + Helpers.Print(Test1()); + Helpers.Print(Test2()); + Helpers.Print(Test3()); + } + + static byte[] Test1() => ""hello""u8; + static Span Test2() => ""dog""u8; + static ReadOnlySpan Test3() => ""cat""u8; +} +"; + var comp = CreateCompilation(source + HelpersSource, targetFramework: TargetFramework.NetCoreApp, options: TestOptions.DebugExe); + + var verifier = CompileAndVerify(comp, expectedOutput: @" +{ 0x68 0x65 0x6C 0x6C 0x6F } +{ 0x64 0x6F 0x67 } +{ 0x63 0x61 0x74 } +").VerifyDiagnostics(); + + verifier.VerifyIL("C.Test1()", @" +{ + // Code size 18 (0x12) + .maxstack 3 + IL_0000: ldc.i4.5 + IL_0001: newarr ""byte"" + IL_0006: dup + IL_0007: ldtoken "".__StaticArrayInitTypeSize=5 .2CF24DBA5FB0A30E26E83B2AC5B9E29E1B161E5C1FA7425E73043362938B9824"" + IL_000c: call ""void System.Runtime.CompilerServices.RuntimeHelpers.InitializeArray(System.Array, System.RuntimeFieldHandle)"" + IL_0011: ret +} +"); + + verifier.VerifyIL("C.Test2()", @" +{ + // Code size 23 (0x17) + .maxstack 3 + IL_0000: ldc.i4.3 + IL_0001: newarr ""byte"" + IL_0006: dup + IL_0007: ldtoken "".__StaticArrayInitTypeSize=3 .CD6357EFDD966DE8C0CB2F876CC89EC74CE35F0968E11743987084BD42FB8944"" + IL_000c: call ""void System.Runtime.CompilerServices.RuntimeHelpers.InitializeArray(System.Array, System.RuntimeFieldHandle)"" + IL_0011: call ""System.Span System.Span.op_Implicit(byte[])"" + IL_0016: ret +} +"); + + verifier.VerifyIL("C.Test3()", @" +{ + // Code size 12 (0xc) + .maxstack 2 + IL_0000: ldsflda "".__StaticArrayInitTypeSize=3 .77AF778B51ABD4A3C51C5DDD97204A9C3AE614EBCCB75A606C3B6865AED6744E"" + IL_0005: ldc.i4.3 + IL_0006: newobj ""System.ReadOnlySpan..ctor(void*, int)"" + IL_000b: ret +} +"); + + comp = CreateCompilation(source + HelpersSource, targetFramework: TargetFramework.NetCoreApp, options: TestOptions.DebugExe, parseOptions: TestOptions.RegularNext); + + CompileAndVerify(comp, expectedOutput: @" +{ 0x68 0x65 0x6C 0x6C 0x6F } +{ 0x64 0x6F 0x67 } +{ 0x63 0x61 0x74 } +").VerifyDiagnostics(); + + comp = CreateCompilation(source + HelpersSource, targetFramework: TargetFramework.NetCoreApp, options: TestOptions.DebugExe, parseOptions: TestOptions.Regular10); + comp.VerifyDiagnostics( + // (13,30): error CS8652: The feature 'Utf8 String Literals' is currently in Preview and *unsupported*. To use Preview features, use the 'preview' language version. + // static byte[] Test1() => "hello"u8; + Diagnostic(ErrorCode.ERR_FeatureInPreview, @"""hello""u8").WithArguments("Utf8 String Literals").WithLocation(13, 30), + // (14,34): error CS8652: The feature 'Utf8 String Literals' is currently in Preview and *unsupported*. To use Preview features, use the 'preview' language version. + // static Span Test2() => "dog"u8; + Diagnostic(ErrorCode.ERR_FeatureInPreview, @"""dog""u8").WithArguments("Utf8 String Literals").WithLocation(14, 34), + // (15,42): error CS8652: The feature 'Utf8 String Literals' is currently in Preview and *unsupported*. To use Preview features, use the 'preview' language version. + // static ReadOnlySpan Test3() => "cat"u8; + Diagnostic(ErrorCode.ERR_FeatureInPreview, @"""cat""u8").WithArguments("Utf8 String Literals").WithLocation(15, 42) + ); + } + + + [Fact] + public void MissingType_01() + { + var source = @" +using System; +class C +{ + static void Main() + { + _ = ""hello""u8; + } +} +"; + var comp = CreateCompilation(source, targetFramework: TargetFramework.NetCoreApp, options: TestOptions.DebugExe); + comp.MakeTypeMissing(SpecialType.System_Byte); + comp.VerifyEmitDiagnostics( + // (7,13): error CS0518: Predefined type 'System.Byte' is not defined or imported + // _ = "hello"u8; + Diagnostic(ErrorCode.ERR_PredefinedTypeNotFound, @"""hello""u8").WithArguments("System.Byte").WithLocation(7, 13) + ); + } + + [ConditionalFact(typeof(CoreClrOnly))] + public void MissingHelpers_07() + { + var source = @" +using System; +class C +{ + static void Main() + { + System.Console.WriteLine(); + Helpers.Print(Test1()); + Helpers.Print(Test2()); + Helpers.Print(Test3()); + } + + static byte[] Test1() => ""hello""u8; + static Span Test2() => ""dog""u8; + static ReadOnlySpan Test3() => ""cat""u8; +} +"; + var comp = CreateCompilation(source + HelpersSource, targetFramework: TargetFramework.NetCoreApp, options: TestOptions.DebugExe); + + comp.MakeMemberMissing(WellKnownMember.System_Span_T__ctor_Array); + comp.MakeMemberMissing(WellKnownMember.System_Span_T__ctor_Pointer); + comp.MakeMemberMissing(WellKnownMember.System_ReadOnlySpan_T__ctor_Array); + + var verifier = CompileAndVerify(comp, expectedOutput: @" +{ 0x68 0x65 0x6C 0x6C 0x6F } +{ 0x64 0x6F 0x67 } +{ 0x63 0x61 0x74 } +").VerifyDiagnostics(); + + verifier.VerifyIL("C.Test1()", @" +{ + // Code size 18 (0x12) + .maxstack 3 + IL_0000: ldc.i4.5 + IL_0001: newarr ""byte"" + IL_0006: dup + IL_0007: ldtoken "".__StaticArrayInitTypeSize=5 .2CF24DBA5FB0A30E26E83B2AC5B9E29E1B161E5C1FA7425E73043362938B9824"" + IL_000c: call ""void System.Runtime.CompilerServices.RuntimeHelpers.InitializeArray(System.Array, System.RuntimeFieldHandle)"" + IL_0011: ret +} +"); + + verifier.VerifyIL("C.Test2()", @" +{ + // Code size 23 (0x17) + .maxstack 3 + IL_0000: ldc.i4.3 + IL_0001: newarr ""byte"" + IL_0006: dup + IL_0007: ldtoken "".__StaticArrayInitTypeSize=3 .CD6357EFDD966DE8C0CB2F876CC89EC74CE35F0968E11743987084BD42FB8944"" + IL_000c: call ""void System.Runtime.CompilerServices.RuntimeHelpers.InitializeArray(System.Array, System.RuntimeFieldHandle)"" + IL_0011: call ""System.Span System.Span.op_Implicit(byte[])"" + IL_0016: ret +} +"); + + verifier.VerifyIL("C.Test3()", @" +{ + // Code size 12 (0xc) + .maxstack 2 + IL_0000: ldsflda "".__StaticArrayInitTypeSize=3 .77AF778B51ABD4A3C51C5DDD97204A9C3AE614EBCCB75A606C3B6865AED6744E"" + IL_0005: ldc.i4.3 + IL_0006: newobj ""System.ReadOnlySpan..ctor(void*, int)"" + IL_000b: ret +} +"); + } + + [ConditionalFact(typeof(CoreClrOnly))] + public void MissingHelpers_08() + { + var source = @" +using System; +class C +{ + static void Main() + { + Helpers.Print(Test2()); + } + + static Span Test2() => ""dog""u8; +} +"; + var comp = CreateCompilation(source + HelpersSource, targetFramework: TargetFramework.NetCoreApp, options: TestOptions.DebugExe); + comp.MakeMemberMissing(WellKnownMember.System_Span_T__ctor_Pointer); + var verifier = CompileAndVerify(comp, expectedOutput: "{ 0x64 0x6F 0x67 }").VerifyDiagnostics(); + + verifier.VerifyIL("C.Test2()", @" +{ + // Code size 23 (0x17) + .maxstack 3 + IL_0000: ldc.i4.3 + IL_0001: newarr ""byte"" + IL_0006: dup + IL_0007: ldtoken "".__StaticArrayInitTypeSize=3 .CD6357EFDD966DE8C0CB2F876CC89EC74CE35F0968E11743987084BD42FB8944"" + IL_000c: call ""void System.Runtime.CompilerServices.RuntimeHelpers.InitializeArray(System.Array, System.RuntimeFieldHandle)"" + IL_0011: call ""System.Span System.Span.op_Implicit(byte[])"" + IL_0016: ret +} +"); + } + + [ConditionalFact(typeof(CoreClrOnly))] + public void MissingHelpers_09() + { + var source = @" +using System; +class C +{ + static void Main() + { + Helpers.Print(Test3()); + } + + static ReadOnlySpan Test3() => ""cat""u8; +} +"; + var comp = CreateCompilation(source + HelpersSource, targetFramework: TargetFramework.NetCoreApp, options: TestOptions.DebugExe); + comp.MakeMemberMissing(WellKnownMember.System_ReadOnlySpan_T__ctor_Pointer); + var verifier = CompileAndVerify(comp, expectedOutput: "{ 0x63 0x61 0x74 }").VerifyDiagnostics(); + + verifier.VerifyIL("C.Test3()", @" +{ + // Code size 23 (0x17) + .maxstack 3 + IL_0000: ldc.i4.3 + IL_0001: newarr ""byte"" + IL_0006: dup + IL_0007: ldtoken "".__StaticArrayInitTypeSize=3 .77AF778B51ABD4A3C51C5DDD97204A9C3AE614EBCCB75A606C3B6865AED6744E"" + IL_000c: call ""void System.Runtime.CompilerServices.RuntimeHelpers.InitializeArray(System.Array, System.RuntimeFieldHandle)"" + IL_0011: call ""System.ReadOnlySpan System.ReadOnlySpan.op_Implicit(byte[])"" + IL_0016: ret +} +"); + } + + [ConditionalFact(typeof(CoreClrOnly))] + public void OverloadResolution_11() + { + var source = @" +using System; +class C +{ + static void Main() + { + System.Console.WriteLine(Test(""s""u8)); + } + + static string Test(ReadOnlySpan a) => ""ReadOnlySpan""; + static string Test(byte[] a) => ""array""; +} +"; + var comp = CreateCompilation(source, targetFramework: TargetFramework.NetCoreApp, options: TestOptions.DebugExe); + CompileAndVerify(comp, expectedOutput: @"array").VerifyDiagnostics(); + } + + [ConditionalFact(typeof(CoreClrOnly))] + public void OverloadResolution_12() + { + var source = @" +using System; +class C +{ + static void Main() + { + System.Console.WriteLine(Test(""s""u8)); + } + + static string Test(Span a) => ""Span""; + static string Test(byte[] a) => ""array""; +} +"; + var comp = CreateCompilation(source, targetFramework: TargetFramework.NetCoreApp, options: TestOptions.DebugExe); + CompileAndVerify(comp, expectedOutput: @"array").VerifyDiagnostics(); + } + + [ConditionalFact(typeof(CoreClrOnly))] + public void OverloadResolution_13() + { + var source = @" +using System; +class C +{ + static void Main() + { + System.Console.WriteLine(Test(""s""u8)); + } + + static string Test(ReadOnlySpan a) => ""ReadOnlySpan""; + static string Test(Span a) => ""Span""; +} +"; + var comp = CreateCompilation(source, targetFramework: TargetFramework.NetCoreApp, options: TestOptions.DebugExe); + CompileAndVerify(comp, expectedOutput: @"Span").VerifyDiagnostics(); + } + + [ConditionalFact(typeof(CoreClrOnly))] + public void OverloadResolution_14() + { + var source = @" +using System; +class C +{ + static void Main() + { + System.Console.WriteLine(Test(""s""u8)); + } + + static string Test(ReadOnlySpan a) => ""ReadOnlySpan""; + static string Test(byte[] a) => ""array""; + static string Test(Span a) => ""Span""; +} +"; + var comp = CreateCompilation(source, targetFramework: TargetFramework.NetCoreApp, options: TestOptions.DebugExe); + CompileAndVerify(comp, expectedOutput: @"array").VerifyDiagnostics(); + } + + [ConditionalFact(typeof(CoreClrOnly))] + public void UserDefinedImplicitConversions_03() + { + var source = @" +using System; +class C +{ + static void Main() + { + C1 x = ""hello""u8; + } +} + +class C1 +{ + public static implicit operator C1(byte[] x) + { + Helpers.Print(x); + return new C1(); + } +} +"; + var comp = CreateCompilation(source + HelpersSource, targetFramework: TargetFramework.NetCoreApp, options: TestOptions.DebugExe); + + CompileAndVerify(comp, expectedOutput: @" +{ 0x68 0x65 0x6C 0x6C 0x6F } +").VerifyDiagnostics(); + } + + [ConditionalFact(typeof(CoreClrOnly))] + public void UserDefinedImplicitConversions_04() + { + var source = @" +using System; +class C +{ + static void Main() + { + var x = (C1)""hello""u8; + } +} + +class C1 +{ + public static implicit operator C1(byte[] x) + { + Helpers.Print(x); + return new C1(); + } +} +"; + var comp = CreateCompilation(source + HelpersSource, targetFramework: TargetFramework.NetCoreApp, options: TestOptions.DebugExe); + + CompileAndVerify(comp, expectedOutput: @" +{ 0x68 0x65 0x6C 0x6C 0x6F } +").VerifyDiagnostics(); + } + + [ConditionalFact(typeof(CoreClrOnly))] + public void UserDefinedImplicitConversions_05() + { + var source = @" +using System; +class C +{ + static void Main() + { + C2 y = ""dog""u8; + C3 z = ""cat""u8; + } +} + +class C2 +{ + public static implicit operator C2(Span x) + { + return new C2(); + } +} + +class C3 +{ + public static implicit operator C3(ReadOnlySpan x) + { + return new C3(); + } +} +"; + var comp = CreateCompilation(source, targetFramework: TargetFramework.NetCoreApp, options: TestOptions.DebugExe); + // PROTOTYPE(UTF8StringLiterals) : Confirm this is the behavior we want. We are relying on user-defined conversions to convert from byte[] to span types. + comp.VerifyDiagnostics( + // (7,16): error CS0029: Cannot implicitly convert type 'byte[]' to 'C2' + // C2 y = "dog"u8; + Diagnostic(ErrorCode.ERR_NoImplicitConv, @"""dog""u8").WithArguments("byte[]", "C2").WithLocation(7, 16), + // (8,16): error CS0029: Cannot implicitly convert type 'byte[]' to 'C3' + // C3 z = "cat"u8; + Diagnostic(ErrorCode.ERR_NoImplicitConv, @"""cat""u8").WithArguments("byte[]", "C3").WithLocation(8, 16) + ); + } + + [ConditionalFact(typeof(CoreClrOnly))] + public void UserDefinedImplicitConversions_06() + { + var source = @" +using System; +class C +{ + static void Main() + { + var y = (C2)""dog""u8; + var z = (C3)""cat""u8; + } +} + +class C2 +{ + public static implicit operator C2(Span x) + { + return new C2(); + } +} + +class C3 +{ + public static implicit operator C3(ReadOnlySpan x) + { + return new C3(); + } +} +"; + var comp = CreateCompilation(source, targetFramework: TargetFramework.NetCoreApp, options: TestOptions.DebugExe); + // PROTOTYPE(UTF8StringLiterals) : Confirm this is the behavior we want. We are relying on user-defined conversions to convert from byte[] to span types. + comp.VerifyDiagnostics( + // (7,17): error CS0030: Cannot convert type 'byte[]' to 'C2' + // var y = (C2)"dog"u8; + Diagnostic(ErrorCode.ERR_NoExplicitConv, @"(C2)""dog""u8").WithArguments("byte[]", "C2").WithLocation(7, 17), + // (8,17): error CS0030: Cannot convert type 'byte[]' to 'C3' + // var z = (C3)"cat"u8; + Diagnostic(ErrorCode.ERR_NoExplicitConv, @"(C3)""cat""u8").WithArguments("byte[]", "C3").WithLocation(8, 17) + ); + } + + [ConditionalFact(typeof(CoreClrOnly))] + public void UserDefinedExplicitConversions_03() + { + var source = @" +using System; +class C +{ + static void Main() + { + C1 x = ""hello""u8; + C2 y = ""dog""u8; + C3 z = ""cat""u8; + } +} + +class C1 +{ + public static explicit operator C1(byte[] x) + { + return new C1(); + } +} + +class C2 +{ + public static explicit operator C2(Span x) + { + return new C2(); + } +} + +class C3 +{ + public static explicit operator C3(ReadOnlySpan x) + { + return new C3(); + } +}"; + var comp = CreateCompilation(source, targetFramework: TargetFramework.NetCoreApp, options: TestOptions.DebugExe); + + comp.VerifyDiagnostics( + // (7,16): error CS0266: Cannot implicitly convert type 'byte[]' to 'C1'. An explicit conversion exists (are you missing a cast?) + // C1 x = "hello"u8; + Diagnostic(ErrorCode.ERR_NoImplicitConvCast, @"""hello""u8").WithArguments("byte[]", "C1").WithLocation(7, 16), + // (8,16): error CS0029: Cannot implicitly convert type 'byte[]' to 'C2' + // C2 y = "dog"u8; + Diagnostic(ErrorCode.ERR_NoImplicitConv, @"""dog""u8").WithArguments("byte[]", "C2").WithLocation(8, 16), + // (9,16): error CS0029: Cannot implicitly convert type 'byte[]' to 'C3' + // C3 z = "cat"u8; + Diagnostic(ErrorCode.ERR_NoImplicitConv, @"""cat""u8").WithArguments("byte[]", "C3").WithLocation(9, 16) + ); + } + + [ConditionalFact(typeof(CoreClrOnly))] + public void UserDefinedExplicitConversions_04() + { + var source = @" +using System; +class C +{ + static void Main() + { + var x = (C1)""hello""u8; + } +} + +class C1 +{ + public static explicit operator C1(byte[] x) + { + Helpers.Print(x); + return new C1(); + } +} +"; + var comp = CreateCompilation(source + HelpersSource, targetFramework: TargetFramework.NetCoreApp, options: TestOptions.DebugExe); + + CompileAndVerify(comp, expectedOutput: @" +{ 0x68 0x65 0x6C 0x6C 0x6F } +").VerifyDiagnostics(); + } + + [ConditionalFact(typeof(CoreClrOnly))] + public void UserDefinedExplicitConversions_05() + { + var source = @" +using System; +class C +{ + static void Main() + { + var y = (C2)""dog""u8; + var z = (C3)""cat""u8; + } +} + +class C2 +{ + public static explicit operator C2(Span x) + { + return new C2(); + } +} + +class C3 +{ + public static explicit operator C3(ReadOnlySpan x) + { + return new C3(); + } +} +"; + var comp = CreateCompilation(source, targetFramework: TargetFramework.NetCoreApp, options: TestOptions.DebugExe); + // PROTOTYPE(UTF8StringLiterals) : Confirm this is the behavior we want. We are relying on user-defined conversions to convert from byte[] to span types. + comp.VerifyDiagnostics( + // (7,17): error CS0030: Cannot convert type 'byte[]' to 'C2' + // var y = (C2)"dog"u8; + Diagnostic(ErrorCode.ERR_NoExplicitConv, @"(C2)""dog""u8").WithArguments("byte[]", "C2").WithLocation(7, 17), + // (8,17): error CS0030: Cannot convert type 'byte[]' to 'C3' + // var z = (C3)"cat"u8; + Diagnostic(ErrorCode.ERR_NoExplicitConv, @"(C3)""cat""u8").WithArguments("byte[]", "C3").WithLocation(8, 17) + ); + } + + [Fact] + public void NaturalType_01() + { + var source = @" +class C +{ + static void Main() + { + System.Console.WriteLine((""hello""u8).GetType()); + } +} +"; + var comp = CreateCompilation(source, options: TestOptions.DebugExe); + + CompileAndVerify(comp, expectedOutput: @"System.Byte[]").VerifyDiagnostics(); + } + + // PROTOTYPE(UTF8StringLiterals) : Test default parameter values and attribute applications } } diff --git a/src/Compilers/CSharp/Test/Syntax/Parsing/LineSpanDirectiveParsingTests.cs b/src/Compilers/CSharp/Test/Syntax/Parsing/LineSpanDirectiveParsingTests.cs index 3c7df69f9561e..e82b854a002aa 100644 --- a/src/Compilers/CSharp/Test/Syntax/Parsing/LineSpanDirectiveParsingTests.cs +++ b/src/Compilers/CSharp/Test/Syntax/Parsing/LineSpanDirectiveParsingTests.cs @@ -1829,5 +1829,49 @@ public void VerifySpan_04() } EOF(); } + + [Fact] + public void NotUTF8StringLiteral_01() + { + string source = @"#line 1 ""file.cs""u8"; + + UsingLineDirective(source, options: null, + // (1,18): error CS1025: Single-line comment or end-of-line expected + // #line 1 "file.cs"u8 + Diagnostic(ErrorCode.ERR_EndOfPPLineExpected, "u8").WithLocation(1, 18) + ); + + N(SyntaxKind.LineDirectiveTrivia); + { + N(SyntaxKind.HashToken); + N(SyntaxKind.LineKeyword); + N(SyntaxKind.NumericLiteralToken, "1"); + N(SyntaxKind.StringLiteralToken, "\"file.cs\""); + N(SyntaxKind.EndOfDirectiveToken); + } + EOF(); + } + + [Fact] + public void NotUTF8StringLiteral_02() + { + string source = @"#line 1 @""file.cs""u8"; + + UsingLineDirective(source, options: null, + // (1,9): error CS1578: Quoted file name, single-line comment or end-of-line expected + // #line 1 @"file.cs"u8 + Diagnostic(ErrorCode.ERR_MissingPPFile, "@").WithLocation(1, 9) + ); + + N(SyntaxKind.LineDirectiveTrivia); + { + N(SyntaxKind.HashToken); + N(SyntaxKind.LineKeyword); + N(SyntaxKind.NumericLiteralToken, "1"); + N(SyntaxKind.EndOfDirectiveToken); + } + EOF(); + + } } } diff --git a/src/Compilers/CSharp/Test/Syntax/Parsing/ParsingTests.cs b/src/Compilers/CSharp/Test/Syntax/Parsing/ParsingTests.cs index 67ca85e9cd066..82c6e3106f9bc 100644 --- a/src/Compilers/CSharp/Test/Syntax/Parsing/ParsingTests.cs +++ b/src/Compilers/CSharp/Test/Syntax/Parsing/ParsingTests.cs @@ -291,6 +291,7 @@ private void Print(SyntaxNodeOrToken node, bool dump) case SyntaxKind.IdentifierToken: case SyntaxKind.NumericLiteralToken: case SyntaxKind.StringLiteralToken: + case SyntaxKind.UTF8StringLiteralToken: if (node.IsMissing) { goto default; diff --git a/src/Compilers/CSharp/Test/Syntax/Parsing/UTF8StringLiteralsParsingTests.cs b/src/Compilers/CSharp/Test/Syntax/Parsing/UTF8StringLiteralsParsingTests.cs new file mode 100644 index 0000000000000..b503937f8a17d --- /dev/null +++ b/src/Compilers/CSharp/Test/Syntax/Parsing/UTF8StringLiteralsParsingTests.cs @@ -0,0 +1,300 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + + +#nullable disable + +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Microsoft.CodeAnalysis.CSharp.Test.Utilities; +using Microsoft.CodeAnalysis.Test.Utilities; +using Xunit; +using Xunit.Abstractions; + +namespace Microsoft.CodeAnalysis.CSharp.UnitTests +{ + public class UTF8StringLiteralsParsingTests : ParsingTests + { + public UTF8StringLiteralsParsingTests(ITestOutputHelper output) : base(output) { } + + [Fact] + public void RegularStringLiteral() + { + UsingExpression(@"""hello"""); + + N(SyntaxKind.StringLiteralExpression); + { + N(SyntaxKind.StringLiteralToken, "\"hello\""); + } + EOF(); + } + + [Fact] + public void UTF8StringLiteral_01() + { + UsingExpression(@"""hello""u8"); + + N(SyntaxKind.UTF8StringLiteralExpression); + { + N(SyntaxKind.UTF8StringLiteralToken, "\"hello\"u8"); + } + EOF(); + } + + [Fact] + public void UTF8StringLiteral_02() + { + UsingExpression(@"""hello""u8", options: TestOptions.RegularNext); + + N(SyntaxKind.UTF8StringLiteralExpression); + { + N(SyntaxKind.UTF8StringLiteralToken, "\"hello\"u8"); + } + EOF(); + } + + [Fact] + public void UTF8StringLiteral_03() + { + UsingExpression(@"""hello""u8", options: TestOptions.Regular10); + + N(SyntaxKind.UTF8StringLiteralExpression); + { + N(SyntaxKind.UTF8StringLiteralToken, "\"hello\"u8"); + } + EOF(); + } + + [Fact] + public void UTF8StringLiteral_04() + { + UsingExpression(@"@""hello""u8"); + + N(SyntaxKind.UTF8StringLiteralExpression); + { + N(SyntaxKind.UTF8StringLiteralToken, "@\"hello\"u8"); + } + EOF(); + } + + [Fact] + public void UTF8StringLiteral_05() + { + UsingExpression(@"@""hello""u8", options: TestOptions.RegularNext); + + N(SyntaxKind.UTF8StringLiteralExpression); + { + N(SyntaxKind.UTF8StringLiteralToken, "@\"hello\"u8"); + } + EOF(); + } + + [Fact] + public void UTF8StringLiteral_06() + { + UsingExpression(@"@""hello""u8", options: TestOptions.Regular10); + + N(SyntaxKind.UTF8StringLiteralExpression); + { + N(SyntaxKind.UTF8StringLiteralToken, "@\"hello\"u8"); + } + EOF(); + } + + [Fact] + public void Errors_01() + { + // The behavior is consistent with how type syffixes are handled on numeric literals, see Errors_07. + UsingExpression(@"@""hello"" u8", + // (1,1): error CS1073: Unexpected token 'u8' + // @"hello" u8 + Diagnostic(ErrorCode.ERR_UnexpectedToken, @"@""hello""").WithArguments("u8").WithLocation(1, 1) + ); + + N(SyntaxKind.StringLiteralExpression); + { + N(SyntaxKind.StringLiteralToken, "@\"hello\""); + } + EOF(); + } + + [Fact] + public void Errors_02() + { + UsingExpression(@"@""hello""u", + // (1,1): error CS1073: Unexpected token 'u' + // @"hello"u + Diagnostic(ErrorCode.ERR_UnexpectedToken, @"@""hello""").WithArguments("u").WithLocation(1, 1) + ); + + N(SyntaxKind.StringLiteralExpression); + { + N(SyntaxKind.StringLiteralToken, "@\"hello\""); + } + EOF(); + } + + [Fact] + public void Errors_03() + { + UsingExpression(@"@""hello""8", + // (1,1): error CS1073: Unexpected token '8' + // @"hello"8 + Diagnostic(ErrorCode.ERR_UnexpectedToken, @"@""hello""").WithArguments("8").WithLocation(1, 1) + ); + + N(SyntaxKind.StringLiteralExpression); + { + N(SyntaxKind.StringLiteralToken, "@\"hello\""); + } + EOF(); + } + + [Fact] + public void Errors_04() + { + UsingExpression(@"@""hello""U8", + // (1,1): error CS1073: Unexpected token 'U8' + // @"hello"U8 + Diagnostic(ErrorCode.ERR_UnexpectedToken, @"@""hello""").WithArguments("U8").WithLocation(1, 1) + ); + + N(SyntaxKind.StringLiteralExpression); + { + N(SyntaxKind.StringLiteralToken, "@\"hello\""); + } + EOF(); + } + + [Fact] + public void Errors_05() + { + // The behavior is consistent with how type syffixes are handled on numeric literals, see Errors_06. + UsingExpression(@"@""hello""u80", + // (1,1): error CS1073: Unexpected token '0' + // @"hello"u80 + Diagnostic(ErrorCode.ERR_UnexpectedToken, @"@""hello""u8").WithArguments("0").WithLocation(1, 1) + ); + + N(SyntaxKind.UTF8StringLiteralExpression); + { + N(SyntaxKind.UTF8StringLiteralToken, "@\"hello\"u8"); + } + EOF(); + } + + [Fact] + public void Errors_06() + { + UsingExpression(@"1L0", + // (1,1): error CS1073: Unexpected token '0' + // 1l0 + Diagnostic(ErrorCode.ERR_UnexpectedToken, "1L").WithArguments("0").WithLocation(1, 1) + ); + + N(SyntaxKind.NumericLiteralExpression); + { + N(SyntaxKind.NumericLiteralToken, "1L"); + } + EOF(); + } + + [Fact] + public void Errors_07() + { + UsingExpression(@"1 L", + // (1,1): error CS1073: Unexpected token 'L' + // 1 L + Diagnostic(ErrorCode.ERR_UnexpectedToken, "1").WithArguments("L").WithLocation(1, 1) + ); + + N(SyntaxKind.NumericLiteralExpression); + { + N(SyntaxKind.NumericLiteralToken, "1"); + } + EOF(); + } + + [Fact] + public void Errors_08() + { + // The behavior is consistent with how type syffixes are handled on numeric literals, see Errors_07. + UsingExpression(@"""hello"" u8", + // (1,1): error CS1073: Unexpected token 'u8' + // "hello" u8 + Diagnostic(ErrorCode.ERR_UnexpectedToken, @"""hello""").WithArguments("u8").WithLocation(1, 1) + ); + + N(SyntaxKind.StringLiteralExpression); + { + N(SyntaxKind.StringLiteralToken, "\"hello\""); + } + EOF(); + } + + [Fact] + public void Errors_09() + { + UsingExpression(@"""hello""u", + // (1,1): error CS1073: Unexpected token 'u' + // "hello"u + Diagnostic(ErrorCode.ERR_UnexpectedToken, @"""hello""").WithArguments("u").WithLocation(1, 1) + ); + + N(SyntaxKind.StringLiteralExpression); + { + N(SyntaxKind.StringLiteralToken, "\"hello\""); + } + EOF(); + } + + [Fact] + public void Errors_10() + { + UsingExpression(@"""hello""8", + // (1,1): error CS1073: Unexpected token '8' + // "hello"8 + Diagnostic(ErrorCode.ERR_UnexpectedToken, @"""hello""").WithArguments("8").WithLocation(1, 1) + ); + + N(SyntaxKind.StringLiteralExpression); + { + N(SyntaxKind.StringLiteralToken, "\"hello\""); + } + EOF(); + } + + [Fact] + public void Errors_11() + { + UsingExpression(@"""hello""U8", + // (1,1): error CS1073: Unexpected token 'U8' + // "hello"U8 + Diagnostic(ErrorCode.ERR_UnexpectedToken, @"""hello""").WithArguments("U8").WithLocation(1, 1) + ); + + N(SyntaxKind.StringLiteralExpression); + { + N(SyntaxKind.StringLiteralToken, "\"hello\""); + } + EOF(); + } + + [Fact] + public void Errors_12() + { + // The behavior is consistent with how type syffixes are handled on numeric literals, see Errors_06. + UsingExpression(@"""hello""u80", + // (1,1): error CS1073: Unexpected token '0' + // "hello"u80 + Diagnostic(ErrorCode.ERR_UnexpectedToken, @"""hello""u8").WithArguments("0").WithLocation(1, 1) + ); + + N(SyntaxKind.UTF8StringLiteralExpression); + { + N(SyntaxKind.UTF8StringLiteralToken, "\"hello\"u8"); + } + EOF(); + } + } +} diff --git a/src/Compilers/Test/Core/TestResource.resx b/src/Compilers/Test/Core/TestResource.resx index d6297d3136bd1..441ff1883a7e9 100644 --- a/src/Compilers/Test/Core/TestResource.resx +++ b/src/Compilers/Test/Core/TestResource.resx @@ -278,6 +278,7 @@ namespace My sbyte @sbyte; short @short; string @string = @"""/*"; + var utf8String = @"""/*"u8; uint @uint; ulong @ulong; ushort @ushort; From a1336e0a76eb8e8a88b29c76e1712a0639f5a34a Mon Sep 17 00:00:00 2001 From: AlekseyTs Date: Fri, 21 Jan 2022 07:56:29 -0800 Subject: [PATCH 2/5] Addd a test --- .../Semantics/Utf8StringsLiteralsTests.cs | 26 +++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/src/Compilers/CSharp/Test/Semantic/Semantics/Utf8StringsLiteralsTests.cs b/src/Compilers/CSharp/Test/Semantic/Semantics/Utf8StringsLiteralsTests.cs index a91a0f522e1ee..92805a55d4bda 100644 --- a/src/Compilers/CSharp/Test/Semantic/Semantics/Utf8StringsLiteralsTests.cs +++ b/src/Compilers/CSharp/Test/Semantic/Semantics/Utf8StringsLiteralsTests.cs @@ -318,7 +318,7 @@ static void Main() public void InvalidContent_01() { var source = @" -using System; + class C { static void Main() @@ -327,7 +327,8 @@ static void Main() } } "; - var comp = CreateCompilation(source + HelpersSource, targetFramework: TargetFramework.NetCoreApp, options: TestOptions.DebugExe); + var comp = CreateCompilation(source, targetFramework: TargetFramework.NetCoreApp, options: TestOptions.DebugExe); + comp.VerifyDiagnostics(); comp.VerifyEmitDiagnostics( // (7,24): error CS8983: The input string cannot be converted into the equivalent UTF8 byte representation. Unable to translate Unicode character \\uD801 at index 6 to specified code page. // byte[] array = "hello \uD801\uD802"; @@ -335,6 +336,27 @@ static void Main() ); } + [ConditionalFact(typeof(CoreClrOnly))] + public void InvalidContent_02() + { + var source = @" +class C +{ + static void Main() + { + _ = ""hello \uD801\uD802""u8; + } +} +"; + var comp = CreateCompilation(source, targetFramework: TargetFramework.NetCoreApp, options: TestOptions.DebugExe); + comp.VerifyDiagnostics(); + comp.VerifyEmitDiagnostics( + // (6,13): error CS9100: The input string cannot be converted into the equivalent UTF8 byte representation. Unable to translate Unicode character \\uD801 at index 6 to specified code page. + // _ = "hello \uD801\uD802"u8; + Diagnostic(ErrorCode.ERR_CannotBeConvertedToUTF8, @"""hello \uD801\uD802""u8").WithArguments(@"Unable to translate Unicode character \\uD801 at index 6 to specified code page.").WithLocation(6, 13) + ); + } + [ConditionalFact(typeof(CoreClrOnly))] public void MissingHelpers_01() { From 33f43d85537d831265de84231ae2771026c3965a Mon Sep 17 00:00:00 2001 From: AlekseyTs Date: Fri, 21 Jan 2022 11:46:08 -0800 Subject: [PATCH 3/5] Add tests --- .../Parsing/UTF8StringLiteralsParsingTests.cs | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/src/Compilers/CSharp/Test/Syntax/Parsing/UTF8StringLiteralsParsingTests.cs b/src/Compilers/CSharp/Test/Syntax/Parsing/UTF8StringLiteralsParsingTests.cs index b503937f8a17d..968a40f29eb26 100644 --- a/src/Compilers/CSharp/Test/Syntax/Parsing/UTF8StringLiteralsParsingTests.cs +++ b/src/Compilers/CSharp/Test/Syntax/Parsing/UTF8StringLiteralsParsingTests.cs @@ -296,5 +296,47 @@ public void Errors_12() } EOF(); } + + [Fact] + public void Interpolation_01() + { + UsingExpression(@"$""hello""u8", + // (1,1): error CS1073: Unexpected token 'u8' + // $"hello"u8 + Diagnostic(ErrorCode.ERR_UnexpectedToken, @"$""hello""").WithArguments("u8").WithLocation(1, 1) + ); + + N(SyntaxKind.InterpolatedStringExpression); + { + N(SyntaxKind.InterpolatedStringStartToken); + N(SyntaxKind.InterpolatedStringText); + { + N(SyntaxKind.InterpolatedStringTextToken); + } + N(SyntaxKind.InterpolatedStringEndToken); + } + EOF(); + } + + [Fact] + public void Interpolation_02() + { + UsingExpression(@"$@""hello""u8", + // (1,1): error CS1073: Unexpected token 'u8' + // $@"hello"u8 + Diagnostic(ErrorCode.ERR_UnexpectedToken, @"$@""hello""").WithArguments("u8").WithLocation(1, 1) + ); + + N(SyntaxKind.InterpolatedStringExpression); + { + N(SyntaxKind.InterpolatedVerbatimStringStartToken); + N(SyntaxKind.InterpolatedStringText); + { + N(SyntaxKind.InterpolatedStringTextToken); + } + N(SyntaxKind.InterpolatedStringEndToken); + } + EOF(); + } } } From ea026f4dc90b3b86ffc6aa3948a482125ed62803 Mon Sep 17 00:00:00 2001 From: AlekseyTs Date: Fri, 21 Jan 2022 15:04:52 -0800 Subject: [PATCH 4/5] PR feedback --- .../CSharp/Portable/Binder/Binder_Expressions.cs | 1 + .../CSharp/Portable/Parser/Lexer_StringLiteral.cs | 12 ++++++++---- .../Syntax/Parsing/UTF8StringLiteralsParsingTests.cs | 8 ++++---- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/Compilers/CSharp/Portable/Binder/Binder_Expressions.cs b/src/Compilers/CSharp/Portable/Binder/Binder_Expressions.cs index 5c0d62b05f355..94901f36fd6cb 100644 --- a/src/Compilers/CSharp/Portable/Binder/Binder_Expressions.cs +++ b/src/Compilers/CSharp/Portable/Binder/Binder_Expressions.cs @@ -5909,6 +5909,7 @@ private BoundUTF8String BindUTF8StringLiteral(LiteralExpressionSyntax node, Bind var value = (string)node.Token.Value; var type = ArrayTypeSymbol.CreateSZArray(Compilation.Assembly, TypeWithAnnotations.Create(GetSpecialType(SpecialType.System_Byte, diagnostics, node))); + // PROTOTYPE(UTF8StringLiterals) : The result type is a deviation from the proposal, but I think it simplifies language rules. Will bring this for discussion to LDM. return new BoundUTF8String(node, value, type); } diff --git a/src/Compilers/CSharp/Portable/Parser/Lexer_StringLiteral.cs b/src/Compilers/CSharp/Portable/Parser/Lexer_StringLiteral.cs index 000e5ab1cd380..f588dd7dbc213 100644 --- a/src/Compilers/CSharp/Portable/Parser/Lexer_StringLiteral.cs +++ b/src/Compilers/CSharp/Portable/Parser/Lexer_StringLiteral.cs @@ -77,13 +77,15 @@ private void ScanStringLiteral(ref TokenInfo info, bool inDirective) } else { - info.Kind = SyntaxKind.StringLiteralToken; - if (!inDirective && TextWindow.PeekChar() == 'u' && TextWindow.PeekChar(1) == '8') { info.Kind = SyntaxKind.UTF8StringLiteralToken; TextWindow.AdvanceChar(2); } + else + { + info.Kind = SyntaxKind.StringLiteralToken; + } info.Text = TextWindow.GetText(intern: true); @@ -191,13 +193,15 @@ private void ScanVerbatimStringLiteral(ref TokenInfo info) _builder.Append(ch); } - info.Kind = SyntaxKind.StringLiteralToken; - if (TextWindow.PeekChar() == 'u' && TextWindow.PeekChar(1) == '8') { info.Kind = SyntaxKind.UTF8StringLiteralToken; TextWindow.AdvanceChar(2); } + else + { + info.Kind = SyntaxKind.StringLiteralToken; + } info.Text = TextWindow.GetText(intern: false); info.StringValue = _builder.ToString(); diff --git a/src/Compilers/CSharp/Test/Syntax/Parsing/UTF8StringLiteralsParsingTests.cs b/src/Compilers/CSharp/Test/Syntax/Parsing/UTF8StringLiteralsParsingTests.cs index 968a40f29eb26..36ff732cdcaff 100644 --- a/src/Compilers/CSharp/Test/Syntax/Parsing/UTF8StringLiteralsParsingTests.cs +++ b/src/Compilers/CSharp/Test/Syntax/Parsing/UTF8StringLiteralsParsingTests.cs @@ -104,7 +104,7 @@ public void UTF8StringLiteral_06() [Fact] public void Errors_01() { - // The behavior is consistent with how type syffixes are handled on numeric literals, see Errors_07. + // The behavior is consistent with how type suffixes are handled on numeric literals, see Errors_07. UsingExpression(@"@""hello"" u8", // (1,1): error CS1073: Unexpected token 'u8' // @"hello" u8 @@ -169,7 +169,7 @@ public void Errors_04() [Fact] public void Errors_05() { - // The behavior is consistent with how type syffixes are handled on numeric literals, see Errors_06. + // The behavior is consistent with how type suffixes are handled on numeric literals, see Errors_06. UsingExpression(@"@""hello""u80", // (1,1): error CS1073: Unexpected token '0' // @"hello"u80 @@ -218,7 +218,7 @@ public void Errors_07() [Fact] public void Errors_08() { - // The behavior is consistent with how type syffixes are handled on numeric literals, see Errors_07. + // The behavior is consistent with how type suffixes are handled on numeric literals, see Errors_07. UsingExpression(@"""hello"" u8", // (1,1): error CS1073: Unexpected token 'u8' // "hello" u8 @@ -283,7 +283,7 @@ public void Errors_11() [Fact] public void Errors_12() { - // The behavior is consistent with how type syffixes are handled on numeric literals, see Errors_06. + // The behavior is consistent with how type suffixes are handled on numeric literals, see Errors_06. UsingExpression(@"""hello""u80", // (1,1): error CS1073: Unexpected token '0' // "hello"u80 From 26fa7eb4700cbea526f4d2d1cc01d5e3c7a11a89 Mon Sep 17 00:00:00 2001 From: AlekseyTs Date: Mon, 24 Jan 2022 16:13:46 -0800 Subject: [PATCH 5/5] PR feedback --- .../CSharp/Portable/Binder/Binder_Expressions.cs | 4 ++++ .../CSharp/Portable/Parser/Lexer_StringLiteral.cs | 2 ++ .../CSharp/Portable/Syntax/SyntaxEquivalence.cs | 2 +- .../Semantic/Semantics/Utf8StringsLiteralsTests.cs | 13 ++++++++++++- 4 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/Compilers/CSharp/Portable/Binder/Binder_Expressions.cs b/src/Compilers/CSharp/Portable/Binder/Binder_Expressions.cs index 94901f36fd6cb..35618660ebf7a 100644 --- a/src/Compilers/CSharp/Portable/Binder/Binder_Expressions.cs +++ b/src/Compilers/CSharp/Portable/Binder/Binder_Expressions.cs @@ -674,6 +674,10 @@ private BoundExpression BindExpressionInternal(ExpressionSyntax node, BindingDia return BindLiteralConstant((LiteralExpressionSyntax)node, diagnostics); case SyntaxKind.UTF8StringLiteralExpression: + // PROTOTYPE(UTF8StringLiterals) : In the raw-string-work we decided to not have a special expression type. + // It's just a string-literal that consumer can check the token kind on. + // Once that feature makes it into this branch, evaluate if we can/want + // to do the same for this feature. return BindUTF8StringLiteral((LiteralExpressionSyntax)node, diagnostics); case SyntaxKind.DefaultLiteralExpression: diff --git a/src/Compilers/CSharp/Portable/Parser/Lexer_StringLiteral.cs b/src/Compilers/CSharp/Portable/Parser/Lexer_StringLiteral.cs index f588dd7dbc213..3af0a64a6cd90 100644 --- a/src/Compilers/CSharp/Portable/Parser/Lexer_StringLiteral.cs +++ b/src/Compilers/CSharp/Portable/Parser/Lexer_StringLiteral.cs @@ -77,6 +77,7 @@ private void ScanStringLiteral(ref TokenInfo info, bool inDirective) } else { + // PROTOTYPE(UTF8StringLiterals) : Should the suffix be case-insensitive? if (!inDirective && TextWindow.PeekChar() == 'u' && TextWindow.PeekChar(1) == '8') { info.Kind = SyntaxKind.UTF8StringLiteralToken; @@ -193,6 +194,7 @@ private void ScanVerbatimStringLiteral(ref TokenInfo info) _builder.Append(ch); } + // PROTOTYPE(UTF8StringLiterals) : Should the suffix be case-insensitive? if (TextWindow.PeekChar() == 'u' && TextWindow.PeekChar(1) == '8') { info.Kind = SyntaxKind.UTF8StringLiteralToken; diff --git a/src/Compilers/CSharp/Portable/Syntax/SyntaxEquivalence.cs b/src/Compilers/CSharp/Portable/Syntax/SyntaxEquivalence.cs index 032f700094cce..c873a7f7e62a3 100644 --- a/src/Compilers/CSharp/Portable/Syntax/SyntaxEquivalence.cs +++ b/src/Compilers/CSharp/Portable/Syntax/SyntaxEquivalence.cs @@ -82,7 +82,7 @@ private static bool AreTokensEquivalent(GreenNode? before, GreenNode? after, Fun case SyntaxKind.NumericLiteralToken: case SyntaxKind.CharacterLiteralToken: case SyntaxKind.StringLiteralToken: - case SyntaxKind.UTF8StringLiteralToken: + case SyntaxKind.UTF8StringLiteralToken: // PROTOTYPE(UTF8StringLiterals) : add test coverage for this code path. case SyntaxKind.InterpolatedStringTextToken: if (((Green.SyntaxToken)before).Text != ((Green.SyntaxToken)after).Text) { diff --git a/src/Compilers/CSharp/Test/Semantic/Semantics/Utf8StringsLiteralsTests.cs b/src/Compilers/CSharp/Test/Semantic/Semantics/Utf8StringsLiteralsTests.cs index 92805a55d4bda..0962aa2b608d4 100644 --- a/src/Compilers/CSharp/Test/Semantic/Semantics/Utf8StringsLiteralsTests.cs +++ b/src/Compilers/CSharp/Test/Semantic/Semantics/Utf8StringsLiteralsTests.cs @@ -1972,8 +1972,19 @@ static class E } "; var comp = CreateCompilation(source, targetFramework: TargetFramework.NetCoreApp, options: TestOptions.DebugExe); - CompileAndVerify(comp, expectedOutput: @"byte[]"). // The behavior has changed + // The behavior has changed + // PROTOTYPE(UTF8StringLiterals) : Add an entry in "docs/compilers/CSharp/Compiler Breaking Changes - DotNet 7.md"? + CompileAndVerify(comp, expectedOutput: @"byte[]"). VerifyDiagnostics(); + + comp = CreateCompilation(source, targetFramework: TargetFramework.NetCoreApp, options: TestOptions.DebugExe, parseOptions: TestOptions.Regular10); + // PROTOTYPE(UTF8StringLiterals) : Confirm we are comfortable with not changing semantics based on language version and keeping an error for this scenario. + // PROTOTYPE(UTF8StringLiterals) : Add an entry in "docs/compilers/CSharp/Compiler Breaking Changes - DotNet 7.md"? + comp.VerifyDiagnostics( + // (9,31): error CS8652: The feature 'Utf8 String Literals' is currently in Preview and *unsupported*. To use Preview features, use the 'preview' language version. + // Console.WriteLine(p.M("")); + Diagnostic(ErrorCode.ERR_FeatureInPreview, @"""""").WithArguments("Utf8 String Literals").WithLocation(9, 31) + ); } [Fact]