From ad61af02da435652ba0662d34205f60fa88537cb Mon Sep 17 00:00:00 2001 From: Tom Bruns Date: Mon, 19 Feb 2024 16:57:22 -0500 Subject: [PATCH 1/7] AVRO-2825: [csharp] Resolve: C# Logical Types throw exception on unknown logical type --- .../codegen/Properties/launchSettings.json | 8 +++ .../main/Properties/launchSettings.json | 8 +++ .../src/apache/main/Schema/LogicalSchema.cs | 2 +- .../apache/main/Util/LogicalTypeFactory.cs | 22 +++--- .../apache/main/Util/UnknownLogicalType.cs | 57 +++++++++++++++ .../apache/test/AvroGen/AvroGenSchemaTests.cs | 5 +- .../src/apache/test/Schema/SchemaTests.cs | 70 ++++++++++++++++++- 7 files changed, 157 insertions(+), 15 deletions(-) create mode 100644 lang/csharp/src/apache/codegen/Properties/launchSettings.json create mode 100644 lang/csharp/src/apache/main/Properties/launchSettings.json create mode 100644 lang/csharp/src/apache/main/Util/UnknownLogicalType.cs diff --git a/lang/csharp/src/apache/codegen/Properties/launchSettings.json b/lang/csharp/src/apache/codegen/Properties/launchSettings.json new file mode 100644 index 00000000000..ecb15cbfda9 --- /dev/null +++ b/lang/csharp/src/apache/codegen/Properties/launchSettings.json @@ -0,0 +1,8 @@ +{ + "profiles": { + "Avro.codegen": { + "commandName": "Project", + "commandLineArgs": "-s C:\\Users\\Thomas.Bruns\\source\\repos\\AzureDevOps\\TQL.Kafka.Samples\\TQL.Kafka.Samples\\TQL.Kafka.Samples.Messages\\v2\\cdc_customers_dbo_tblcustomers.avsc .\\tab" + } + } +} \ No newline at end of file diff --git a/lang/csharp/src/apache/main/Properties/launchSettings.json b/lang/csharp/src/apache/main/Properties/launchSettings.json new file mode 100644 index 00000000000..277fa29573e --- /dev/null +++ b/lang/csharp/src/apache/main/Properties/launchSettings.json @@ -0,0 +1,8 @@ +{ + "profiles": { + "Avro.main": { + "commandName": "Project", + "commandLineArgs": "C:\\Users\\Thomas.Bruns\\source\\repos\\AzureDevOps\\TQL.Kafka.Samples\\TQL.Kafka.Samples\\TQL.Kafka.Samples.Messages\\v2\\cdc_customers_dbo_tblcustomers.avsc" + } + } +} \ No newline at end of file diff --git a/lang/csharp/src/apache/main/Schema/LogicalSchema.cs b/lang/csharp/src/apache/main/Schema/LogicalSchema.cs index 181260f2ca2..c2e589eeb33 100644 --- a/lang/csharp/src/apache/main/Schema/LogicalSchema.cs +++ b/lang/csharp/src/apache/main/Schema/LogicalSchema.cs @@ -54,7 +54,7 @@ private LogicalSchema(Schema baseSchema, string logicalTypeName, PropertyMap pr { BaseSchema = baseSchema ?? throw new ArgumentNullException(nameof(baseSchema)); LogicalTypeName = logicalTypeName; - LogicalType = LogicalTypeFactory.Instance.GetFromLogicalSchema(this); + LogicalType = LogicalTypeFactory.Instance.GetFromLogicalSchema(this, true); } /// diff --git a/lang/csharp/src/apache/main/Util/LogicalTypeFactory.cs b/lang/csharp/src/apache/main/Util/LogicalTypeFactory.cs index f4086ab5a27..d6f8dce9512 100644 --- a/lang/csharp/src/apache/main/Util/LogicalTypeFactory.cs +++ b/lang/csharp/src/apache/main/Util/LogicalTypeFactory.cs @@ -45,7 +45,7 @@ private LogicalTypeFactory() { TimeMicrosecond.LogicalTypeName, new TimeMicrosecond() }, { TimestampMillisecond.LogicalTypeName, new TimestampMillisecond() }, { TimestampMicrosecond.LogicalTypeName, new TimestampMicrosecond() }, - { Uuid.LogicalTypeName, new Uuid() } + { Uuid.LogicalTypeName, new Uuid() }, }; } @@ -67,22 +67,22 @@ public void Register(LogicalType logicalType) /// A . public LogicalType GetFromLogicalSchema(LogicalSchema schema, bool ignoreInvalidOrUnknown = false) { - try - { - if (!_logicalTypes.TryGetValue(schema.LogicalTypeName, out LogicalType logicalType)) - throw new AvroTypeException("Logical type '" + schema.LogicalTypeName + "' is not supported."); + LogicalType logicalType = null; + if (_logicalTypes.TryGetValue(schema.LogicalTypeName, out logicalType)) + { logicalType.ValidateSchema(schema); - - return logicalType; } - catch (AvroTypeException) + else if (ignoreInvalidOrUnknown) + { + logicalType = new UnknownLogicalType(schema); + } + else { - if (!ignoreInvalidOrUnknown) - throw; + throw new AvroTypeException("Logical type '" + schema.LogicalTypeName + "' is not supported."); } - return null; + return logicalType; } } } diff --git a/lang/csharp/src/apache/main/Util/UnknownLogicalType.cs b/lang/csharp/src/apache/main/Util/UnknownLogicalType.cs new file mode 100644 index 00000000000..d480712e871 --- /dev/null +++ b/lang/csharp/src/apache/main/Util/UnknownLogicalType.cs @@ -0,0 +1,57 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Avro.Util +{ + public class UnknownLogicalType : LogicalType + { + public LogicalSchema Schema { get; } + + public UnknownLogicalType(LogicalSchema schema) : base(schema.LogicalTypeName) + { + this.Schema = schema; + } + + public override object ConvertToBaseValue(object logicalValue, LogicalSchema schema) + { + throw new NotImplementedException(); + } + + public override object ConvertToLogicalValue(object baseValue, LogicalSchema schema) + { + throw new NotImplementedException(); + } + + public override Type GetCSharpType(bool nullible) + { + // handle all Primitive Types + switch (this.Schema.BaseSchema.Name) + { + case @"string": + return typeof(System.String); + case @"boolean": + return typeof(System.Boolean); + case @"int": + return typeof(System.Int32); + case @"long": + return typeof(System.Int64); + case @"float": + return typeof(System.Single); + case @"double": + return typeof(System.Double); + case @"bytes": + return typeof(System.Byte[]); + default: + return typeof(System.Object); + } + } + + public override bool IsInstanceOfLogicalType(object logicalValue) + { + // => throw new NotImplementedException(); + return true; + } + + } +} diff --git a/lang/csharp/src/apache/test/AvroGen/AvroGenSchemaTests.cs b/lang/csharp/src/apache/test/AvroGen/AvroGenSchemaTests.cs index 807acbda92a..4db445ea53a 100644 --- a/lang/csharp/src/apache/test/AvroGen/AvroGenSchemaTests.cs +++ b/lang/csharp/src/apache/test/AvroGen/AvroGenSchemaTests.cs @@ -619,7 +619,10 @@ public void NotSupportedSchema(string schema, Type expectedException) string schemaFileName = Path.Combine(outputDir, $"{uniqueId}.avsc"); System.IO.File.WriteAllText(schemaFileName, schema); - Assert.That(AvroGenTool.GenSchema(schemaFileName, outputDir, new Dictionary(), false), Is.EqualTo(1)); + // We now support unknown logical types + //Assert.That(AvroGenTool.GenSchema(schemaFileName, outputDir, new Dictionary(), false), Is.EqualTo(1)); + Assert.That(AvroGenTool.GenSchema(schemaFileName, outputDir, new Dictionary(), false), Is.EqualTo(0)); + } finally { diff --git a/lang/csharp/src/apache/test/Schema/SchemaTests.cs b/lang/csharp/src/apache/test/Schema/SchemaTests.cs index 319e9a95be3..aca35e1dbd1 100644 --- a/lang/csharp/src/apache/test/Schema/SchemaTests.cs +++ b/lang/csharp/src/apache/test/Schema/SchemaTests.cs @@ -19,6 +19,7 @@ using System.Collections.Generic; using NUnit.Framework; using System.Linq; +using Avro.Util; namespace Avro.Test { @@ -548,12 +549,77 @@ public void TestLogicalPrimitive(string s, string baseType, string logicalType) testToString(sc); } + // Make sure unknown type is carried thru to LogicalTypeName [TestCase("{\"type\": \"int\", \"logicalType\": \"unknown\"}", "unknown")] public void TestUnknownLogical(string s, string unknownType) { - var err = Assert.Throws(() => Schema.Parse(s)); + //var err = Assert.Throws(() => Schema.Parse(s)); + //Assert.AreEqual("Logical type '" + unknownType + "' is not supported.", err.Message); - Assert.AreEqual("Logical type '" + unknownType + "' is not supported.", err.Message); + var schema = Schema.Parse(s); + Assert.IsInstanceOf(typeof(LogicalSchema), schema); + + var logicalSchema = schema as LogicalSchema; + Assert.IsInstanceOf(typeof(UnknownLogicalType), logicalSchema.LogicalType); + + Assert.AreEqual(logicalSchema.LogicalTypeName, unknownType); + } + + /* + { + "fields": [ + { + "default": 0, + "name": "firstField", + "type": "int" + }, + { + "default": null, + "name": "secondField", + "type": [ + "null", + { + "logicalType": "varchar", + "maxLength": 65, + "type": "string" + } + ] + } + ], + "name": "sample_schema", + "type": "record" + } + */ + + // Before Change will throw Avro.AvroTypeException: 'Logical type 'varchar' is not supported.' + // Per AVRO Spec (v1.8.0 - v1.11.1) ... Logical Types Section + // Language implementations must ignore unknown logical types when reading, and should use the underlying Avro type. + [TestCase("{\"fields\": [{\"default\": 0,\"name\": \"firstField\",\"type\": \"int\"},{\"default\": null,\"name\": \"secondField\",\"type\": [\"null\",{\"logicalType\": \"varchar\",\"maxLength\": 65,\"type\": \"string\"}]}],\"name\": \"sample_schema\",\"type\": \"record\"}")] + public void TestUnknownLogicalType(string schemaText) + { + var schema = Avro.Schema.Parse(schemaText); + Assert.IsNotNull(schema); + + var secondField = ((RecordSchema)schema).Fields.FirstOrDefault(f => f.Name == @"secondField"); + Assert.IsNotNull(secondField); + + var secondFieldSchema = ((Field)secondField).Schema; + Assert.IsNotNull(secondFieldSchema); + + var secondFieldUnionSchema = (UnionSchema)secondFieldSchema; + Assert.IsNotNull(secondFieldUnionSchema); + + var props = secondFieldUnionSchema.Schemas.Where(s => s.Props != null).ToList(); + Assert.IsNotNull(props); + Assert.IsTrue(props.Count == 1); + + var prop = props[0]; + // Confirm that the unknown logical type is ignored and the underlying AVRO type is used + Assert.IsTrue(prop.Name == @"string"); + var logicalSchema = prop as LogicalSchema; + Assert.IsInstanceOf(typeof(UnknownLogicalType), logicalSchema.LogicalType); + + Assert.AreEqual(logicalSchema.LogicalTypeName, @"varchar"); } [TestCase("{\"type\": \"map\", \"values\": \"long\"}", "long")] From e5a7dedf9959687637f621bfdecbe478a5a89cc4 Mon Sep 17 00:00:00 2001 From: Tom Bruns Date: Tue, 20 Feb 2024 15:48:22 -0500 Subject: [PATCH 2/7] [AVRO-3941] CSharp Resolve missing namespace issue --- lang/csharp/src/apache/codegen/Avro.codegen.csproj | 4 ++++ lang/csharp/src/apache/codegen/AvroGen.cs | 12 +++++++++++- .../apache/codegen/Properties/launchSettings.json | 2 +- lang/csharp/src/apache/main/CodeGen/CodeGen.cs | 13 +++++++++++-- .../src/apache/test/AvroGen/AvroGenSchemaTests.cs | 8 ++++---- 5 files changed, 31 insertions(+), 8 deletions(-) diff --git a/lang/csharp/src/apache/codegen/Avro.codegen.csproj b/lang/csharp/src/apache/codegen/Avro.codegen.csproj index 94aa8123119..d37dc60514b 100644 --- a/lang/csharp/src/apache/codegen/Avro.codegen.csproj +++ b/lang/csharp/src/apache/codegen/Avro.codegen.csproj @@ -62,6 +62,10 @@ + + + + diff --git a/lang/csharp/src/apache/codegen/AvroGen.cs b/lang/csharp/src/apache/codegen/AvroGen.cs index 3b07ca59fdc..f0dc2ce2557 100644 --- a/lang/csharp/src/apache/codegen/AvroGen.cs +++ b/lang/csharp/src/apache/codegen/AvroGen.cs @@ -155,6 +155,10 @@ static void Usage() " --namespace Map an Avro schema/protocol namespace to a C# namespace.\n" + " The format is \"my.avro.namespace:my.csharp.namespace\".\n" + " May be specified multiple times to map multiple namespaces.\n" + + " If the source schema is missing a top level namespace\n" + + $" the namespace [{CodeGen.NO_SCHEMA_NAMESPACE}] will be injected.\n" + + " You can replace this using the --namespace command line option\n" + + " or edit the generated class file afterwards.\n" + " --skip-directories Skip creation of namespace directories. It will generate classes right inside output directory\n", AppDomain.CurrentDomain.FriendlyName); } @@ -170,6 +174,9 @@ public static int GenProtocol(string infile, string outdir, codegen.AddProtocol(text, namespaceMapping); codegen.GenerateCode(); + + bool replaceTempNamespace = namespaceMapping.Any(n => n.Key == CodeGen.NO_SCHEMA_NAMESPACE); + codegen.WriteTypes(outdir); } catch (Exception ex) @@ -191,7 +198,10 @@ public static int GenSchema(string infile, string outdir, codegen.AddSchema(text, namespaceMapping); codegen.GenerateCode(); - codegen.WriteTypes(outdir, skipDirectories); + + var overrideClassNamepace = namespaceMapping.FirstOrDefault(n => n.Key.ToLower() == CodeGen.NO_SCHEMA_NAMESPACE.ToLower()).Value; + + codegen.WriteTypes(outdir, skipDirectories, overrideClassNamepace); } catch (Exception ex) { diff --git a/lang/csharp/src/apache/codegen/Properties/launchSettings.json b/lang/csharp/src/apache/codegen/Properties/launchSettings.json index ecb15cbfda9..a54467aa70a 100644 --- a/lang/csharp/src/apache/codegen/Properties/launchSettings.json +++ b/lang/csharp/src/apache/codegen/Properties/launchSettings.json @@ -2,7 +2,7 @@ "profiles": { "Avro.codegen": { "commandName": "Project", - "commandLineArgs": "-s C:\\Users\\Thomas.Bruns\\source\\repos\\AzureDevOps\\TQL.Kafka.Samples\\TQL.Kafka.Samples\\TQL.Kafka.Samples.Messages\\v2\\cdc_customers_dbo_tblcustomers.avsc .\\tab" + "commandLineArgs": " -s C:\\Users\\Thomas.Bruns\\source\\repos\\AzureDevOps\\TQL.Kafka.Samples\\TQL.Kafka.Samples\\TQL.Kafka.Samples.Messages\\v1\\cdc_tql_dbo_tbldrops.avsc .\\tab --namespace NO_SCHEMA_NAMESPACE:TQL.DEMO" } } } \ No newline at end of file diff --git a/lang/csharp/src/apache/main/CodeGen/CodeGen.cs b/lang/csharp/src/apache/main/CodeGen/CodeGen.cs index 7e793627201..15199d2f66d 100644 --- a/lang/csharp/src/apache/main/CodeGen/CodeGen.cs +++ b/lang/csharp/src/apache/main/CodeGen/CodeGen.cs @@ -34,6 +34,8 @@ namespace Avro /// public class CodeGen { + public static string NO_SCHEMA_NAMESPACE = @"NO_SCHEMA_NAMESPACE"; + /// /// Gets object that contains all the generated types. /// @@ -895,7 +897,9 @@ protected virtual CodeTypeDeclaration processRecord(Schema schema) string nspace = recordSchema.Namespace; if (string.IsNullOrEmpty(nspace)) { - throw new CodeGenException("Namespace required for record schema " + recordSchema.Name); + // if the schema does not have a namespace defined, insert a temp value so code gen will not fail + //throw new CodeGenException("Namespace required for record schema " + recordSchema.Name); + nspace = NO_SCHEMA_NAMESPACE; } CodeNamespace codens = AddNamespace(nspace); @@ -1185,7 +1189,7 @@ public virtual IDictionary GetTypes() /// /// name of directory to write to. /// skip creation of directories based on schema namespace - public virtual void WriteTypes(string outputdir, bool skipDirectories = false) + public virtual void WriteTypes(string outputdir, bool skipDirectories = false, string overrideClassNamepace = null) { var cscp = new CSharpCodeProvider(); @@ -1209,6 +1213,11 @@ public virtual void WriteTypes(string outputdir, bool skipDirectories = false) } Directory.CreateDirectory(dir); + if(ns.Name == CodeGen.NO_SCHEMA_NAMESPACE && !string.IsNullOrEmpty(overrideClassNamepace)) + { + ns.Name = overrideClassNamepace; + } + var new_ns = new CodeNamespace(ns.Name); new_ns.Comments.Add(CodeGenUtil.Instance.FileComment); foreach (CodeNamespaceImport nci in CodeGenUtil.Instance.NamespaceImports) diff --git a/lang/csharp/src/apache/test/AvroGen/AvroGenSchemaTests.cs b/lang/csharp/src/apache/test/AvroGen/AvroGenSchemaTests.cs index 4db445ea53a..b9b2e74b01e 100644 --- a/lang/csharp/src/apache/test/AvroGen/AvroGenSchemaTests.cs +++ b/lang/csharp/src/apache/test/AvroGen/AvroGenSchemaTests.cs @@ -606,9 +606,9 @@ public void GenerateSchemaWithNamespaceMapping( AvroGenHelper.TestSchema(schema, typeNamesToCheck, new Dictionary { { namespaceMappingFrom, namespaceMappingTo } }, generatedFilesToCheck); } - [TestCase(_logicalTypesWithCustomConversion, typeof(AvroTypeException))] - [TestCase(_customConversionWithLogicalTypes, typeof(SchemaParseException))] - public void NotSupportedSchema(string schema, Type expectedException) + [TestCase(_logicalTypesWithCustomConversion, typeof(AvroTypeException), 0)] + [TestCase(_customConversionWithLogicalTypes, typeof(SchemaParseException), 1)] + public void NotSupportedSchema(string schema, Type expectedException, int expectedResult) { // Create temp folder string outputDir = AvroGenHelper.CreateEmptyTemporaryFolder(out string uniqueId); @@ -621,7 +621,7 @@ public void NotSupportedSchema(string schema, Type expectedException) // We now support unknown logical types //Assert.That(AvroGenTool.GenSchema(schemaFileName, outputDir, new Dictionary(), false), Is.EqualTo(1)); - Assert.That(AvroGenTool.GenSchema(schemaFileName, outputDir, new Dictionary(), false), Is.EqualTo(0)); + Assert.That(AvroGenTool.GenSchema(schemaFileName, outputDir, new Dictionary(), false), Is.EqualTo(expectedResult)); } finally From 89095295036d0dbd180fa4265205f0d666b3c0d9 Mon Sep 17 00:00:00 2001 From: Tom Bruns Date: Wed, 21 Feb 2024 08:55:50 -0500 Subject: [PATCH 3/7] Resolve requested changes on PR 2751 --- .../codegen/Properties/launchSettings.json | 8 - .../main/Properties/launchSettings.json | 8 - .../apache/main/Util/UnknownLogicalType.cs | 128 +++++++++++- .../apache/test/AvroGen/AvroGenSchemaTests.cs | 9 +- lang/csharp/src/apache/test/File/FileTests.cs | 33 +++ .../src/apache/test/Schema/SchemaTests.cs | 4 +- .../test/Util/UnknownLogicalTypeTests.cs | 188 ++++++++++++++++++ 7 files changed, 345 insertions(+), 33 deletions(-) delete mode 100644 lang/csharp/src/apache/codegen/Properties/launchSettings.json delete mode 100644 lang/csharp/src/apache/main/Properties/launchSettings.json create mode 100644 lang/csharp/src/apache/test/Util/UnknownLogicalTypeTests.cs diff --git a/lang/csharp/src/apache/codegen/Properties/launchSettings.json b/lang/csharp/src/apache/codegen/Properties/launchSettings.json deleted file mode 100644 index ecb15cbfda9..00000000000 --- a/lang/csharp/src/apache/codegen/Properties/launchSettings.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "profiles": { - "Avro.codegen": { - "commandName": "Project", - "commandLineArgs": "-s C:\\Users\\Thomas.Bruns\\source\\repos\\AzureDevOps\\TQL.Kafka.Samples\\TQL.Kafka.Samples\\TQL.Kafka.Samples.Messages\\v2\\cdc_customers_dbo_tblcustomers.avsc .\\tab" - } - } -} \ No newline at end of file diff --git a/lang/csharp/src/apache/main/Properties/launchSettings.json b/lang/csharp/src/apache/main/Properties/launchSettings.json deleted file mode 100644 index 277fa29573e..00000000000 --- a/lang/csharp/src/apache/main/Properties/launchSettings.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "profiles": { - "Avro.main": { - "commandName": "Project", - "commandLineArgs": "C:\\Users\\Thomas.Bruns\\source\\repos\\AzureDevOps\\TQL.Kafka.Samples\\TQL.Kafka.Samples\\TQL.Kafka.Samples.Messages\\v2\\cdc_customers_dbo_tblcustomers.avsc" - } - } -} \ No newline at end of file diff --git a/lang/csharp/src/apache/main/Util/UnknownLogicalType.cs b/lang/csharp/src/apache/main/Util/UnknownLogicalType.cs index d480712e871..2e3229e8b06 100644 --- a/lang/csharp/src/apache/main/Util/UnknownLogicalType.cs +++ b/lang/csharp/src/apache/main/Util/UnknownLogicalType.cs @@ -1,28 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ using System; using System.Collections.Generic; using System.Text; namespace Avro.Util { + /// + /// Class UnknownLogicalType. + /// Implements the + /// + /// public class UnknownLogicalType : LogicalType { + /// + /// Gets the schema. + /// + /// The schema. public LogicalSchema Schema { get; } + /// + /// Initializes a new instance of the class. + /// + /// The schema. public UnknownLogicalType(LogicalSchema schema) : base(schema.LogicalTypeName) { this.Schema = schema; } + /// + /// Converts a logical value to an instance of its base type. + /// + /// The logical value to convert. + /// The schema that represents the target of the conversion. + /// An object representing the encoded value of the base type. public override object ConvertToBaseValue(object logicalValue, LogicalSchema schema) - { - throw new NotImplementedException(); + { + switch (schema.Name) + { + case @"string": + return (System.String)logicalValue; + case @"boolean": + return (System.Boolean)logicalValue; + case @"int": + return (System.Int32)logicalValue; + case @"long": + return (System.Int64)logicalValue; + case @"float": + return (System.Single)logicalValue; + case @"double": + return (System.Double)logicalValue; + case @"bytes": + return (System.Byte[])logicalValue; + default: + return logicalValue; + } } + /// + /// Converts a base value to an instance of the logical type. + /// + /// The base value to convert. + /// The schema that represents the target of the conversion. + /// An object representing the encoded value of the logical type. public override object ConvertToLogicalValue(object baseValue, LogicalSchema schema) { - throw new NotImplementedException(); + switch (schema.Name) + { + case @"string": + return (System.String)baseValue; + case @"boolean": + return (System.Boolean)baseValue; + case @"int": + return (System.Int32)baseValue; + case @"long": + return (System.Int64)baseValue; + case @"float": + return (System.Single)baseValue; + case @"double": + return (System.Double)baseValue; + case @"bytes": + return (System.Byte[])baseValue; + default: + return baseValue; + } } + /// + /// Retrieve the .NET type that is represented by the logical type implementation. + /// + /// A flag indicating whether it should be nullible. + /// Type. public override Type GetCSharpType(bool nullible) { // handle all Primitive Types @@ -31,26 +114,49 @@ public override Type GetCSharpType(bool nullible) case @"string": return typeof(System.String); case @"boolean": - return typeof(System.Boolean); + return nullible ? typeof(System.Boolean?) : typeof(System.Boolean); case @"int": - return typeof(System.Int32); + return nullible ? typeof(System.Int32?) : typeof(System.Int32); case @"long": - return typeof(System.Int64); + return nullible ? typeof(System.Int64?) : typeof(System.Int64); case @"float": - return typeof(System.Single); + return nullible ? typeof(System.Single?) : typeof(System.Single); case @"double": - return typeof(System.Double); + return nullible ? typeof(System.Double?) : typeof(System.Double); case @"bytes": - return typeof(System.Byte[]); + return nullible ? typeof(System.Byte?[]) : typeof(System.Byte[]); default: return typeof(System.Object); } } + /// + /// Determines if a given object is an instance of the logical type. + /// + /// The logical value to test. + /// true if [is instance of logical type] [the specified logical value]; otherwise, false. public override bool IsInstanceOfLogicalType(object logicalValue) { - // => throw new NotImplementedException(); - return true; + // handle all Primitive Types + switch (this.Schema.BaseSchema.Name) + { + case @"string": + return logicalValue is System.String; + case @"boolean": + return logicalValue is System.Boolean; + case @"int": + return logicalValue is System.Int32; + case @"long": + return logicalValue is System.Int64; + case @"float": + return logicalValue is System.Single; + case @"double": + return logicalValue is System.Double; + case @"bytes": + return logicalValue is System.Byte[]; + default: + return true; + } } } diff --git a/lang/csharp/src/apache/test/AvroGen/AvroGenSchemaTests.cs b/lang/csharp/src/apache/test/AvroGen/AvroGenSchemaTests.cs index 4db445ea53a..d940a1bef83 100644 --- a/lang/csharp/src/apache/test/AvroGen/AvroGenSchemaTests.cs +++ b/lang/csharp/src/apache/test/AvroGen/AvroGenSchemaTests.cs @@ -606,9 +606,9 @@ public void GenerateSchemaWithNamespaceMapping( AvroGenHelper.TestSchema(schema, typeNamesToCheck, new Dictionary { { namespaceMappingFrom, namespaceMappingTo } }, generatedFilesToCheck); } - [TestCase(_logicalTypesWithCustomConversion, typeof(AvroTypeException))] - [TestCase(_customConversionWithLogicalTypes, typeof(SchemaParseException))] - public void NotSupportedSchema(string schema, Type expectedException) + //[TestCase(_logicalTypesWithCustomConversion, typeof(AvroTypeException), 0)] + [TestCase(_customConversionWithLogicalTypes, typeof(SchemaParseException), 1)] + public void NotSupportedSchema(string schema, Type expectedException, int expectedResult) { // Create temp folder string outputDir = AvroGenHelper.CreateEmptyTemporaryFolder(out string uniqueId); @@ -620,8 +620,7 @@ public void NotSupportedSchema(string schema, Type expectedException) System.IO.File.WriteAllText(schemaFileName, schema); // We now support unknown logical types - //Assert.That(AvroGenTool.GenSchema(schemaFileName, outputDir, new Dictionary(), false), Is.EqualTo(1)); - Assert.That(AvroGenTool.GenSchema(schemaFileName, outputDir, new Dictionary(), false), Is.EqualTo(0)); + Assert.That(AvroGenTool.GenSchema(schemaFileName, outputDir, new Dictionary(), false), Is.EqualTo(expectedResult)); } finally diff --git a/lang/csharp/src/apache/test/File/FileTests.cs b/lang/csharp/src/apache/test/File/FileTests.cs index 0ef81c9766f..abb3f9c6076 100644 --- a/lang/csharp/src/apache/test/File/FileTests.cs +++ b/lang/csharp/src/apache/test/File/FileTests.cs @@ -35,6 +35,34 @@ public class FileTests const string specificSchema = "{\"type\":\"record\",\"name\":\"Foo\",\"namespace\":\"Avro.Test.File\",\"fields\":" + "[{\"name\":\"name\",\"type\":[\"null\",\"string\"]},{\"name\":\"age\",\"type\":\"int\"}]}"; + /// + /// This test case added to confirm standalone serialization / deserialization behavior of new type UnknownLogicalType + /// + const string unknowLogicalTypeSchema = @" +{ + ""type"" : ""record"", + ""name"" : ""Foo"", + ""namespace"" : ""Avro.Test.File"", + ""fields"": [ + { + ""name"" :""name"", + ""type"": [ + ""null"", + { + ""logicalType"": ""varchar"", + ""maxLength"": 65, + ""type"": ""string"" + } + ] + }, + { + ""name"" : ""age"", + ""type"" : ""int"" + } + ] +} +"; + private static IEnumerable TestSpecificDataSource() { foreach (Codec.Type codecType in Enum.GetValues(typeof(Codec.Type))) @@ -100,6 +128,11 @@ private static IEnumerable TestSpecificDataSource() new object[] { "Bob", 9 }, new object[] { null, 48 } }, codecType).SetName("{m}(Case3,{2})"); + + yield return new TestCaseData(unknowLogicalTypeSchema, new object[] + { + new object[] { "John", 23 } + }, codecType).SetName("{m}(Case4,{2})"); } } diff --git a/lang/csharp/src/apache/test/Schema/SchemaTests.cs b/lang/csharp/src/apache/test/Schema/SchemaTests.cs index aca35e1dbd1..b1b69fffc02 100644 --- a/lang/csharp/src/apache/test/Schema/SchemaTests.cs +++ b/lang/csharp/src/apache/test/Schema/SchemaTests.cs @@ -557,9 +557,11 @@ public void TestUnknownLogical(string s, string unknownType) //Assert.AreEqual("Logical type '" + unknownType + "' is not supported.", err.Message); var schema = Schema.Parse(s); + Assert.IsNotNull(schema); // make sure Variable is not null Assert.IsInstanceOf(typeof(LogicalSchema), schema); var logicalSchema = schema as LogicalSchema; + Assert.IsNotNull(logicalSchema); // make sure Variable is not null Assert.IsInstanceOf(typeof(UnknownLogicalType), logicalSchema.LogicalType); Assert.AreEqual(logicalSchema.LogicalTypeName, unknownType); @@ -603,7 +605,7 @@ public void TestUnknownLogicalType(string schemaText) var secondField = ((RecordSchema)schema).Fields.FirstOrDefault(f => f.Name == @"secondField"); Assert.IsNotNull(secondField); - var secondFieldSchema = ((Field)secondField).Schema; + var secondFieldSchema = (secondField).Schema; Assert.IsNotNull(secondFieldSchema); var secondFieldUnionSchema = (UnionSchema)secondFieldSchema; diff --git a/lang/csharp/src/apache/test/Util/UnknownLogicalTypeTests.cs b/lang/csharp/src/apache/test/Util/UnknownLogicalTypeTests.cs new file mode 100644 index 00000000000..539b8df04dd --- /dev/null +++ b/lang/csharp/src/apache/test/Util/UnknownLogicalTypeTests.cs @@ -0,0 +1,188 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.Data.SqlTypes; +using System.Globalization; +using System.Net.Sockets; +using System.Numerics; +using Avro.Util; +using NUnit.Framework; +using NUnit.Framework.Constraints; + +namespace Avro.test.Util +{ + /// + /// This tests added to confirm standalone operation of new type UnknownLogicalType that implements LogicalType + /// + [TestFixture] + class UnknownLogicalTypeTests + { + [TestCase(typeof(System.String), "", "{\"type\": \"string\", \"logicalType\": \"unknown\"}")] + [TestCase(typeof(System.Boolean), true, "{\"type\": \"boolean\", \"logicalType\": \"unknown\"}")] + [TestCase(typeof(System.Int32), Int32.MinValue, "{\"type\": \"int\", \"logicalType\": \"unknown\"}")] + [TestCase(typeof(System.Int64), Int64.MinValue, "{\"type\": \"long\", \"logicalType\": \"unknown\"}")] + [TestCase(typeof(System.Single), Single.MinValue, "{\"type\": \"float\", \"logicalType\": \"unknown\"}")] + [TestCase(typeof(System.Double), Double.MinValue, "{\"type\": \"double\", \"logicalType\": \"unknown\"}")] + [TestCase(typeof(System.Byte[]), new byte[] { }, "{\"type\": \"bytes\", \"logicalType\": \"unknown\"}")] + public void TestConvertToBaseValue_IsTrue(Type baseType, object logicalValue, string schemaText) + { + var schema = (LogicalSchema)Schema.Parse(schemaText); + + var logicalType = new UnknownLogicalType(schema); + + var baseValue = logicalType.ConvertToBaseValue(logicalValue, schema); + + Assert.AreEqual(baseValue, Convert.ChangeType(logicalValue, baseType)); + } + + [TestCase(typeof(System.Byte[]), "", "{\"type\": \"string\", \"logicalType\": \"unknown\"}")] + [TestCase(typeof(System.Double), true, "{\"type\": \"boolean\", \"logicalType\": \"unknown\"}")] + [TestCase(typeof(System.Single), Int32.MinValue, "{\"type\": \"int\", \"logicalType\": \"unknown\"}")] + [TestCase(typeof(System.Boolean), Int64.MinValue, "{\"type\": \"long\", \"logicalType\": \"unknown\"}")] + [TestCase(typeof(System.Int32), Single.MinValue, "{\"type\": \"float\", \"logicalType\": \"unknown\"}")] + [TestCase(typeof(System.Int64), Double.MinValue, "{\"type\": \"double\", \"logicalType\": \"unknown\"}")] + [TestCase(typeof(System.String), new byte[] { }, "{\"type\": \"bytes\", \"logicalType\": \"unknown\"}")] + public void TestConvertToBaseValue_IsFalse(Type baseType, object logicalValue, string schemaText) + { + var schema = (LogicalSchema)Schema.Parse(schemaText); + + var logicalType = new UnknownLogicalType(schema); + + var baseValue = logicalType.ConvertToBaseValue(logicalValue, schema); + + Assert.AreNotEqual(baseValue.GetType(), baseType); + } + + [TestCase(typeof(System.String), "", "{\"type\": \"string\", \"logicalType\": \"unknown\"}")] + [TestCase(typeof(System.Boolean), true, "{\"type\": \"boolean\", \"logicalType\": \"unknown\"}")] + [TestCase(typeof(System.Int32), Int32.MinValue, "{\"type\": \"int\", \"logicalType\": \"unknown\"}")] + [TestCase(typeof(System.Int64), Int64.MinValue, "{\"type\": \"long\", \"logicalType\": \"unknown\"}")] + [TestCase(typeof(System.Single), Single.MinValue, "{\"type\": \"float\", \"logicalType\": \"unknown\"}")] + [TestCase(typeof(System.Double), Double.MinValue, "{\"type\": \"double\", \"logicalType\": \"unknown\"}")] + [TestCase(typeof(System.Byte[]), new byte[] { }, "{\"type\": \"bytes\", \"logicalType\": \"unknown\"}")] + public void TestConvertToLogicalValue_IsTrue(Type baseType, object logicalValue, string schemaText) + { + var schema = (LogicalSchema)Schema.Parse(schemaText); + + var logicalType = new UnknownLogicalType(schema); + + var baseValue = logicalType.ConvertToLogicalValue(logicalValue, schema); + + Assert.AreEqual(baseValue, Convert.ChangeType(logicalValue, baseType)); + } + + [TestCase(typeof(System.Byte[]), "", "{\"type\": \"string\", \"logicalType\": \"unknown\"}")] + [TestCase(typeof(System.Double), true, "{\"type\": \"boolean\", \"logicalType\": \"unknown\"}")] + [TestCase(typeof(System.Single), Int32.MinValue, "{\"type\": \"int\", \"logicalType\": \"unknown\"}")] + [TestCase(typeof(System.Boolean), Int64.MinValue, "{\"type\": \"long\", \"logicalType\": \"unknown\"}")] + [TestCase(typeof(System.Int32), Single.MinValue, "{\"type\": \"float\", \"logicalType\": \"unknown\"}")] + [TestCase(typeof(System.Int64), Double.MinValue, "{\"type\": \"double\", \"logicalType\": \"unknown\"}")] + [TestCase(typeof(System.String), new byte[] { }, "{\"type\": \"bytes\", \"logicalType\": \"unknown\"}")] + public void TestConvertToLogicalValue_IsFalse(Type baseType, object logicalValue, string schemaText) + { + var schema = (LogicalSchema)Schema.Parse(schemaText); + + var logicalType = new UnknownLogicalType(schema); + + var baseValue = logicalType.ConvertToLogicalValue(logicalValue, schema); + + Assert.AreNotEqual(baseValue.GetType(), baseType); + } + + [TestCase(typeof(System.String), false, "{\"type\": \"string\", \"logicalType\": \"unknown\"}")] + [TestCase(typeof(System.Boolean), false, "{\"type\": \"boolean\", \"logicalType\": \"unknown\"}")] + [TestCase(typeof(System.Int32), false, "{\"type\": \"int\", \"logicalType\": \"unknown\"}")] + [TestCase(typeof(System.Int64), false, "{\"type\": \"long\", \"logicalType\": \"unknown\"}")] + [TestCase(typeof(System.Single), false, "{\"type\": \"float\", \"logicalType\": \"unknown\"}")] + [TestCase(typeof(System.Double), false, "{\"type\": \"double\", \"logicalType\": \"unknown\"}")] + [TestCase(typeof(System.Byte[]), false, "{\"type\": \"bytes\", \"logicalType\": \"unknown\"}")] + [TestCase(typeof(System.String), true, "{\"type\": \"string\", \"logicalType\": \"unknown\"}")] + [TestCase(typeof(System.Boolean?), true, "{\"type\": \"boolean\", \"logicalType\": \"unknown\"}")] + [TestCase(typeof(System.Int32?), true, "{\"type\": \"int\", \"logicalType\": \"unknown\"}")] + [TestCase(typeof(System.Int64?), true, "{\"type\": \"long\", \"logicalType\": \"unknown\"}")] + [TestCase(typeof(System.Single?), true, "{\"type\": \"float\", \"logicalType\": \"unknown\"}")] + [TestCase(typeof(System.Double?), true, "{\"type\": \"double\", \"logicalType\": \"unknown\"}")] + [TestCase(typeof(System.Byte?[]), true, "{\"type\": \"bytes\", \"logicalType\": \"unknown\"}")] + public void TestGetCSharpType_IsTrue(Type type, bool isNullable, string schemaText) + { + var schema = (LogicalSchema)Schema.Parse(schemaText); + + var logicalType = new UnknownLogicalType(schema); + + Assert.AreEqual(logicalType.GetCSharpType(isNullable), type); + } + + //[TestCase(typeof(System.String), true, "{\"type\": \"string\", \"logicalType\": \"unknown\"}")] + [TestCase(typeof(System.Boolean), true, "{\"type\": \"boolean\", \"logicalType\": \"unknown\"}")] + [TestCase(typeof(System.Int32), true, "{\"type\": \"int\", \"logicalType\": \"unknown\"}")] + [TestCase(typeof(System.Int64), true, "{\"type\": \"long\", \"logicalType\": \"unknown\"}")] + [TestCase(typeof(System.Single), true, "{\"type\": \"float\", \"logicalType\": \"unknown\"}")] + [TestCase(typeof(System.Double), true, "{\"type\": \"double\", \"logicalType\": \"unknown\"}")] + [TestCase(typeof(System.Byte[]), true, "{\"type\": \"bytes\", \"logicalType\": \"unknown\"}")] + //[TestCase(typeof(System.String), false, "{\"type\": \"string\", \"logicalType\": \"unknown\"}")] + [TestCase(typeof(System.Boolean?), false, "{\"type\": \"boolean\", \"logicalType\": \"unknown\"}")] + [TestCase(typeof(System.Int32?), false, "{\"type\": \"int\", \"logicalType\": \"unknown\"}")] + [TestCase(typeof(System.Int64?), false, "{\"type\": \"long\", \"logicalType\": \"unknown\"}")] + [TestCase(typeof(System.Single?), false, "{\"type\": \"float\", \"logicalType\": \"unknown\"}")] + [TestCase(typeof(System.Double?), false, "{\"type\": \"double\", \"logicalType\": \"unknown\"}")] + [TestCase(typeof(System.Byte?[]), false, "{\"type\": \"bytes\", \"logicalType\": \"unknown\"}")] + public void TestGetCSharpType_IsFalse(Type type, bool isNullable, string schemaText) + { + var schema = (LogicalSchema)Schema.Parse(schemaText); + + var logicalType = new UnknownLogicalType(schema); + + Assert.AreNotEqual(logicalType.GetCSharpType(isNullable), type); + } + + [TestCase("", "{\"type\": \"string\", \"logicalType\": \"unknown\"}")] + [TestCase(true, "{\"type\": \"boolean\", \"logicalType\": \"unknown\"}")] + [TestCase(Int32.MinValue, "{\"type\": \"int\", \"logicalType\": \"unknown\"}")] + [TestCase(Int64.MinValue, "{\"type\": \"long\", \"logicalType\": \"unknown\"}")] + [TestCase(Single.MinValue, "{\"type\": \"float\", \"logicalType\": \"unknown\"}")] + [TestCase(Double.MinValue, "{\"type\": \"double\", \"logicalType\": \"unknown\"}")] + [TestCase(new byte[] { } , "{\"type\": \"bytes\", \"logicalType\": \"unknown\"}")] + public void TestIsInstanceOfLogicalType_IsTrue(object logicalValue, string schemaText) + { + var schema = (LogicalSchema)Schema.Parse(schemaText); + + var logicalType = new UnknownLogicalType(schema); + + Assert.IsTrue(logicalType.IsInstanceOfLogicalType(logicalValue)); + } + + [TestCase(Int32.MinValue, "{\"type\": \"string\", \"logicalType\": \"unknown\"}")] + [TestCase(new byte[] { }, "{\"type\": \"boolean\", \"logicalType\": \"unknown\"}")] + [TestCase(Int64.MinValue, "{\"type\": \"int\", \"logicalType\": \"unknown\"}")] + [TestCase(Single.MinValue, "{\"type\": \"long\", \"logicalType\": \"unknown\"}")] + [TestCase(Double.MinValue, "{\"type\": \"float\", \"logicalType\": \"unknown\"}")] + [TestCase(new byte[] { }, "{\"type\": \"double\", \"logicalType\": \"unknown\"}")] + [TestCase("", "{\"type\": \"bytes\", \"logicalType\": \"unknown\"}")] + public void TestIsInstanceOfLogicalType_IsFalse(object logicalValue, string schemaText) + { + var schema = (LogicalSchema)Schema.Parse(schemaText); + + var logicalType = new UnknownLogicalType(schema); + + Assert.IsFalse(logicalType.IsInstanceOfLogicalType(logicalValue)); + } + + // See also a new test in Avro.Tests.File in TestSpecificDataSource using unknowLogicalTypeSchema + } +} From 235d329118764005c2ace77bc7b26e472bcd5e14 Mon Sep 17 00:00:00 2001 From: Tom Bruns Date: Wed, 21 Feb 2024 09:02:10 -0500 Subject: [PATCH 4/7] uncomment testcase after testing --- lang/csharp/src/apache/test/AvroGen/AvroGenSchemaTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lang/csharp/src/apache/test/AvroGen/AvroGenSchemaTests.cs b/lang/csharp/src/apache/test/AvroGen/AvroGenSchemaTests.cs index d940a1bef83..be22d55aaf3 100644 --- a/lang/csharp/src/apache/test/AvroGen/AvroGenSchemaTests.cs +++ b/lang/csharp/src/apache/test/AvroGen/AvroGenSchemaTests.cs @@ -606,7 +606,7 @@ public void GenerateSchemaWithNamespaceMapping( AvroGenHelper.TestSchema(schema, typeNamesToCheck, new Dictionary { { namespaceMappingFrom, namespaceMappingTo } }, generatedFilesToCheck); } - //[TestCase(_logicalTypesWithCustomConversion, typeof(AvroTypeException), 0)] + [TestCase(_logicalTypesWithCustomConversion, typeof(AvroTypeException), 0)] [TestCase(_customConversionWithLogicalTypes, typeof(SchemaParseException), 1)] public void NotSupportedSchema(string schema, Type expectedException, int expectedResult) { From f7b4872a347938d090193a55beba3ff2a31bcefa Mon Sep 17 00:00:00 2001 From: Tom Bruns Date: Wed, 21 Feb 2024 10:16:24 -0500 Subject: [PATCH 5/7] savepoint remove local setting files --- .../src/apache/codegen/Properties/launchSettings.json | 8 -------- .../csharp/src/apache/main/Properties/launchSettings.json | 8 -------- 2 files changed, 16 deletions(-) delete mode 100644 lang/csharp/src/apache/codegen/Properties/launchSettings.json delete mode 100644 lang/csharp/src/apache/main/Properties/launchSettings.json diff --git a/lang/csharp/src/apache/codegen/Properties/launchSettings.json b/lang/csharp/src/apache/codegen/Properties/launchSettings.json deleted file mode 100644 index a54467aa70a..00000000000 --- a/lang/csharp/src/apache/codegen/Properties/launchSettings.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "profiles": { - "Avro.codegen": { - "commandName": "Project", - "commandLineArgs": " -s C:\\Users\\Thomas.Bruns\\source\\repos\\AzureDevOps\\TQL.Kafka.Samples\\TQL.Kafka.Samples\\TQL.Kafka.Samples.Messages\\v1\\cdc_tql_dbo_tbldrops.avsc .\\tab --namespace NO_SCHEMA_NAMESPACE:TQL.DEMO" - } - } -} \ No newline at end of file diff --git a/lang/csharp/src/apache/main/Properties/launchSettings.json b/lang/csharp/src/apache/main/Properties/launchSettings.json deleted file mode 100644 index 277fa29573e..00000000000 --- a/lang/csharp/src/apache/main/Properties/launchSettings.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "profiles": { - "Avro.main": { - "commandName": "Project", - "commandLineArgs": "C:\\Users\\Thomas.Bruns\\source\\repos\\AzureDevOps\\TQL.Kafka.Samples\\TQL.Kafka.Samples\\TQL.Kafka.Samples.Messages\\v2\\cdc_customers_dbo_tblcustomers.avsc" - } - } -} \ No newline at end of file From 60463b5da7c885ce4e90ff9cf2d81c7682d59b0b Mon Sep 17 00:00:00 2001 From: Tom Bruns Date: Thu, 22 Feb 2024 07:16:53 -0500 Subject: [PATCH 6/7] backout chags for a different pr, adjust unit test --- lang/csharp/src/apache/codegen/Avro.codegen.csproj | 4 ---- lang/csharp/src/apache/codegen/AvroGen.cs | 12 +----------- lang/csharp/src/apache/main/CodeGen/CodeGen.cs | 13 ++----------- .../src/apache/test/AvroGen/AvroGenSchemaTests.cs | 8 ++++---- 4 files changed, 7 insertions(+), 30 deletions(-) diff --git a/lang/csharp/src/apache/codegen/Avro.codegen.csproj b/lang/csharp/src/apache/codegen/Avro.codegen.csproj index d37dc60514b..94aa8123119 100644 --- a/lang/csharp/src/apache/codegen/Avro.codegen.csproj +++ b/lang/csharp/src/apache/codegen/Avro.codegen.csproj @@ -62,10 +62,6 @@ - - - - diff --git a/lang/csharp/src/apache/codegen/AvroGen.cs b/lang/csharp/src/apache/codegen/AvroGen.cs index f0dc2ce2557..3b07ca59fdc 100644 --- a/lang/csharp/src/apache/codegen/AvroGen.cs +++ b/lang/csharp/src/apache/codegen/AvroGen.cs @@ -155,10 +155,6 @@ static void Usage() " --namespace Map an Avro schema/protocol namespace to a C# namespace.\n" + " The format is \"my.avro.namespace:my.csharp.namespace\".\n" + " May be specified multiple times to map multiple namespaces.\n" + - " If the source schema is missing a top level namespace\n" + - $" the namespace [{CodeGen.NO_SCHEMA_NAMESPACE}] will be injected.\n" + - " You can replace this using the --namespace command line option\n" + - " or edit the generated class file afterwards.\n" + " --skip-directories Skip creation of namespace directories. It will generate classes right inside output directory\n", AppDomain.CurrentDomain.FriendlyName); } @@ -174,9 +170,6 @@ public static int GenProtocol(string infile, string outdir, codegen.AddProtocol(text, namespaceMapping); codegen.GenerateCode(); - - bool replaceTempNamespace = namespaceMapping.Any(n => n.Key == CodeGen.NO_SCHEMA_NAMESPACE); - codegen.WriteTypes(outdir); } catch (Exception ex) @@ -198,10 +191,7 @@ public static int GenSchema(string infile, string outdir, codegen.AddSchema(text, namespaceMapping); codegen.GenerateCode(); - - var overrideClassNamepace = namespaceMapping.FirstOrDefault(n => n.Key.ToLower() == CodeGen.NO_SCHEMA_NAMESPACE.ToLower()).Value; - - codegen.WriteTypes(outdir, skipDirectories, overrideClassNamepace); + codegen.WriteTypes(outdir, skipDirectories); } catch (Exception ex) { diff --git a/lang/csharp/src/apache/main/CodeGen/CodeGen.cs b/lang/csharp/src/apache/main/CodeGen/CodeGen.cs index 15199d2f66d..7e793627201 100644 --- a/lang/csharp/src/apache/main/CodeGen/CodeGen.cs +++ b/lang/csharp/src/apache/main/CodeGen/CodeGen.cs @@ -34,8 +34,6 @@ namespace Avro /// public class CodeGen { - public static string NO_SCHEMA_NAMESPACE = @"NO_SCHEMA_NAMESPACE"; - /// /// Gets object that contains all the generated types. /// @@ -897,9 +895,7 @@ protected virtual CodeTypeDeclaration processRecord(Schema schema) string nspace = recordSchema.Namespace; if (string.IsNullOrEmpty(nspace)) { - // if the schema does not have a namespace defined, insert a temp value so code gen will not fail - //throw new CodeGenException("Namespace required for record schema " + recordSchema.Name); - nspace = NO_SCHEMA_NAMESPACE; + throw new CodeGenException("Namespace required for record schema " + recordSchema.Name); } CodeNamespace codens = AddNamespace(nspace); @@ -1189,7 +1185,7 @@ public virtual IDictionary GetTypes() /// /// name of directory to write to. /// skip creation of directories based on schema namespace - public virtual void WriteTypes(string outputdir, bool skipDirectories = false, string overrideClassNamepace = null) + public virtual void WriteTypes(string outputdir, bool skipDirectories = false) { var cscp = new CSharpCodeProvider(); @@ -1213,11 +1209,6 @@ public virtual void WriteTypes(string outputdir, bool skipDirectories = false, s } Directory.CreateDirectory(dir); - if(ns.Name == CodeGen.NO_SCHEMA_NAMESPACE && !string.IsNullOrEmpty(overrideClassNamepace)) - { - ns.Name = overrideClassNamepace; - } - var new_ns = new CodeNamespace(ns.Name); new_ns.Comments.Add(CodeGenUtil.Instance.FileComment); foreach (CodeNamespaceImport nci in CodeGenUtil.Instance.NamespaceImports) diff --git a/lang/csharp/src/apache/test/AvroGen/AvroGenSchemaTests.cs b/lang/csharp/src/apache/test/AvroGen/AvroGenSchemaTests.cs index b9b2e74b01e..4db445ea53a 100644 --- a/lang/csharp/src/apache/test/AvroGen/AvroGenSchemaTests.cs +++ b/lang/csharp/src/apache/test/AvroGen/AvroGenSchemaTests.cs @@ -606,9 +606,9 @@ public void GenerateSchemaWithNamespaceMapping( AvroGenHelper.TestSchema(schema, typeNamesToCheck, new Dictionary { { namespaceMappingFrom, namespaceMappingTo } }, generatedFilesToCheck); } - [TestCase(_logicalTypesWithCustomConversion, typeof(AvroTypeException), 0)] - [TestCase(_customConversionWithLogicalTypes, typeof(SchemaParseException), 1)] - public void NotSupportedSchema(string schema, Type expectedException, int expectedResult) + [TestCase(_logicalTypesWithCustomConversion, typeof(AvroTypeException))] + [TestCase(_customConversionWithLogicalTypes, typeof(SchemaParseException))] + public void NotSupportedSchema(string schema, Type expectedException) { // Create temp folder string outputDir = AvroGenHelper.CreateEmptyTemporaryFolder(out string uniqueId); @@ -621,7 +621,7 @@ public void NotSupportedSchema(string schema, Type expectedException, int expect // We now support unknown logical types //Assert.That(AvroGenTool.GenSchema(schemaFileName, outputDir, new Dictionary(), false), Is.EqualTo(1)); - Assert.That(AvroGenTool.GenSchema(schemaFileName, outputDir, new Dictionary(), false), Is.EqualTo(expectedResult)); + Assert.That(AvroGenTool.GenSchema(schemaFileName, outputDir, new Dictionary(), false), Is.EqualTo(0)); } finally From 42b0fbd0a665cf409b85e329151f5b5d3e2a07e8 Mon Sep 17 00:00:00 2001 From: Tom Bruns Date: Thu, 22 Feb 2024 08:23:30 -0500 Subject: [PATCH 7/7] changes from code review --- .../src/apache/test/AvroGen/AvroGenSchemaTests.cs | 10 ++++------ lang/csharp/src/apache/test/Schema/SchemaTests.cs | 3 --- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/lang/csharp/src/apache/test/AvroGen/AvroGenSchemaTests.cs b/lang/csharp/src/apache/test/AvroGen/AvroGenSchemaTests.cs index 4db445ea53a..cc0dccdef78 100644 --- a/lang/csharp/src/apache/test/AvroGen/AvroGenSchemaTests.cs +++ b/lang/csharp/src/apache/test/AvroGen/AvroGenSchemaTests.cs @@ -606,9 +606,9 @@ public void GenerateSchemaWithNamespaceMapping( AvroGenHelper.TestSchema(schema, typeNamesToCheck, new Dictionary { { namespaceMappingFrom, namespaceMappingTo } }, generatedFilesToCheck); } - [TestCase(_logicalTypesWithCustomConversion, typeof(AvroTypeException))] - [TestCase(_customConversionWithLogicalTypes, typeof(SchemaParseException))] - public void NotSupportedSchema(string schema, Type expectedException) + [TestCase(_logicalTypesWithCustomConversion, typeof(AvroTypeException), 0)] + [TestCase(_customConversionWithLogicalTypes, typeof(SchemaParseException), 1)] + public void NotSupportedSchema(string schema, Type expectedException, int expectedResult) { // Create temp folder string outputDir = AvroGenHelper.CreateEmptyTemporaryFolder(out string uniqueId); @@ -619,9 +619,7 @@ public void NotSupportedSchema(string schema, Type expectedException) string schemaFileName = Path.Combine(outputDir, $"{uniqueId}.avsc"); System.IO.File.WriteAllText(schemaFileName, schema); - // We now support unknown logical types - //Assert.That(AvroGenTool.GenSchema(schemaFileName, outputDir, new Dictionary(), false), Is.EqualTo(1)); - Assert.That(AvroGenTool.GenSchema(schemaFileName, outputDir, new Dictionary(), false), Is.EqualTo(0)); + Assert.That(AvroGenTool.GenSchema(schemaFileName, outputDir, new Dictionary(), false), Is.EqualTo(expectedResult)); } finally diff --git a/lang/csharp/src/apache/test/Schema/SchemaTests.cs b/lang/csharp/src/apache/test/Schema/SchemaTests.cs index b1b69fffc02..4cc4e1d0158 100644 --- a/lang/csharp/src/apache/test/Schema/SchemaTests.cs +++ b/lang/csharp/src/apache/test/Schema/SchemaTests.cs @@ -553,9 +553,6 @@ public void TestLogicalPrimitive(string s, string baseType, string logicalType) [TestCase("{\"type\": \"int\", \"logicalType\": \"unknown\"}", "unknown")] public void TestUnknownLogical(string s, string unknownType) { - //var err = Assert.Throws(() => Schema.Parse(s)); - //Assert.AreEqual("Logical type '" + unknownType + "' is not supported.", err.Message); - var schema = Schema.Parse(s); Assert.IsNotNull(schema); // make sure Variable is not null Assert.IsInstanceOf(typeof(LogicalSchema), schema);