This is an automated email from the ASF dual-hosted git repository.
blachniet pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/avro.git
The following commit(s) were added to refs/heads/master by this push:
new 0fe596a AVRO-2499: Cache classes referenced in a union for C# reflect
API (#652)
0fe596a is described below
commit 0fe596a123d03e3a2ada9d34a79685bc9d4be9de
Author: paddypawprints <[email protected]>
AuthorDate: Sun Sep 29 05:32:05 2019 -0700
AVRO-2499: Cache classes referenced in a union for C# reflect API (#652)
---
lang/csharp/src/apache/main/Reflect/ClassCache.cs | 28 ++-
lang/csharp/src/apache/main/Reflect/README.md | 126 ++++++++++++-
.../src/apache/test/Reflect/TestFromAvroProject.cs | 13 ++
lang/csharp/src/apache/test/Reflect/TestReflect.cs | 7 +-
lang/csharp/src/apache/test/Reflect/TestUnion.cs | 198 +++++++++++++++++++++
5 files changed, 361 insertions(+), 11 deletions(-)
diff --git a/lang/csharp/src/apache/main/Reflect/ClassCache.cs
b/lang/csharp/src/apache/main/Reflect/ClassCache.cs
index 27ac8de..9237e18 100644
--- a/lang/csharp/src/apache/main/Reflect/ClassCache.cs
+++ b/lang/csharp/src/apache/main/Reflect/ClassCache.cs
@@ -1,4 +1,4 @@
-/*
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -253,6 +253,32 @@ namespace Avro.Reflect
case NamedSchema ns:
EnumCache.AddEnumNameMapItem(ns, objType);
break;
+ case UnionSchema us:
+ if (us.Schemas.Count == 2 && (us.Schemas[0].Tag ==
Schema.Type.Null || us.Schemas[1].Tag == Schema.Type.Null) && objType.IsClass)
+ {
+ // in this case objType will match the non null type
in the union
+ foreach (var o in us.Schemas)
+ {
+ if (o.Tag != Schema.Type.Null)
+ {
+ LoadClassCache(objType, o);
+ }
+ }
+
+ }
+ else
+ {
+ // check the schema types are registered
+ foreach (var o in us.Schemas)
+ {
+ if (o.Tag == Schema.Type.Record && GetClass(o as
RecordSchema) == null)
+ {
+ throw new AvroException($"Class for union
record type {o.Fullname} is not registered. Create a ClassCache object and call
LoadClassCache");
+ }
+ }
+ }
+
+ break;
}
}
}
diff --git a/lang/csharp/src/apache/main/Reflect/README.md
b/lang/csharp/src/apache/main/Reflect/README.md
index dfb4836..3573c6a 100644
--- a/lang/csharp/src/apache/main/Reflect/README.md
+++ b/lang/csharp/src/apache/main/Reflect/README.md
@@ -4,7 +4,7 @@ This namespace contains classes that implement Avro
serialization and deserializ
## Serialization
-The approach starts with the schema and interates both the schema and the
dotnet object together in a depth first manner per the specification.
Serialization is the same as the Generic serializer except where the serializer
encounters:
+The approach starts with the schema and iterates both the schema and the
dotnet type together in a depth first manner per the specification.
Serialization is the same as the Generic serializer except where the serializer
encounters:
- *A fixed type*: if the corresponding dotnet object type is a byte[] of the
correct length then the object is serialized, otherwise an exception is thrown.
- *A record type*: the serializer matches the schema property name to the
dotnet object property name and then reursively serializes the schema property
and the dotnet object property
- *An array type*: See array serialization/deserialization.
@@ -35,13 +35,13 @@ public Func<Type, object> RecordFactory {get;set;}
```
You might want to do this if your class contains interfaces and/or if you use
an IoC container.
-See the section on Arrays. The ArrayHelper specifies the type of object
created when an array is deserialized. The default is List<T>.
+See the section on Arrays. The ArrayHelper specifies the type of object
created when an array is deserialized. The default is List\<T>.
-The type created for Map objects is specified by the Deserializer property
MapType. *This must be a two (or more) parameter generic type where the first
type paramater is string and the second is undefined* e.g. List<string,>.
+The type created for Map objects is specified by the Deserializer property
MapType. *This must be a two (or more) parameter generic type where the first
type paramater is string and the second is undefined* e.g. Dictionary<string,>.
```csharp
public Type MapType { get; set; }
```
-By default the MapType is List<string,>
+By default the MapType is Dictionary<string,>
```
Basic deserialization is performed as in the following example:
@@ -196,3 +196,121 @@ _Example_: ConcurrentQueue
var reader = new
ReflectReader<ConcurrentQueue<ConcurrentQueueRec>>(schema, schema, cache);
```
+## Unions
+
+All union constructs are supported however record types that are first defined
in unions may need manual type registration.
+
+### Automatic Type Registration
+
+Types associated with unions of this form can be automatically registered and
no special handling is needed.
+
+```json
+ ["null", { "type": "record", "name": "X"}]
+```
+
+_Example_:
+
+```csharp
+ public class MyClass
+ {
+ public string A { get; set; }
+ public double C { get; set; }
+ }
+
+ // ...
+
+ var nullableSchema = @"
+ [
+ ""null"",
+ { ""type"" : ""record"", ""name"" : ""Dervied2"", ""fields"" :
+ [
+ { ""name"" : ""A"", ""type"" : ""string""},
+ { ""name"" : ""C"", ""type"" : ""double""}
+ ]
+ },
+
+ ]
+ ";
+ var schema = Schema.Parse(nullableSchema);
+ var derived2write = new MyClass() { A = "derived2", C = 3.14 };
+
+ var writer = new ReflectWriter<MyClass>(schema);
+ var reader = new ReflectReader<MyClass>(schema, schema);
+
+ // etc.
+```
+
+### Manual Registration
+
+Where a record type is defined inside a union and the union does not
+follow the "nullable construct" above, the CSharp type and schema need to be
manually registered. Registration is done using the ClassCache method
LoadClassCache.
+
+```csharp
+ cache.LoadClassCache(typeof(MyClass), recordSchema);
+```
+
+Note that the `recordSchema` used here is the schema corresponding to the
`MyClass` type within the overall union schema. See the example below.
+
+```csharp
+ public class BaseClass
+ {
+ public string A { get; set; }
+ }
+
+ public class Derived1 : BaseClass
+ {
+ public int B { get; set; }
+ }
+
+ public class Derived2 : BaseClass
+ {
+ public double C { get; set; }
+ }
+
+ public void SerializeExample()
+ {
+ var baseClassSchema = @"
+ [
+ { ""type"" : ""record"", ""name"" : ""Dervied1"", ""fields"" :
+ [
+ { ""name"" : ""A"", ""type"" : ""string""},
+ { ""name"" : ""B"", ""type"" : ""int""}
+ ]
+ },
+ { ""type"" : ""record"", ""name"" : ""Dervied2"", ""fields"" :
+ [
+ { ""name"" : ""A"", ""type"" : ""string""},
+ { ""name"" : ""C"", ""type"" : ""double""}
+ ]
+ },
+
+ ]
+ ";
+
+ var schema = Schema.Parse(baseClassSchema);
+ var derived1write = new Derived1() { A = "derived1", B = 7 };
+ var derived2write = new Derived2() { A = "derived2", C = 3.14 };
+
+ // union types (except for [null, type]) need to be manually
registered
+ var unionSchema = schema as UnionSchema;
+ var cache = new ClassCache();
+ cache.LoadClassCache(typeof(Derived1), unionSchema[0]);
+ cache.LoadClassCache(typeof(Derived2), unionSchema[1]);
+ var x = schema as RecordSchema;
+
+ var writer = new ReflectWriter<BaseClass>(schema, cache);
+ var reader = new ReflectReader<BaseClass>(schema, schema, cache);
+
+ using (var stream = new MemoryStream(256))
+ {
+ var encoder = new BinaryEncoder(stream);
+ writer.Write(derived1write, encoder);
+ writer.Write(derived2write, encoder);
+ stream.Seek(0, SeekOrigin.Begin);
+
+ var decoder = new BinaryDecoder(stream);
+ var derived1read = (Derived1)reader.Read(decoder);
+ var derived2read = (Derived2)reader.Read(decoder);
+ }
+ }
+```
diff --git a/lang/csharp/src/apache/test/Reflect/TestFromAvroProject.cs
b/lang/csharp/src/apache/test/Reflect/TestFromAvroProject.cs
index dff2a69..dc17bcc 100644
--- a/lang/csharp/src/apache/test/Reflect/TestFromAvroProject.cs
+++ b/lang/csharp/src/apache/test/Reflect/TestFromAvroProject.cs
@@ -79,6 +79,8 @@ namespace Avro.Test
public A myA { get; set; }
+ public A myNullableA { get; set; }
+
public MyEnum myE { get; set; }
public List<byte[]> myArray { get; set; }
@@ -138,6 +140,7 @@ namespace Avro.Test
{ ""name"" : ""myNull"", ""type"" : ""null"" },
{ ""name"" : ""myFixed"", ""type"" : ""MyFixed"" },
{ ""name"" : ""myA"", ""type"" : ""A"" },
+ { ""name"" : ""myNullableA"", ""type"" : [ ""null"", ""A""
] },
{ ""name"" : ""myE"", ""type"" : ""MyEnum"" },
{ ""name"" : ""myArray"", ""type"" : { ""type"" :
""array"", ""items"" : ""bytes"" } },
{ ""name"" : ""myArray2"", ""type"" : { ""type"" :
""array"", ""items"" : { ""type"" : ""record"", ""name"" : ""newRec"",
""fields"" : [ { ""name"" : ""f1"", ""type"" : ""long""} ] } } },
@@ -264,6 +267,15 @@ namespace Avro.Test
Assert.IsNotNull(zz.myA);
Assert.AreEqual(z.myA.f1, zz.myA.f1);
}
+ if (z.myNullableA == null)
+ {
+ Assert.IsNull(zz.myNullableA);
+ }
+ else
+ {
+ Assert.IsNotNull(zz.myNullableA);
+ Assert.AreEqual(z.myNullableA.f1, zz.myNullableA.f1);
+ }
Assert.AreEqual(z.myE, zz.myE);
if (z.myArray == null)
{
@@ -347,6 +359,7 @@ namespace Avro.Test
myNull = null,
myFixed = new byte[16] { 0x01, 0x02, 0x03, 0x04, 0x01, 0x02,
0x03, 0x04, 0x01, 0x02, 0x03, 0x04, 0x01, 0x02, 0x03, 0x04 },
myA = new A() { f1 = 3L },
+ myNullableA = new A() { f1 = 4L },
myE = MyEnum.B,
myArray = new List<byte[]>() { new byte[] { 0x01, 0x02, 0x03,
0x04 } },
myArray2 = new List<newRec>() { new newRec() { f1 = 4L } },
diff --git a/lang/csharp/src/apache/test/Reflect/TestReflect.cs
b/lang/csharp/src/apache/test/Reflect/TestReflect.cs
index 67d9b3e..bea5ef2 100644
--- a/lang/csharp/src/apache/test/Reflect/TestReflect.cs
+++ b/lang/csharp/src/apache/test/Reflect/TestReflect.cs
@@ -15,17 +15,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-using System;
+
using System.Collections;
using System.IO;
-using System.Linq;
using NUnit.Framework;
using Avro.IO;
-using System.CodeDom;
-using System.CodeDom.Compiler;
-using Avro;
using Avro.Reflect;
-using System.Reflection;
namespace Avro.Test
{
diff --git a/lang/csharp/src/apache/test/Reflect/TestUnion.cs
b/lang/csharp/src/apache/test/Reflect/TestUnion.cs
new file mode 100644
index 0000000..db0d4a9
--- /dev/null
+++ b/lang/csharp/src/apache/test/Reflect/TestUnion.cs
@@ -0,0 +1,198 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using System;
+using System.Collections.Generic;
+using System.Collections.Concurrent;
+using System.IO;
+using Avro.IO;
+using Avro.Reflect;
+using NUnit.Framework;
+using System.Collections;
+
+namespace Avro.Test
+{
+ [TestFixture]
+ public class TestUnion
+ {
+ public const string BaseClassSchema = @"
+ [
+ { ""type"" : ""record"", ""name"" : ""Dervied1"", ""fields"" :
+ [
+ { ""name"" : ""A"", ""type"" : ""string""},
+ { ""name"" : ""B"", ""type"" : ""int""}
+ ]
+ },
+ { ""type"" : ""record"", ""name"" : ""Dervied2"", ""fields"" :
+ [
+ { ""name"" : ""A"", ""type"" : ""string""},
+ { ""name"" : ""C"", ""type"" : ""double""}
+ ]
+ },
+
+ ]
+ ";
+
+ public class BaseClass
+ {
+ public string A { get; set; }
+ }
+
+ public class Derived1 : BaseClass
+ {
+ public int B { get; set; }
+ }
+
+ public class Derived2 : BaseClass
+ {
+ public double C { get; set; }
+ }
+
+ /// <summary>
+ /// Test with a union that represents derived classes.
+ /// </summary>
+ [TestCase]
+ public void BaseClassTest()
+ {
+ var schema = Schema.Parse(BaseClassSchema);
+ var derived1write = new Derived1() { A = "derived1", B = 7 };
+ var derived2write = new Derived2() { A = "derived2", C = 3.14 };
+
+ // union types (except for [null, type]) need to be manually
registered
+ var unionSchema = schema as UnionSchema;
+ var cache = new ClassCache();
+ cache.LoadClassCache(typeof(Derived1), unionSchema[0]);
+ cache.LoadClassCache(typeof(Derived2), unionSchema[1]);
+ var x = schema as RecordSchema;
+
+ var writer = new ReflectWriter<BaseClass>(schema, cache);
+ var reader = new ReflectReader<BaseClass>(schema, schema, cache);
+
+ using (var stream = new MemoryStream(256))
+ {
+ var encoder = new BinaryEncoder(stream);
+ writer.Write(derived1write, encoder);
+ writer.Write(derived2write, encoder);
+ stream.Seek(0, SeekOrigin.Begin);
+
+ var decoder = new BinaryDecoder(stream);
+ var derived1read = (Derived1)reader.Read(decoder);
+ var derived2read = (Derived2)reader.Read(decoder);
+ Assert.AreEqual(derived1read.A, derived1write.A);
+ Assert.AreEqual(derived1read.B, derived1write.B);
+ Assert.AreEqual(derived2read.A, derived2write.A);
+ Assert.AreEqual(derived2read.C, derived2write.C);
+ }
+ }
+
+ /// <summary>
+ /// If you fail to manually register types within a union that has
more than one non-null
+ /// schema, creating a <see cref="ReflectWriter{T}"/> throws an
exception.
+ /// </summary>
+ [TestCase]
+ public void ThrowsIfClassesNotLoadedTest()
+ {
+ var schema = Schema.Parse(BaseClassSchema);
+ var cache = new ClassCache();
+ Assert.Throws<AvroException>(() => new
ReflectWriter<BaseClass>(schema, cache));
+ }
+
+ [TestCase]
+ public void NullableTest()
+ {
+ var nullableSchema = @"
+ [
+ ""null"",
+ { ""type"" : ""record"", ""name"" : ""Dervied2"", ""fields"" :
+ [
+ { ""name"" : ""A"", ""type"" : ""string""},
+ { ""name"" : ""C"", ""type"" : ""double""}
+ ]
+ },
+
+ ]
+ ";
+ var schema = Schema.Parse(nullableSchema);
+ var derived2write = new Derived2() { A = "derived2", C = 3.14 };
+
+ var writer = new ReflectWriter<Derived2>(schema);
+ var reader = new ReflectReader<Derived2>(schema, schema);
+
+ using (var stream = new MemoryStream(256))
+ {
+ var encoder = new BinaryEncoder(stream);
+ writer.Write(derived2write, encoder);
+ writer.Write(null, encoder);
+ stream.Seek(0, SeekOrigin.Begin);
+
+ var decoder = new BinaryDecoder(stream);
+ var derived2read = reader.Read(decoder);
+ var derived2null = reader.Read(decoder);
+ Assert.AreEqual(derived2read.A, derived2write.A);
+ Assert.AreEqual(derived2read.C, derived2write.C);
+ Assert.IsNull(derived2null);
+ }
+ }
+
+ [TestCase]
+ public void HeterogeneousTest()
+ {
+ var heterogeneousSchema = @"
+ [
+ ""string"",
+ ""null"",
+ { ""type"" : ""record"", ""name"" : ""Dervied2"", ""fields"" :
+ [
+ { ""name"" : ""A"", ""type"" : ""string""},
+ { ""name"" : ""C"", ""type"" : ""double""}
+ ]
+ },
+
+ ]
+ ";
+ var schema = Schema.Parse(heterogeneousSchema);
+ var derived2write = new Derived2() { A = "derived2", C = 3.14 };
+
+ // union types (except for [null, type]) need to be manually
registered
+ var unionSchema = schema as UnionSchema;
+ var cache = new ClassCache();
+ cache.LoadClassCache(typeof(Derived2), unionSchema[2]);
+
+ var writer = new ReflectWriter<object>(schema, cache);
+ var reader = new ReflectReader<object>(schema, schema, cache);
+
+ using (var stream = new MemoryStream(256))
+ {
+ var encoder = new BinaryEncoder(stream);
+ writer.Write(derived2write, encoder);
+ writer.Write("string value", encoder);
+ writer.Write(null, encoder);
+ stream.Seek(0, SeekOrigin.Begin);
+
+ var decoder = new BinaryDecoder(stream);
+ var derived2read = (Derived2)reader.Read(decoder);
+ var stringRead = (string)reader.Read(decoder);
+ var nullRead = reader.Read(decoder);
+ Assert.AreEqual(derived2read.A, derived2write.A);
+ Assert.AreEqual(derived2read.C, derived2write.C);
+ Assert.AreEqual(stringRead, "string value");
+ Assert.IsNull(nullRead);
+ }
+ }
+ }
+}
