adamreeve commented on code in PR #34133: URL: https://github.com/apache/arrow/pull/34133#discussion_r1119262146
########## csharp/src/Apache.Arrow/C/CArrowSchema.cs: ########## @@ -0,0 +1,339 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Runtime.InteropServices; +using Apache.Arrow.Types; + +[UnmanagedFunctionPointer(CallingConvention.StdCall)] +public delegate void ReleaseCArrowSchema(IntPtr schema); + +namespace Apache.Arrow.C +{ + /// <summary> + /// An Arrow C Data Interface Schema, which represents a type, field, or schema. + /// </summary> + /// + /// <remarks> + /// This is used to export <see cref="ArrowType"/>, <see cref="Field"/>, or + /// <see cref="Schema"/> to other languages. It matches the layout of the + /// ArrowSchema struct described in https://github.com/apache/arrow/blob/main/cpp/src/arrow/c/abi.h. + /// </remarks> + [StructLayout(LayoutKind.Sequential)] + unsafe public struct CArrowSchema + { + [MarshalAs(UnmanagedType.LPStr)] Review Comment: Should format and name be `UnmanagedType.LPUTF8Str`? ########## csharp/src/Apache.Arrow/C/CArrowSchema.cs: ########## @@ -0,0 +1,339 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Runtime.InteropServices; +using Apache.Arrow.Types; + +[UnmanagedFunctionPointer(CallingConvention.StdCall)] +public delegate void ReleaseCArrowSchema(IntPtr schema); + +namespace Apache.Arrow.C +{ + /// <summary> + /// An Arrow C Data Interface Schema, which represents a type, field, or schema. + /// </summary> + /// + /// <remarks> + /// This is used to export <see cref="ArrowType"/>, <see cref="Field"/>, or + /// <see cref="Schema"/> to other languages. It matches the layout of the + /// ArrowSchema struct described in https://github.com/apache/arrow/blob/main/cpp/src/arrow/c/abi.h. + /// </remarks> + [StructLayout(LayoutKind.Sequential)] + unsafe public struct CArrowSchema + { + [MarshalAs(UnmanagedType.LPStr)] + public string format; + [MarshalAs(UnmanagedType.LPStr)] + public string name; + [MarshalAs(UnmanagedType.LPStr)] + public string metadata; + public long flags; + public long n_children; + public IntPtr* children; + public IntPtr dictionary; + [MarshalAs(UnmanagedType.FunctionPtr)] + + public ReleaseCArrowSchema release; + // Check this out: https://github.com/G-Research/ParquetSharp/blob/386d91bd5e6fe6cb81583803447023c1359957c8/csharp/ParquetHandle.cs#L8 + public IntPtr private_data; + + private static string GetFormat(IArrowType datatype) + { + TypeFormatter formatter = new TypeFormatter(); + datatype.Accept(formatter); + return formatter.formatString; + } + + private static long GetFlags(IArrowType datatype, bool nullable = true) + { + long flags = 0; + + if (nullable) + { + flags |= ArrowFlagNullable; + } + + if (datatype is DictionaryType) + { + if (((DictionaryType)datatype).Ordered) + { + flags |= ArrowFlagDictionaryOrdered; + } + } + + // TODO: when we implement MapType, make sure to set the KEYS_SORTED flag. + return flags; + } + + private static IntPtr* ConstructChildren(IArrowType datatype) + { + if (datatype is NestedType) + { + IReadOnlyList<Field> fields = ((NestedType)datatype).Fields; Review Comment: You can merge the type check and cast: ```suggestion if (datatype is NestedType nestedType) { IReadOnlyList<Field> fields = nestedType.Fields; ``` (same for `ConstructDictionary` below) ########## csharp/test/Apache.Arrow.Tests/CDataInterfaceSchemaTests.cs: ########## @@ -0,0 +1,300 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.InteropServices; +using Apache.Arrow.C; +using Apache.Arrow.Types; +using Python.Runtime; +using Xunit; + +namespace Apache.Arrow.Tests +{ + public class CDataSchemaTest Review Comment: Would it make sense to skip these tests if `PYTHONNET_PYDLL` isn't set? That way people can still run the tests normally in an IDE without having to configure the Python integration. ########## csharp/src/Apache.Arrow/C/CArrowSchema.cs: ########## @@ -0,0 +1,339 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Runtime.InteropServices; +using Apache.Arrow.Types; + +[UnmanagedFunctionPointer(CallingConvention.StdCall)] +public delegate void ReleaseCArrowSchema(IntPtr schema); + +namespace Apache.Arrow.C +{ + /// <summary> + /// An Arrow C Data Interface Schema, which represents a type, field, or schema. + /// </summary> + /// + /// <remarks> + /// This is used to export <see cref="ArrowType"/>, <see cref="Field"/>, or + /// <see cref="Schema"/> to other languages. It matches the layout of the + /// ArrowSchema struct described in https://github.com/apache/arrow/blob/main/cpp/src/arrow/c/abi.h. + /// </remarks> + [StructLayout(LayoutKind.Sequential)] + unsafe public struct CArrowSchema + { + [MarshalAs(UnmanagedType.LPStr)] + public string format; + [MarshalAs(UnmanagedType.LPStr)] + public string name; + [MarshalAs(UnmanagedType.LPStr)] + public string metadata; + public long flags; + public long n_children; + public IntPtr* children; + public IntPtr dictionary; + [MarshalAs(UnmanagedType.FunctionPtr)] + + public ReleaseCArrowSchema release; + // Check this out: https://github.com/G-Research/ParquetSharp/blob/386d91bd5e6fe6cb81583803447023c1359957c8/csharp/ParquetHandle.cs#L8 + public IntPtr private_data; + + private static string GetFormat(IArrowType datatype) + { + TypeFormatter formatter = new TypeFormatter(); + datatype.Accept(formatter); + return formatter.formatString; + } + + private static long GetFlags(IArrowType datatype, bool nullable = true) + { + long flags = 0; + + if (nullable) + { + flags |= ArrowFlagNullable; + } + + if (datatype is DictionaryType) + { + if (((DictionaryType)datatype).Ordered) + { + flags |= ArrowFlagDictionaryOrdered; + } + } + + // TODO: when we implement MapType, make sure to set the KEYS_SORTED flag. + return flags; + } + + private static IntPtr* ConstructChildren(IArrowType datatype) + { + if (datatype is NestedType) + { + IReadOnlyList<Field> fields = ((NestedType)datatype).Fields; + int numFields = fields.Count; + + IntPtr* pointerList = (IntPtr*)Marshal.AllocHGlobal(numFields * sizeof(IntPtr)); + + for (var i = 0; i < numFields; i++) + { + var cSchema = new CArrowSchema(); + CArrowSchema.ExportField(fields[i], out cSchema); + IntPtr exportedSchema = cSchema.AllocateAsPtr(); + pointerList[i] = exportedSchema; + } + + return pointerList; + } + else + { + return (IntPtr*)IntPtr.Zero; + } + } + + private static IntPtr ConstructDictionary(IArrowType datatype) + { + if (datatype is DictionaryType) + { + var cSchema = new CArrowSchema(); + IArrowType valueType = ((DictionaryType)datatype).ValueType; + CArrowSchema.ExportDataType(valueType, out cSchema); + return cSchema.AllocateAsPtr(); + } + else + { + return IntPtr.Zero; + } + } + + /// <summary> + /// Initialize the exported C schema as an Arrow type. + /// </summary> + /// <param name="datatype">The Arrow type to export.</param> + /// <param name="schema">An uninitialized CArrowSchema.</param> + public static void ExportDataType(IArrowType datatype, out CArrowSchema schema) + { + schema.format = GetFormat(datatype); + schema.name = null; + schema.metadata = null; + schema.flags = GetFlags(datatype); + + schema.children = ConstructChildren(datatype); + schema.n_children = datatype is NestedType ? ((NestedType)datatype).Fields.Count : 0; + + schema.dictionary = ConstructDictionary(datatype); + + schema.release = (IntPtr self) => + { + var schema = Marshal.PtrToStructure<CArrowSchema>(self); + if (schema.n_children > 0) + { + for (int i = 0; i < schema.n_children; i++) + { + FreePtr(schema.children[i]); + } + Marshal.FreeHGlobal((IntPtr)schema.children); + } + + if (schema.dictionary != IntPtr.Zero) + { + FreePtr(schema.dictionary); + } + Marshal.DestroyStructure<CArrowSchema>(self); + }; + + schema.private_data = IntPtr.Zero; + } + + /// <summary> + /// Initialize the exported C schema as a field. + /// </summary> + /// <param name="field">Field to export.</param> + /// <param name="schema">An uninitialized CArrowSchema.</param> + public static void ExportField(Field field, out CArrowSchema schema) + { + ExportDataType(field.DataType, out schema); + schema.name = field.Name; + // TODO: field metadata + schema.metadata = null; + schema.flags = GetFlags(field.DataType, field.IsNullable); + } + + /// <summary> + /// Initialize the exported C schema as a schema. + /// </summary> + /// <param name="schema">Schema to export.</param> + /// <param name="outSchema">An uninitialized CArrowSchema</param> + public static void ExportSchema(Schema schema, out CArrowSchema outSchema) + { + // TODO: top-level metadata + var structType = new StructType(schema.Fields.Values.ToList()); + ExportDataType(structType, out outSchema); + } + + /// <summary> + /// Allocate an unmanaged pointer and copy this instances data to it. + /// </summary> + /// <remarks> + /// To avoid a memory leak, you must call <see cref="FreePtr"/> on this + /// pointer when done using it. + /// </remarks> + public IntPtr AllocateAsPtr() + { + IntPtr ptr = Marshal.AllocHGlobal(Marshal.SizeOf(this)); + Marshal.StructureToPtr<CArrowSchema>(this, ptr, false); + return ptr; + } + + /// <summary> + /// Free a pointer that was allocated in <see cref="AllocateAsPtr"/>. + /// </summary> + /// <remarks> + /// Do not call this on a pointer that was allocated elsewhere. + /// </remarks> + public static void FreePtr(IntPtr ptr) + { + var schema = Marshal.PtrToStructure<CArrowSchema>(ptr); + if (schema.release != null) + { + // Call release if not already called. + schema.release(ptr); + } + Marshal.FreeHGlobal(ptr); + } + + /// <summary> + /// Export to an existing pointer + /// </summary> + /// <param name="ptr">An allocated but uninitialized pointer.</param> + public IntPtr Export(IntPtr ptr) Review Comment: It seems a bit surprising that this returns an IntPtr, should this just be a void method? ########## csharp/src/Apache.Arrow/C/ImportedArrowSchema.cs: ########## @@ -0,0 +1,205 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Runtime.InteropServices; +using Apache.Arrow.Types; + +namespace Apache.Arrow.C +{ + /// <summary> + /// A <see cref="CArrowSchema"/> imported from somewhere else. + /// </summary> + /// + /// <example> + /// Typically, when importing a schema we will allocate an uninitialized + /// <see cref="CArrowSchema"/>, pass the pointer to the foreign function, + /// then construct this class with the initialized pointer. + /// + /// <code> + /// var cSchema = new CArrowSchema(); + /// IntPtr importedPtr = cSchema.AllocateAsPtr(); Review Comment: Would it tidy things up slightly to have a static method that allocated an uninitialized schema without needing to create a `CArrowSchema` instance first? ########## csharp/src/Apache.Arrow/C/CArrowSchema.cs: ########## @@ -0,0 +1,339 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Runtime.InteropServices; +using Apache.Arrow.Types; + +[UnmanagedFunctionPointer(CallingConvention.StdCall)] +public delegate void ReleaseCArrowSchema(IntPtr schema); + +namespace Apache.Arrow.C +{ + /// <summary> + /// An Arrow C Data Interface Schema, which represents a type, field, or schema. + /// </summary> + /// + /// <remarks> + /// This is used to export <see cref="ArrowType"/>, <see cref="Field"/>, or + /// <see cref="Schema"/> to other languages. It matches the layout of the + /// ArrowSchema struct described in https://github.com/apache/arrow/blob/main/cpp/src/arrow/c/abi.h. + /// </remarks> + [StructLayout(LayoutKind.Sequential)] + unsafe public struct CArrowSchema + { + [MarshalAs(UnmanagedType.LPStr)] + public string format; + [MarshalAs(UnmanagedType.LPStr)] + public string name; + [MarshalAs(UnmanagedType.LPStr)] + public string metadata; + public long flags; + public long n_children; + public IntPtr* children; Review Comment: Should `children` just be `IntPtr` rather than `IntPtr*`? That would allow the struct itself to not need to be unsafe, although might make dealing with `children` internally a bit more fiddly. ########## csharp/test/Apache.Arrow.Tests/CDataInterfaceSchemaTests.cs: ########## @@ -0,0 +1,300 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.InteropServices; +using Apache.Arrow.C; +using Apache.Arrow.Types; +using Python.Runtime; +using Xunit; + +namespace Apache.Arrow.Tests +{ + public class CDataSchemaTest + { + public CDataSchemaTest() + { + PythonEngine.Initialize(); + } + + public static Schema GetTestSchema() Review Comment: `GetTestSchema`, `GetPythonFields` and `GetPythonSchema` could be private ########## csharp/test/Apache.Arrow.Tests/CDataInterfaceSchemaTests.cs: ########## @@ -0,0 +1,300 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.InteropServices; +using Apache.Arrow.C; +using Apache.Arrow.Types; +using Python.Runtime; +using Xunit; + +namespace Apache.Arrow.Tests +{ + public class CDataSchemaTest + { + public CDataSchemaTest() + { + PythonEngine.Initialize(); + } + + public static Schema GetTestSchema() + { + // TODO: Add more types + using (Py.GIL()) + { + var schema = new Schema.Builder() + .Field(f => f.Name("null").DataType(NullType.Default).Nullable(true)) + .Field(f => f.Name("bool").DataType(BooleanType.Default).Nullable(true)) + .Field(f => f.Name("i8").DataType(Int8Type.Default).Nullable(true)) + .Field(f => f.Name("u8").DataType(UInt8Type.Default).Nullable(true)) + .Field(f => f.Name("i16").DataType(Int16Type.Default).Nullable(true)) + .Field(f => f.Name("u16").DataType(UInt16Type.Default).Nullable(true)) + .Field(f => f.Name("i32").DataType(Int32Type.Default).Nullable(true)) + .Field(f => f.Name("u32").DataType(UInt32Type.Default).Nullable(true)) + .Field(f => f.Name("i64").DataType(Int64Type.Default).Nullable(true)) + .Field(f => f.Name("u64").DataType(UInt64Type.Default).Nullable(true)) + + .Field(f => f.Name("f16").DataType(HalfFloatType.Default).Nullable(true)) + .Field(f => f.Name("f32").DataType(FloatType.Default).Nullable(true)) + .Field(f => f.Name("f64").DataType(DoubleType.Default).Nullable(true)) + + .Field(f => f.Name("binary").DataType(BinaryType.Default).Nullable(false)) + .Field(f => f.Name("string").DataType(StringType.Default).Nullable(false)) + + .Field(f => f.Name("date32").DataType(Date32Type.Default).Nullable(false)) + .Field(f => f.Name("date64").DataType(Date64Type.Default).Nullable(false)) + .Field(f => f.Name("time32_s").DataType(new Time32Type(TimeUnit.Second)).Nullable(false)) + .Field(f => f.Name("time32_ms").DataType(new Time32Type(TimeUnit.Millisecond)).Nullable(false)) + .Field(f => f.Name("time64_us").DataType(new Time64Type(TimeUnit.Microsecond)).Nullable(false)) + .Field(f => f.Name("time64_ns").DataType(new Time64Type(TimeUnit.Nanosecond)).Nullable(false)) + + .Field(f => f.Name("list_string").DataType(new ListType(StringType.Default)).Nullable(false)) + .Field(f => f.Name("list_list_i32").DataType(new ListType(new ListType(Int32Type.Default))).Nullable(false)) + + .Field(f => f.Name("dict_string").DataType(new DictionaryType(Int32Type.Default, StringType.Default, false)).Nullable(false)) + .Field(f => f.Name("dict_string_ordered").DataType(new DictionaryType(Int32Type.Default, StringType.Default, true)).Nullable(false)) + .Field(f => f.Name("list_dict_string").DataType(new ListType(new DictionaryType(Int32Type.Default, StringType.Default, false))).Nullable(false)) + + .Build(); + return schema; + } + } + + public static IEnumerable<dynamic> GetPythonFields() + { + using (Py.GIL()) + { + dynamic pa = Py.Import("pyarrow"); + yield return pa.field("null", pa.GetAttr("null").Invoke(), true); + yield return pa.field("bool", pa.bool_(), true); + yield return pa.field("i8", pa.int8(), true); + yield return pa.field("u8", pa.uint8(), true); + yield return pa.field("i16", pa.int16(), true); + yield return pa.field("u16", pa.uint16(), true); + yield return pa.field("i32", pa.int32(), true); + yield return pa.field("u32", pa.uint32(), true); + yield return pa.field("i64", pa.int64(), true); + yield return pa.field("u64", pa.uint64(), true); + + yield return pa.field("f16", pa.float16(), true); + yield return pa.field("f32", pa.float32(), true); + yield return pa.field("f64", pa.float64(), true); + + yield return pa.field("binary", pa.binary(), false); + yield return pa.field("string", pa.utf8(), false); + + yield return pa.field("date32", pa.date32(), false); + yield return pa.field("date64", pa.date64(), false); + yield return pa.field("time32_s", pa.time32("s"), false); + yield return pa.field("time32_ms", pa.time32("ms"), false); + yield return pa.field("time64_us", pa.time64("us"), false); + yield return pa.field("time64_ns", pa.time64("ns"), false); + + yield return pa.field("list_string", pa.list_(pa.utf8()), false); + yield return pa.field("list_list_i32", pa.list_(pa.list_(pa.int32())), false); + + yield return pa.field("dict_string", pa.dictionary(pa.int32(), pa.utf8(), false), false); + yield return pa.field("dict_string_ordered", pa.dictionary(pa.int32(), pa.utf8(), true), false); + yield return pa.field("list_dict_string", pa.list_(pa.dictionary(pa.int32(), pa.utf8(), false)), false); + } + } + + public static dynamic GetPythonSchema() + { + using (Py.GIL()) + { + dynamic pa = Py.Import("pyarrow"); + return pa.schema(GetPythonFields().ToList()); + } + } + + // Schemas created in Python, used in CSharp + [Fact] + public void ImportType() + { + PythonEngine.Initialize(); Review Comment: The extra calls to `PythonEngine.Initialize` look unnecessary as it's already initialized in the constructor. ########## csharp/src/Apache.Arrow/C/CArrowSchema.cs: ########## @@ -0,0 +1,339 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Runtime.InteropServices; +using Apache.Arrow.Types; + +[UnmanagedFunctionPointer(CallingConvention.StdCall)] +public delegate void ReleaseCArrowSchema(IntPtr schema); + +namespace Apache.Arrow.C +{ + /// <summary> + /// An Arrow C Data Interface Schema, which represents a type, field, or schema. + /// </summary> + /// + /// <remarks> + /// This is used to export <see cref="ArrowType"/>, <see cref="Field"/>, or + /// <see cref="Schema"/> to other languages. It matches the layout of the + /// ArrowSchema struct described in https://github.com/apache/arrow/blob/main/cpp/src/arrow/c/abi.h. + /// </remarks> + [StructLayout(LayoutKind.Sequential)] + unsafe public struct CArrowSchema + { + [MarshalAs(UnmanagedType.LPStr)] + public string format; + [MarshalAs(UnmanagedType.LPStr)] + public string name; + [MarshalAs(UnmanagedType.LPStr)] + public string metadata; Review Comment: From the C Data interface docs it looks like `metadata` isn't a string but is a binary array. C# stores strings as UTF-16 so they can't store arbitrary binary data. Maybe representing this as an `IntPtr` would be most appropriate? ########## csharp/src/Apache.Arrow/C/CArrowSchema.cs: ########## @@ -0,0 +1,339 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Runtime.InteropServices; +using Apache.Arrow.Types; + +[UnmanagedFunctionPointer(CallingConvention.StdCall)] +public delegate void ReleaseCArrowSchema(IntPtr schema); + +namespace Apache.Arrow.C +{ + /// <summary> + /// An Arrow C Data Interface Schema, which represents a type, field, or schema. + /// </summary> + /// + /// <remarks> + /// This is used to export <see cref="ArrowType"/>, <see cref="Field"/>, or + /// <see cref="Schema"/> to other languages. It matches the layout of the + /// ArrowSchema struct described in https://github.com/apache/arrow/blob/main/cpp/src/arrow/c/abi.h. + /// </remarks> + [StructLayout(LayoutKind.Sequential)] + unsafe public struct CArrowSchema + { + [MarshalAs(UnmanagedType.LPStr)] + public string format; + [MarshalAs(UnmanagedType.LPStr)] + public string name; + [MarshalAs(UnmanagedType.LPStr)] + public string metadata; + public long flags; + public long n_children; + public IntPtr* children; + public IntPtr dictionary; + [MarshalAs(UnmanagedType.FunctionPtr)] + + public ReleaseCArrowSchema release; + // Check this out: https://github.com/G-Research/ParquetSharp/blob/386d91bd5e6fe6cb81583803447023c1359957c8/csharp/ParquetHandle.cs#L8 + public IntPtr private_data; + + private static string GetFormat(IArrowType datatype) + { + TypeFormatter formatter = new TypeFormatter(); + datatype.Accept(formatter); + return formatter.formatString; + } + + private static long GetFlags(IArrowType datatype, bool nullable = true) + { + long flags = 0; + + if (nullable) + { + flags |= ArrowFlagNullable; + } + + if (datatype is DictionaryType) + { + if (((DictionaryType)datatype).Ordered) + { + flags |= ArrowFlagDictionaryOrdered; + } + } + + // TODO: when we implement MapType, make sure to set the KEYS_SORTED flag. + return flags; + } + + private static IntPtr* ConstructChildren(IArrowType datatype) + { + if (datatype is NestedType) + { + IReadOnlyList<Field> fields = ((NestedType)datatype).Fields; + int numFields = fields.Count; + + IntPtr* pointerList = (IntPtr*)Marshal.AllocHGlobal(numFields * sizeof(IntPtr)); + + for (var i = 0; i < numFields; i++) + { + var cSchema = new CArrowSchema(); + CArrowSchema.ExportField(fields[i], out cSchema); + IntPtr exportedSchema = cSchema.AllocateAsPtr(); + pointerList[i] = exportedSchema; + } + + return pointerList; + } + else + { + return (IntPtr*)IntPtr.Zero; + } + } + + private static IntPtr ConstructDictionary(IArrowType datatype) + { + if (datatype is DictionaryType) + { + var cSchema = new CArrowSchema(); + IArrowType valueType = ((DictionaryType)datatype).ValueType; + CArrowSchema.ExportDataType(valueType, out cSchema); + return cSchema.AllocateAsPtr(); + } + else + { + return IntPtr.Zero; + } + } + + /// <summary> + /// Initialize the exported C schema as an Arrow type. + /// </summary> + /// <param name="datatype">The Arrow type to export.</param> + /// <param name="schema">An uninitialized CArrowSchema.</param> + public static void ExportDataType(IArrowType datatype, out CArrowSchema schema) Review Comment: Is there a reason that these `Export*` methods all use an out parameter rather than just returning a `CArrowSchema`? ########## csharp/src/Apache.Arrow/C/ImportedArrowSchema.cs: ########## @@ -0,0 +1,205 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Runtime.InteropServices; +using Apache.Arrow.Types; + +namespace Apache.Arrow.C +{ + /// <summary> + /// A <see cref="CArrowSchema"/> imported from somewhere else. + /// </summary> + /// + /// <example> + /// Typically, when importing a schema we will allocate an uninitialized + /// <see cref="CArrowSchema"/>, pass the pointer to the foreign function, + /// then construct this class with the initialized pointer. + /// + /// <code> + /// var cSchema = new CArrowSchema(); + /// IntPtr importedPtr = cSchema.AllocateAsPtr(); + /// foreign_export_function(importedPtr); + /// var importedType = new ImportedArrowSchema(importedPtr); + /// ArrowType arrowType = importedType.GetAsType(); + /// <code> + /// </example> + public sealed class ImportedArrowSchema : IDisposable + { + private readonly CArrowSchema _data; + private readonly IntPtr _handle; + private readonly bool _isRoot; + + public ImportedArrowSchema(IntPtr handle) + { + _data = Marshal.PtrToStructure<CArrowSchema>(handle); + if (_data.release == null) + { + throw new Exception("Tried to import a schema that has already been released."); + } + _handle = handle; + _isRoot = true; + } + + private ImportedArrowSchema(IntPtr handle, bool isRoot) : this(handle) + { + _isRoot = isRoot; + } + + public void Dispose() + { + // We only call release on a root-level schema, not child ones. + if (_isRoot) + { + _data.release(_handle); + } + } + + public ArrowType GetAsType() + { + if (_data.dictionary != IntPtr.Zero) + { + ArrowType indices_type = _data.format switch Review Comment: ```suggestion ArrowType indicesType = _data.format switch ``` ########## csharp/src/Apache.Arrow/C/CArrowSchema.cs: ########## @@ -0,0 +1,519 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +using System; +using System.IO; +using System.Linq; +using System.Runtime.InteropServices; +using Apache.Arrow.Types; + +[UnmanagedFunctionPointer(CallingConvention.StdCall)] +public delegate void ReleaseCArrowSchema(IntPtr schema); + +namespace Apache.Arrow.C +{ + /// <summary> + /// An Arrow C Data Interface Schema, which represents a type, field, or schema. + /// </summary> + /// + /// <remarks> + /// This is used to export <see cref="ArrowType"/>, <see cref="Field"/>, or + /// <see cref="Schema"/> to other languages. It matches the layout of the + /// ArrowSchema struct described in https://github.com/apache/arrow/blob/main/cpp/src/arrow/c/abi.h. + /// </remarks> + [StructLayout(LayoutKind.Sequential)] + unsafe public struct CArrowSchema + { + [MarshalAs(UnmanagedType.LPStr)] + public string format; + [MarshalAs(UnmanagedType.LPStr)] + public string name; + [MarshalAs(UnmanagedType.LPStr)] + public string metadata; + public long flags; + public long n_children; + public IntPtr* children; + public IntPtr dictionary; + [MarshalAs(UnmanagedType.FunctionPtr)] + + public ReleaseCArrowSchema release; + // Check this out: https://github.com/G-Research/ParquetSharp/blob/386d91bd5e6fe6cb81583803447023c1359957c8/csharp/ParquetHandle.cs#L8 + public IntPtr private_data; + + private static string GetFormat(IArrowType datatype) + { + TypeFormatter formatter = new TypeFormatter(); + datatype.Accept(formatter); + return formatter.format_string; + } + + private static long GetFlags(IArrowType datatype, bool nullable = true) + { + long flags = 0; + + if (nullable) + { + flags |= ARROW_FLAG_NULLABLE; + } + + if (datatype is DictionaryType) + { + if (((DictionaryType)datatype).Ordered) + { + flags |= ARROW_FLAG_DICTIONARY_ORDERED; + } + } + + // TODO: when we implement MapType, make sure to set the KEYS_SORTED flag. + return flags; + } + + private static IntPtr* ConstructChildren(IArrowType datatype) + { + if (datatype is NestedType) + { + var fields = ((NestedType)datatype).Fields; + int n_fields = fields.Count; + + IntPtr* pointer_list = (IntPtr*)Marshal.AllocHGlobal(n_fields * sizeof(IntPtr)); + + for (var i = 0; i < n_fields; i++) + { + var c_schema = new CArrowSchema(); + CArrowSchema.ExportField(fields[i], out c_schema); + IntPtr exported_schema = c_schema.AllocateAsPtr(); + pointer_list[i] = exported_schema; + } + + return pointer_list; + } + else + { + return (IntPtr*)IntPtr.Zero; + } + } + + private static IntPtr ConstructDictionary(IArrowType datatype) + { + if (datatype is DictionaryType) + { + var c_schema = new CArrowSchema(); + var value_type = ((DictionaryType)datatype).ValueType; + CArrowSchema.ExportDataType(value_type, out c_schema); + return c_schema.AllocateAsPtr(); + } + else + { + return IntPtr.Zero; + } + } + + /// <summary> + /// Initialize the exported C schema as an Arrow type. + /// </summary> + /// <param name="datatype">The Arrow type to export.</param> + /// <param name="schema">An uninitialized CArrowSchema.</param> + public static void ExportDataType(IArrowType datatype, out CArrowSchema schema) + { + schema.format = GetFormat(datatype); + schema.name = null; + schema.metadata = null; + schema.flags = GetFlags(datatype); + + schema.children = ConstructChildren(datatype); + schema.n_children = datatype is NestedType ? ((NestedType)datatype).Fields.Count : 0; Review Comment: You could pass the schema by reference instead: ```C# void SetChildren(datatype, ref CArrowSchema schema); ``` Or another alternative: ```C# schema.children = ConstructChildren(datatype, out var numChildren); schema.children = numChildren ``` ########## csharp/src/Apache.Arrow/C/CArrowSchema.cs: ########## @@ -0,0 +1,339 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Runtime.InteropServices; +using Apache.Arrow.Types; + +[UnmanagedFunctionPointer(CallingConvention.StdCall)] +public delegate void ReleaseCArrowSchema(IntPtr schema); + +namespace Apache.Arrow.C +{ + /// <summary> + /// An Arrow C Data Interface Schema, which represents a type, field, or schema. + /// </summary> + /// + /// <remarks> + /// This is used to export <see cref="ArrowType"/>, <see cref="Field"/>, or + /// <see cref="Schema"/> to other languages. It matches the layout of the + /// ArrowSchema struct described in https://github.com/apache/arrow/blob/main/cpp/src/arrow/c/abi.h. + /// </remarks> + [StructLayout(LayoutKind.Sequential)] + unsafe public struct CArrowSchema + { + [MarshalAs(UnmanagedType.LPStr)] + public string format; + [MarshalAs(UnmanagedType.LPStr)] + public string name; + [MarshalAs(UnmanagedType.LPStr)] + public string metadata; + public long flags; + public long n_children; + public IntPtr* children; + public IntPtr dictionary; + [MarshalAs(UnmanagedType.FunctionPtr)] + + public ReleaseCArrowSchema release; + // Check this out: https://github.com/G-Research/ParquetSharp/blob/386d91bd5e6fe6cb81583803447023c1359957c8/csharp/ParquetHandle.cs#L8 + public IntPtr private_data; + + private static string GetFormat(IArrowType datatype) + { + TypeFormatter formatter = new TypeFormatter(); + datatype.Accept(formatter); + return formatter.formatString; + } + + private static long GetFlags(IArrowType datatype, bool nullable = true) + { + long flags = 0; + + if (nullable) + { + flags |= ArrowFlagNullable; + } + + if (datatype is DictionaryType) + { + if (((DictionaryType)datatype).Ordered) + { + flags |= ArrowFlagDictionaryOrdered; + } + } + + // TODO: when we implement MapType, make sure to set the KEYS_SORTED flag. + return flags; + } + + private static IntPtr* ConstructChildren(IArrowType datatype) + { + if (datatype is NestedType) + { + IReadOnlyList<Field> fields = ((NestedType)datatype).Fields; + int numFields = fields.Count; + + IntPtr* pointerList = (IntPtr*)Marshal.AllocHGlobal(numFields * sizeof(IntPtr)); + + for (var i = 0; i < numFields; i++) + { + var cSchema = new CArrowSchema(); + CArrowSchema.ExportField(fields[i], out cSchema); Review Comment: You can inline the variable declaration, and there's no need to include the class name when calling a static method from within the class: ```suggestion ExportField(fields[i], out CArrowSchema cSchema); ``` or ```suggestion ExportField(fields[i], out var cSchema); ``` ########## csharp/src/Apache.Arrow/C/CArrowSchema.cs: ########## @@ -0,0 +1,339 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Runtime.InteropServices; +using Apache.Arrow.Types; + +[UnmanagedFunctionPointer(CallingConvention.StdCall)] +public delegate void ReleaseCArrowSchema(IntPtr schema); + +namespace Apache.Arrow.C +{ + /// <summary> + /// An Arrow C Data Interface Schema, which represents a type, field, or schema. + /// </summary> + /// + /// <remarks> + /// This is used to export <see cref="ArrowType"/>, <see cref="Field"/>, or + /// <see cref="Schema"/> to other languages. It matches the layout of the + /// ArrowSchema struct described in https://github.com/apache/arrow/blob/main/cpp/src/arrow/c/abi.h. + /// </remarks> + [StructLayout(LayoutKind.Sequential)] + unsafe public struct CArrowSchema + { + [MarshalAs(UnmanagedType.LPStr)] + public string format; + [MarshalAs(UnmanagedType.LPStr)] + public string name; + [MarshalAs(UnmanagedType.LPStr)] + public string metadata; + public long flags; + public long n_children; + public IntPtr* children; + public IntPtr dictionary; + [MarshalAs(UnmanagedType.FunctionPtr)] + + public ReleaseCArrowSchema release; + // Check this out: https://github.com/G-Research/ParquetSharp/blob/386d91bd5e6fe6cb81583803447023c1359957c8/csharp/ParquetHandle.cs#L8 + public IntPtr private_data; + + private static string GetFormat(IArrowType datatype) + { + TypeFormatter formatter = new TypeFormatter(); + datatype.Accept(formatter); + return formatter.formatString; + } + + private static long GetFlags(IArrowType datatype, bool nullable = true) + { + long flags = 0; + + if (nullable) + { + flags |= ArrowFlagNullable; + } + + if (datatype is DictionaryType) + { + if (((DictionaryType)datatype).Ordered) + { + flags |= ArrowFlagDictionaryOrdered; + } + } + + // TODO: when we implement MapType, make sure to set the KEYS_SORTED flag. + return flags; + } + + private static IntPtr* ConstructChildren(IArrowType datatype) + { + if (datatype is NestedType) + { + IReadOnlyList<Field> fields = ((NestedType)datatype).Fields; + int numFields = fields.Count; + + IntPtr* pointerList = (IntPtr*)Marshal.AllocHGlobal(numFields * sizeof(IntPtr)); + + for (var i = 0; i < numFields; i++) + { + var cSchema = new CArrowSchema(); + CArrowSchema.ExportField(fields[i], out cSchema); + IntPtr exportedSchema = cSchema.AllocateAsPtr(); + pointerList[i] = exportedSchema; + } + + return pointerList; + } + else + { + return (IntPtr*)IntPtr.Zero; + } + } + + private static IntPtr ConstructDictionary(IArrowType datatype) + { + if (datatype is DictionaryType) + { + var cSchema = new CArrowSchema(); + IArrowType valueType = ((DictionaryType)datatype).ValueType; + CArrowSchema.ExportDataType(valueType, out cSchema); + return cSchema.AllocateAsPtr(); + } + else + { + return IntPtr.Zero; + } + } + + /// <summary> + /// Initialize the exported C schema as an Arrow type. + /// </summary> + /// <param name="datatype">The Arrow type to export.</param> + /// <param name="schema">An uninitialized CArrowSchema.</param> + public static void ExportDataType(IArrowType datatype, out CArrowSchema schema) + { + schema.format = GetFormat(datatype); + schema.name = null; + schema.metadata = null; + schema.flags = GetFlags(datatype); + + schema.children = ConstructChildren(datatype); + schema.n_children = datatype is NestedType ? ((NestedType)datatype).Fields.Count : 0; + + schema.dictionary = ConstructDictionary(datatype); + + schema.release = (IntPtr self) => + { + var schema = Marshal.PtrToStructure<CArrowSchema>(self); + if (schema.n_children > 0) + { + for (int i = 0; i < schema.n_children; i++) + { + FreePtr(schema.children[i]); + } + Marshal.FreeHGlobal((IntPtr)schema.children); + } + + if (schema.dictionary != IntPtr.Zero) + { + FreePtr(schema.dictionary); + } + Marshal.DestroyStructure<CArrowSchema>(self); + }; + + schema.private_data = IntPtr.Zero; + } + + /// <summary> + /// Initialize the exported C schema as a field. + /// </summary> + /// <param name="field">Field to export.</param> + /// <param name="schema">An uninitialized CArrowSchema.</param> + public static void ExportField(Field field, out CArrowSchema schema) + { + ExportDataType(field.DataType, out schema); + schema.name = field.Name; + // TODO: field metadata + schema.metadata = null; + schema.flags = GetFlags(field.DataType, field.IsNullable); + } + + /// <summary> + /// Initialize the exported C schema as a schema. + /// </summary> + /// <param name="schema">Schema to export.</param> + /// <param name="outSchema">An uninitialized CArrowSchema</param> + public static void ExportSchema(Schema schema, out CArrowSchema outSchema) + { + // TODO: top-level metadata + var structType = new StructType(schema.Fields.Values.ToList()); + ExportDataType(structType, out outSchema); + } + + /// <summary> + /// Allocate an unmanaged pointer and copy this instances data to it. + /// </summary> + /// <remarks> + /// To avoid a memory leak, you must call <see cref="FreePtr"/> on this + /// pointer when done using it. + /// </remarks> + public IntPtr AllocateAsPtr() + { + IntPtr ptr = Marshal.AllocHGlobal(Marshal.SizeOf(this)); + Marshal.StructureToPtr<CArrowSchema>(this, ptr, false); + return ptr; + } + + /// <summary> + /// Free a pointer that was allocated in <see cref="AllocateAsPtr"/>. + /// </summary> + /// <remarks> + /// Do not call this on a pointer that was allocated elsewhere. + /// </remarks> + public static void FreePtr(IntPtr ptr) + { + var schema = Marshal.PtrToStructure<CArrowSchema>(ptr); + if (schema.release != null) + { + // Call release if not already called. + schema.release(ptr); + } + Marshal.FreeHGlobal(ptr); + } + + /// <summary> + /// Export to an existing pointer + /// </summary> + /// <param name="ptr">An allocated but uninitialized pointer.</param> + public IntPtr Export(IntPtr ptr) + { + Marshal.StructureToPtr<CArrowSchema>(this, ptr, false); + return ptr; + } + + public const int ArrowFlagDictionaryOrdered = 1; + public const int ArrowFlagNullable = 2; + public const int ArrowFlagMapKeysSorted = 4; + + private class TypeFormatter : + IArrowTypeVisitor<NullType>, + IArrowTypeVisitor<BooleanType>, + IArrowTypeVisitor<Int8Type>, + IArrowTypeVisitor<Int16Type>, + IArrowTypeVisitor<Int32Type>, + IArrowTypeVisitor<Int64Type>, + IArrowTypeVisitor<UInt8Type>, + IArrowTypeVisitor<UInt16Type>, + IArrowTypeVisitor<UInt32Type>, + IArrowTypeVisitor<UInt64Type>, + IArrowTypeVisitor<HalfFloatType>, + IArrowTypeVisitor<FloatType>, + IArrowTypeVisitor<DoubleType>, + IArrowTypeVisitor<Decimal128Type>, + IArrowTypeVisitor<Decimal256Type>, + IArrowTypeVisitor<Date32Type>, + IArrowTypeVisitor<Date64Type>, + IArrowTypeVisitor<Time32Type>, + IArrowTypeVisitor<Time64Type>, + IArrowTypeVisitor<TimestampType>, + IArrowTypeVisitor<StringType>, + IArrowTypeVisitor<BinaryType>, + IArrowTypeVisitor<FixedSizeBinaryType>, + IArrowTypeVisitor<ListType>, + IArrowTypeVisitor<StructType>, + IArrowTypeVisitor<DictionaryType> + { + public string formatString; Review Comment: ```suggestion public string FormatString; ``` ########## csharp/src/Apache.Arrow/C/ImportedArrowSchema.cs: ########## @@ -0,0 +1,205 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Runtime.InteropServices; +using Apache.Arrow.Types; + +namespace Apache.Arrow.C +{ + /// <summary> + /// A <see cref="CArrowSchema"/> imported from somewhere else. + /// </summary> + /// + /// <example> + /// Typically, when importing a schema we will allocate an uninitialized + /// <see cref="CArrowSchema"/>, pass the pointer to the foreign function, + /// then construct this class with the initialized pointer. + /// + /// <code> + /// var cSchema = new CArrowSchema(); + /// IntPtr importedPtr = cSchema.AllocateAsPtr(); + /// foreign_export_function(importedPtr); + /// var importedType = new ImportedArrowSchema(importedPtr); + /// ArrowType arrowType = importedType.GetAsType(); + /// <code> + /// </example> + public sealed class ImportedArrowSchema : IDisposable + { + private readonly CArrowSchema _data; + private readonly IntPtr _handle; + private readonly bool _isRoot; + + public ImportedArrowSchema(IntPtr handle) + { + _data = Marshal.PtrToStructure<CArrowSchema>(handle); + if (_data.release == null) + { + throw new Exception("Tried to import a schema that has already been released."); + } + _handle = handle; + _isRoot = true; + } + + private ImportedArrowSchema(IntPtr handle, bool isRoot) : this(handle) + { + _isRoot = isRoot; + } + + public void Dispose() + { + // We only call release on a root-level schema, not child ones. + if (_isRoot) + { + _data.release(_handle); + } + } + + public ArrowType GetAsType() + { + if (_data.dictionary != IntPtr.Zero) + { + ArrowType indices_type = _data.format switch + { + "c" => new Int8Type(), + "C" => new UInt8Type(), + "s" => new Int16Type(), + "S" => new UInt16Type(), + "i" => new Int32Type(), + "I" => new UInt32Type(), + "l" => new Int64Type(), + "L" => new UInt64Type(), + _ => throw new InvalidDataException($"Indices must be an integer, but got format string {_data.format}"), + }; + + var dictionarySchema = new ImportedArrowSchema(_data.dictionary, /*is_root*/ false); + ArrowType dictionaryType = dictionarySchema.GetAsType(); + + bool ordered = (_data.flags & CArrowSchema.ArrowFlagNullable) == CArrowSchema.ArrowFlagNullable; + + return new DictionaryType(indices_type, dictionaryType, ordered); + } + + // Special handling for nested types + if (_data.format == "+l") + { + if (_data.n_children != 1) + { + throw new Exception("Expected list type to have exactly one child."); + } + ImportedArrowSchema childSchema; + unsafe + { + if (_data.children[0] == null) + { + throw new Exception("Expected list type child to be non-null."); + } + childSchema = new ImportedArrowSchema(_data.children[0]); + } + + Field childField = childSchema.GetAsField(); + + return new ListType(childField); + } + else if (_data.format == "+s") + { + var child_schemas = new ImportedArrowSchema[_data.n_children]; + unsafe + { + for (int i = 0; i < _data.n_children; i++) + { + if (_data.children[i] == null) + { + throw new Exception("Expected struct type child to be non-null."); + } + child_schemas[i] = new ImportedArrowSchema(_data.children[i]); + } + + } + + List<Field> childFields = child_schemas.Select(schema => schema.GetAsField()).ToList(); + + return new StructType(childFields); + } + // TODO: Map type and large list type + + return _data.format switch + { + // Primitives + "n" => new NullType(), + "b" => new BooleanType(), + "c" => new Int8Type(), + "C" => new UInt8Type(), + "s" => new Int16Type(), + "S" => new UInt16Type(), + "i" => new Int32Type(), + "I" => new UInt32Type(), + "l" => new Int64Type(), + "L" => new UInt64Type(), + "e" => new HalfFloatType(), + "f" => new FloatType(), + "g" => new DoubleType(), + // Binary data + "z" => new BinaryType(), + //"Z" => new LargeBinaryType() // Not yet implemented + "u" => new StringType(), + //"U" => new LargeStringType(), // Not yet implemented + // TODO: decimal + // TODO: fixed-width binary + // Date and time + "tdD" => new Date32Type(), + "tdm" => new Date64Type(), + "tts" => new Time32Type(TimeUnit.Second), + "ttm" => new Time32Type(TimeUnit.Millisecond), + "ttu" => new Time64Type(TimeUnit.Microsecond), + "ttn" => new Time64Type(TimeUnit.Nanosecond), + // TODO: timestamp with timezone, + // TODO: duration not yet implemented + "tiM" => new IntervalType(IntervalUnit.YearMonth), + "tiD" => new IntervalType(IntervalUnit.DayTime), + //"tin" => new IntervalType(IntervalUnit.MonthDayNanosecond), // Not yet implemented + _ => throw new NotSupportedException("Data type is not yet supported in import.") + }; + } + + public Field GetAsField() + { + string fieldName = string.IsNullOrEmpty(_data.name) ? "" : _data.name; + + bool nullable = (_data.flags & CArrowSchema.ArrowFlagNullable) == CArrowSchema.ArrowFlagNullable; + + return new Field(fieldName, GetAsType(), nullable); + } + + public Schema GetAsSchema() + { + ArrowType fullType = GetAsType(); + if (fullType is StructType) + { + StructType structType = (StructType)fullType; Review Comment: ```suggestion if (fullType is StructType structType) { ``` ########## csharp/src/Apache.Arrow/C/ImportedArrowSchema.cs: ########## @@ -0,0 +1,205 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Runtime.InteropServices; +using Apache.Arrow.Types; + +namespace Apache.Arrow.C +{ + /// <summary> + /// A <see cref="CArrowSchema"/> imported from somewhere else. + /// </summary> + /// + /// <example> + /// Typically, when importing a schema we will allocate an uninitialized + /// <see cref="CArrowSchema"/>, pass the pointer to the foreign function, + /// then construct this class with the initialized pointer. + /// + /// <code> + /// var cSchema = new CArrowSchema(); + /// IntPtr importedPtr = cSchema.AllocateAsPtr(); + /// foreign_export_function(importedPtr); + /// var importedType = new ImportedArrowSchema(importedPtr); Review Comment: ```suggestion /// using var importedType = new ImportedArrowSchema(importedPtr); ``` ? Although then you need to be careful that you don't free the pointer before `Dispose` is called, so maybe it's better to do: ```c# using (var importedType = new ImportedArrowSchema(importedPtr)) { ArrowType arrowType = importedType.GetAsType(); } ``` It looks like we no longer need the data after we have a C# field/schema/type though, so could this class be replaced with a set of static methods that could be used something like: ```C# ArrowType type = CDataInterface.ImportType(importedPtr); ``` That should simplify the API and make it less likely we try to call release after the pointer data has been freed. ########## csharp/test/Apache.Arrow.Tests/CDataInterfaceSchemaTests.cs: ########## @@ -0,0 +1,300 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.InteropServices; +using Apache.Arrow.C; +using Apache.Arrow.Types; +using Python.Runtime; +using Xunit; + +namespace Apache.Arrow.Tests +{ + public class CDataSchemaTest + { + public CDataSchemaTest() + { + PythonEngine.Initialize(); + } + + public static Schema GetTestSchema() + { + // TODO: Add more types + using (Py.GIL()) + { + var schema = new Schema.Builder() + .Field(f => f.Name("null").DataType(NullType.Default).Nullable(true)) + .Field(f => f.Name("bool").DataType(BooleanType.Default).Nullable(true)) + .Field(f => f.Name("i8").DataType(Int8Type.Default).Nullable(true)) + .Field(f => f.Name("u8").DataType(UInt8Type.Default).Nullable(true)) + .Field(f => f.Name("i16").DataType(Int16Type.Default).Nullable(true)) + .Field(f => f.Name("u16").DataType(UInt16Type.Default).Nullable(true)) + .Field(f => f.Name("i32").DataType(Int32Type.Default).Nullable(true)) + .Field(f => f.Name("u32").DataType(UInt32Type.Default).Nullable(true)) + .Field(f => f.Name("i64").DataType(Int64Type.Default).Nullable(true)) + .Field(f => f.Name("u64").DataType(UInt64Type.Default).Nullable(true)) + + .Field(f => f.Name("f16").DataType(HalfFloatType.Default).Nullable(true)) + .Field(f => f.Name("f32").DataType(FloatType.Default).Nullable(true)) + .Field(f => f.Name("f64").DataType(DoubleType.Default).Nullable(true)) + + .Field(f => f.Name("binary").DataType(BinaryType.Default).Nullable(false)) + .Field(f => f.Name("string").DataType(StringType.Default).Nullable(false)) + + .Field(f => f.Name("date32").DataType(Date32Type.Default).Nullable(false)) + .Field(f => f.Name("date64").DataType(Date64Type.Default).Nullable(false)) + .Field(f => f.Name("time32_s").DataType(new Time32Type(TimeUnit.Second)).Nullable(false)) + .Field(f => f.Name("time32_ms").DataType(new Time32Type(TimeUnit.Millisecond)).Nullable(false)) + .Field(f => f.Name("time64_us").DataType(new Time64Type(TimeUnit.Microsecond)).Nullable(false)) + .Field(f => f.Name("time64_ns").DataType(new Time64Type(TimeUnit.Nanosecond)).Nullable(false)) + + .Field(f => f.Name("list_string").DataType(new ListType(StringType.Default)).Nullable(false)) + .Field(f => f.Name("list_list_i32").DataType(new ListType(new ListType(Int32Type.Default))).Nullable(false)) + + .Field(f => f.Name("dict_string").DataType(new DictionaryType(Int32Type.Default, StringType.Default, false)).Nullable(false)) + .Field(f => f.Name("dict_string_ordered").DataType(new DictionaryType(Int32Type.Default, StringType.Default, true)).Nullable(false)) + .Field(f => f.Name("list_dict_string").DataType(new ListType(new DictionaryType(Int32Type.Default, StringType.Default, false))).Nullable(false)) + + .Build(); + return schema; + } + } + + public static IEnumerable<dynamic> GetPythonFields() + { + using (Py.GIL()) + { + dynamic pa = Py.Import("pyarrow"); + yield return pa.field("null", pa.GetAttr("null").Invoke(), true); + yield return pa.field("bool", pa.bool_(), true); + yield return pa.field("i8", pa.int8(), true); + yield return pa.field("u8", pa.uint8(), true); + yield return pa.field("i16", pa.int16(), true); + yield return pa.field("u16", pa.uint16(), true); + yield return pa.field("i32", pa.int32(), true); + yield return pa.field("u32", pa.uint32(), true); + yield return pa.field("i64", pa.int64(), true); + yield return pa.field("u64", pa.uint64(), true); + + yield return pa.field("f16", pa.float16(), true); + yield return pa.field("f32", pa.float32(), true); + yield return pa.field("f64", pa.float64(), true); + + yield return pa.field("binary", pa.binary(), false); + yield return pa.field("string", pa.utf8(), false); + + yield return pa.field("date32", pa.date32(), false); + yield return pa.field("date64", pa.date64(), false); + yield return pa.field("time32_s", pa.time32("s"), false); + yield return pa.field("time32_ms", pa.time32("ms"), false); + yield return pa.field("time64_us", pa.time64("us"), false); + yield return pa.field("time64_ns", pa.time64("ns"), false); + + yield return pa.field("list_string", pa.list_(pa.utf8()), false); + yield return pa.field("list_list_i32", pa.list_(pa.list_(pa.int32())), false); + + yield return pa.field("dict_string", pa.dictionary(pa.int32(), pa.utf8(), false), false); + yield return pa.field("dict_string_ordered", pa.dictionary(pa.int32(), pa.utf8(), true), false); + yield return pa.field("list_dict_string", pa.list_(pa.dictionary(pa.int32(), pa.utf8(), false)), false); + } + } + + public static dynamic GetPythonSchema() + { + using (Py.GIL()) + { + dynamic pa = Py.Import("pyarrow"); + return pa.schema(GetPythonFields().ToList()); + } + } + + // Schemas created in Python, used in CSharp + [Fact] + public void ImportType() + { + PythonEngine.Initialize(); + Schema schema = GetTestSchema(); + IEnumerable<dynamic> pyFields = GetPythonFields(); + + foreach ((Field field, dynamic pyField) in schema.Fields.Values.AsEnumerable() + .Zip(pyFields)) + { + var cSchema = new CArrowSchema(); + IntPtr importedPtr = cSchema.AllocateAsPtr(); + + using (Py.GIL()) + { + dynamic py_datatype = pyField.type; Review Comment: ```suggestion dynamic pyDatatype = pyField.type; ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
