This is an automated email from the ASF dual-hosted git repository.
lidavidm pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new 37a9f27e65 GH-33032: [C#] Support fixed-size lists (#35716)
37a9f27e65 is described below
commit 37a9f27e658474a929f351c89c90159b401fb8ca
Author: Curt Hagenlocher <[email protected]>
AuthorDate: Mon Aug 21 13:23:45 2023 -0700
GH-33032: [C#] Support fixed-size lists (#35716)
### What changes are included in this PR?
Support fixed-size lists in the C# implementation.
Adds Archery support for Lists, Structs and Fixed-size Lists in the C#
implementation.
### Are these changes tested?
Yes.
### Are there any user-facing changes?
Fixed-size lists are now supported for C#.
* Closes: #33032
* Closes: #32886
Authored-by: Curt Hagenlocher <[email protected]>
Signed-off-by: David Li <[email protected]>
---
.../Apache.Arrow/Arrays/ArrayDataConcatenator.cs | 10 ++
.../Apache.Arrow/Arrays/ArrayDataTypeComparer.cs | 11 ++
.../Arrays/ArrowArrayBuilderFactory.cs | 2 +
.../src/Apache.Arrow/Arrays/ArrowArrayFactory.cs | 2 +
csharp/src/Apache.Arrow/Arrays/DateArrayBuilder.cs | 2 +-
.../Apache.Arrow/Arrays/DelegatingArrayBuilder.cs | 6 +
.../src/Apache.Arrow/Arrays/FixedSizeListArray.cs | 190 +++++++++++++++++++++
csharp/src/Apache.Arrow/Arrays/TimeArrayBuilder.cs | 2 +-
csharp/src/Apache.Arrow/C/CArrowArrayImporter.cs | 23 ++-
csharp/src/Apache.Arrow/C/CArrowSchemaExporter.cs | 2 +
csharp/src/Apache.Arrow/C/CArrowSchemaImporter.cs | 21 +++
csharp/src/Apache.Arrow/Field.cs | 10 +-
.../Apache.Arrow/Interfaces/IArrowArrayBuilder.cs | 4 +-
.../Apache.Arrow/Ipc/ArrowReaderImplementation.cs | 2 +-
csharp/src/Apache.Arrow/Ipc/ArrowStreamWriter.cs | 8 +
.../Apache.Arrow/Ipc/ArrowTypeFlatbufferBuilder.cs | 8 +
csharp/src/Apache.Arrow/Ipc/MessageSerializer.cs | 16 +-
.../Types/{IArrowType.cs => FixedSizeListType.cs} | 59 +++----
csharp/src/Apache.Arrow/Types/IArrowType.cs | 3 +-
.../IntegrationCommand.cs | 80 ++++++++-
.../test/Apache.Arrow.IntegrationTest/JsonFile.cs | 3 +
.../test/Apache.Arrow.Tests/ArrayBuilderTests.cs | 126 ++++++++++++++
.../test/Apache.Arrow.Tests/ArrayTypeComparer.cs | 11 ++
.../ArrowArrayConcatenatorTests.cs | 38 +++++
.../test/Apache.Arrow.Tests/ArrowReaderVerifier.cs | 20 ++-
.../CDataInterfacePythonTests.cs | 16 +-
csharp/test/Apache.Arrow.Tests/TableTests.cs | 2 +-
csharp/test/Apache.Arrow.Tests/TestData.cs | 28 +++
dev/archery/archery/integration/datagen.py | 6 +-
docs/source/status.rst | 2 +-
30 files changed, 643 insertions(+), 70 deletions(-)
diff --git a/csharp/src/Apache.Arrow/Arrays/ArrayDataConcatenator.cs
b/csharp/src/Apache.Arrow/Arrays/ArrayDataConcatenator.cs
index 2b5e59517c..8859ecd7f0 100644
--- a/csharp/src/Apache.Arrow/Arrays/ArrayDataConcatenator.cs
+++ b/csharp/src/Apache.Arrow/Arrays/ArrayDataConcatenator.cs
@@ -48,6 +48,7 @@ namespace Apache.Arrow
IArrowTypeVisitor<BinaryType>,
IArrowTypeVisitor<StringType>,
IArrowTypeVisitor<ListType>,
+ IArrowTypeVisitor<FixedSizeListType>,
IArrowTypeVisitor<StructType>
{
public ArrayData Result { get; private set; }
@@ -100,6 +101,15 @@ namespace Apache.Arrow
Result = new ArrayData(type, _totalLength, _totalNullCount, 0,
new ArrowBuffer[] { validityBuffer, offsetBuffer }, new[] { child });
}
+ public void Visit(FixedSizeListType type)
+ {
+ CheckData(type, 1);
+ ArrowBuffer validityBuffer = ConcatenateValidityBuffer();
+ ArrayData child = Concatenate(SelectChildren(0), _allocator);
+
+ Result = new ArrayData(type, _totalLength, _totalNullCount, 0,
new ArrowBuffer[] { validityBuffer }, new[] { child });
+ }
+
public void Visit(StructType type)
{
CheckData(type, 1);
diff --git a/csharp/src/Apache.Arrow/Arrays/ArrayDataTypeComparer.cs
b/csharp/src/Apache.Arrow/Arrays/ArrayDataTypeComparer.cs
index 1e0524e53e..8a6bfed29a 100644
--- a/csharp/src/Apache.Arrow/Arrays/ArrayDataTypeComparer.cs
+++ b/csharp/src/Apache.Arrow/Arrays/ArrayDataTypeComparer.cs
@@ -26,6 +26,7 @@ namespace Apache.Arrow
IArrowTypeVisitor<Time64Type>,
IArrowTypeVisitor<FixedSizeBinaryType>,
IArrowTypeVisitor<ListType>,
+ IArrowTypeVisitor<FixedSizeListType>,
IArrowTypeVisitor<StructType>
{
private readonly IArrowType _expectedType;
@@ -102,6 +103,16 @@ namespace Apache.Arrow
}
}
+ public void Visit(FixedSizeListType actualType)
+ {
+ if (_expectedType is FixedSizeListType expectedType
+ && actualType.ListSize == expectedType.ListSize
+ && CompareNested(expectedType, actualType))
+ {
+ _dataTypeMatch = true;
+ }
+ }
+
public void Visit(StructType actualType)
{
if (_expectedType is StructType expectedType
diff --git a/csharp/src/Apache.Arrow/Arrays/ArrowArrayBuilderFactory.cs
b/csharp/src/Apache.Arrow/Arrays/ArrowArrayBuilderFactory.cs
index ef80edb838..1b972d0187 100644
--- a/csharp/src/Apache.Arrow/Arrays/ArrowArrayBuilderFactory.cs
+++ b/csharp/src/Apache.Arrow/Arrays/ArrowArrayBuilderFactory.cs
@@ -68,6 +68,8 @@ namespace Apache.Arrow
return new Time64Array.Builder(dataType as Time64Type);
case ArrowTypeId.List:
return new ListArray.Builder(dataType as ListType);
+ case ArrowTypeId.FixedSizeList:
+ return new FixedSizeListArray.Builder(dataType as
FixedSizeListType);
case ArrowTypeId.Decimal128:
return new Decimal128Array.Builder(dataType as
Decimal128Type);
case ArrowTypeId.Decimal256:
diff --git a/csharp/src/Apache.Arrow/Arrays/ArrowArrayFactory.cs
b/csharp/src/Apache.Arrow/Arrays/ArrowArrayFactory.cs
index 845cbbd3e5..f82037bff4 100644
--- a/csharp/src/Apache.Arrow/Arrays/ArrowArrayFactory.cs
+++ b/csharp/src/Apache.Arrow/Arrays/ArrowArrayFactory.cs
@@ -83,6 +83,8 @@ namespace Apache.Arrow
#else
throw new NotSupportedException("Half-float arrays are not
supported by this target framework.");
#endif
+ case ArrowTypeId.FixedSizeList:
+ return new FixedSizeListArray(data);
case ArrowTypeId.Interval:
case ArrowTypeId.Map:
default:
diff --git a/csharp/src/Apache.Arrow/Arrays/DateArrayBuilder.cs
b/csharp/src/Apache.Arrow/Arrays/DateArrayBuilder.cs
index dcbb76930b..29246fa250 100644
--- a/csharp/src/Apache.Arrow/Arrays/DateArrayBuilder.cs
+++ b/csharp/src/Apache.Arrow/Arrays/DateArrayBuilder.cs
@@ -157,7 +157,7 @@ namespace Apache.Arrow
/// Append a null date to the array.
/// </summary>
/// <returns>Returns the builder (for fluent-style
composition).</returns>
- public TBuilder AppendNull()
+ public override TBuilder AppendNull()
{
InnerBuilder.AppendNull();
return this as TBuilder;
diff --git a/csharp/src/Apache.Arrow/Arrays/DelegatingArrayBuilder.cs
b/csharp/src/Apache.Arrow/Arrays/DelegatingArrayBuilder.cs
index f2ab3ee13c..827b00c58f 100644
--- a/csharp/src/Apache.Arrow/Arrays/DelegatingArrayBuilder.cs
+++ b/csharp/src/Apache.Arrow/Arrays/DelegatingArrayBuilder.cs
@@ -98,5 +98,11 @@ namespace Apache.Arrow
InnerBuilder.Clear();
return this as TBuilder;
}
+
+ /// <summary>
+ /// Appends a null value
+ /// </summary>
+ /// <returns>Returns the builder (for fluent-style
composition).</returns>
+ public abstract TBuilder AppendNull();
}
}
diff --git a/csharp/src/Apache.Arrow/Arrays/FixedSizeListArray.cs
b/csharp/src/Apache.Arrow/Arrays/FixedSizeListArray.cs
new file mode 100644
index 0000000000..f60daedb57
--- /dev/null
+++ b/csharp/src/Apache.Arrow/Arrays/FixedSizeListArray.cs
@@ -0,0 +1,190 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+using System;
+using Apache.Arrow.Memory;
+using Apache.Arrow.Types;
+
+namespace Apache.Arrow
+{
+ public class FixedSizeListArray : Array
+ {
+ public class Builder : IArrowArrayBuilder<FixedSizeListArray, Builder>
+ {
+ public IArrowArrayBuilder<IArrowArray,
IArrowArrayBuilder<IArrowArray>> ValueBuilder { get; }
+
+ public int Length { get; protected set; }
+
+ private ArrowBuffer.BitmapBuilder ValidityBufferBuilder { get; }
+
+ public int NullCount => ValidityBufferBuilder.UnsetBitCount;
+
+ private FixedSizeListType DataType { get; }
+
+ private int ExpectedValueLength => Length * DataType.ListSize;
+
+ public Builder(IArrowType valueDataType, int listSize) : this(new
FixedSizeListType(valueDataType, listSize))
+ {
+ }
+
+ public Builder(Field valueField, int listSize) : this(new
FixedSizeListType(valueField, listSize))
+ {
+ }
+
+ internal Builder(FixedSizeListType dataType)
+ {
+ ValueBuilder =
ArrowArrayBuilderFactory.Build(dataType.ValueDataType);
+ ValidityBufferBuilder = new ArrowBuffer.BitmapBuilder();
+ DataType = dataType;
+ }
+
+ /// <summary>
+ /// Start a new fixed-length list slot
+ ///
+ /// This function should be called before beginning to append
elements to the
+ /// value builder
+ /// </summary>
+ /// <returns></returns>
+ public Builder Append()
+ {
+ ValidateChildLength();
+
+ ValidityBufferBuilder.Append(true);
+ Length++;
+
+ return this;
+ }
+
+ public Builder AppendNull()
+ {
+ ValidateChildLength();
+
+ ValidityBufferBuilder.Append(false);
+ for (int i = 0; i < DataType.ListSize; i++)
+ {
+ ValueBuilder.AppendNull();
+ }
+ Length++;
+
+ return this;
+ }
+
+ public FixedSizeListArray Build(MemoryAllocator allocator =
default)
+ {
+ ValidateChildLength();
+
+ int nullCount = NullCount;
+ ArrowBuffer validityBuffer = nullCount > 0
+ ?
ValidityBufferBuilder.Build(allocator)
+ : ArrowBuffer.Empty;
+
+ return new FixedSizeListArray(DataType, Length,
+ ValueBuilder.Build(allocator),
+ validityBuffer, nullCount, 0);
+ }
+
+ public Builder Reserve(int capacity)
+ {
+ ValidityBufferBuilder.Reserve(capacity);
+ ValueBuilder.Reserve(DataType.ListSize * capacity);
+ return this;
+ }
+
+ public Builder Resize(int length)
+ {
+ ValidateChildLength();
+
+ ValidityBufferBuilder.Resize(length);
+ ValueBuilder.Resize(DataType.ListSize * length);
+ Length = length;
+ return this;
+ }
+
+ public Builder Clear()
+ {
+ ValueBuilder.Clear();
+ ValidityBufferBuilder.Clear();
+ Length = 0;
+ return this;
+ }
+
+ void ValidateChildLength()
+ {
+ if (ValueBuilder.Length != ExpectedValueLength)
+ {
+ int actualLength = ValueBuilder.Length -
ExpectedValueLength + DataType.ListSize;
+ throw new ArgumentOutOfRangeException($"Lists of length:
{actualLength} do not conform to the fixed size: " + DataType.ListSize);
+ }
+ }
+ }
+
+ public IArrowArray Values { get; }
+
+ public FixedSizeListArray(IArrowType dataType, int length,
+ IArrowArray values, ArrowBuffer nullBitmapBuffer,
+ int nullCount = 0, int offset = 0)
+ : this(new ArrayData(dataType, length, nullCount, offset,
+ new[] { nullBitmapBuffer }, new[] { values.Data }),
+ values)
+ {
+ }
+
+ public FixedSizeListArray(ArrayData data)
+ : this(data, ArrowArrayFactory.BuildArray(data.Children[0]))
+ {
+ }
+
+ private FixedSizeListArray(ArrayData data, IArrowArray values) :
base(data)
+ {
+ data.EnsureBufferCount(1);
+ data.EnsureDataType(ArrowTypeId.FixedSizeList);
+ Values = values;
+ }
+
+ public override void Accept(IArrowArrayVisitor visitor) =>
Accept(this, visitor);
+
+ public IArrowArray GetSlicedValues(int index)
+ {
+ if (index < 0 || index >= Length)
+ {
+ throw new ArgumentOutOfRangeException(nameof(index));
+ }
+
+ if (IsNull(index))
+ {
+ return null;
+ }
+
+ if (!(Values is Array array))
+ {
+ return default;
+ }
+
+ index += Data.Offset;
+
+ int length = ((FixedSizeListType)Data.DataType).ListSize;
+ return array.Slice(index * length, length);
+ }
+
+ protected override void Dispose(bool disposing)
+ {
+ if (disposing)
+ {
+ Values?.Dispose();
+ }
+ base.Dispose(disposing);
+ }
+ }
+}
diff --git a/csharp/src/Apache.Arrow/Arrays/TimeArrayBuilder.cs
b/csharp/src/Apache.Arrow/Arrays/TimeArrayBuilder.cs
index da93db8471..88f8aa8c45 100644
--- a/csharp/src/Apache.Arrow/Arrays/TimeArrayBuilder.cs
+++ b/csharp/src/Apache.Arrow/Arrays/TimeArrayBuilder.cs
@@ -88,7 +88,7 @@ namespace Apache.Arrow
/// Append a null time to the array.
/// </summary>
/// <returns>Returns the builder (for fluent-style
composition).</returns>
- public TBuilder AppendNull()
+ public override TBuilder AppendNull()
{
InnerBuilder.AppendNull();
return this as TBuilder;
diff --git a/csharp/src/Apache.Arrow/C/CArrowArrayImporter.cs
b/csharp/src/Apache.Arrow/C/CArrowArrayImporter.cs
index 2f4ebed4b0..9b7bcb7abe 100644
--- a/csharp/src/Apache.Arrow/C/CArrowArrayImporter.cs
+++ b/csharp/src/Apache.Arrow/C/CArrowArrayImporter.cs
@@ -161,6 +161,10 @@ namespace Apache.Arrow.C
children = ProcessListChildren(cArray,
((ListType)type).ValueDataType);
buffers = ImportListBuffers(cArray);
break;
+ case ArrowTypeId.FixedSizeList:
+ children = ProcessListChildren(cArray,
((FixedSizeListType)type).ValueDataType);
+ buffers = ImportFixedSizeListBuffers(cArray);
+ break;
case ArrowTypeId.Struct:
children = ProcessStructChildren(cArray,
((StructType)type).Fields);
buffers = new ArrowBuffer[] {
ImportValidityBuffer(cArray) };
@@ -236,7 +240,7 @@ namespace Apache.Arrow.C
{
if (cArray->n_buffers != 3)
{
- throw new InvalidOperationException("Byte arrays are
expected to have exactly three child arrays");
+ throw new InvalidOperationException("Byte arrays are
expected to have exactly three buffers");
}
int length = checked((int)cArray->length);
@@ -256,7 +260,7 @@ namespace Apache.Arrow.C
{
if (cArray->n_buffers != 2)
{
- throw new InvalidOperationException("List arrays are
expected to have exactly two children");
+ throw new InvalidOperationException("List arrays are
expected to have exactly two buffers");
}
int length = checked((int)cArray->length);
@@ -269,11 +273,24 @@ namespace Apache.Arrow.C
return buffers;
}
+ private ArrowBuffer[] ImportFixedSizeListBuffers(CArrowArray*
cArray)
+ {
+ if (cArray->n_buffers != 1)
+ {
+ throw new InvalidOperationException("Fixed-size list
arrays are expected to have exactly one buffer");
+ }
+
+ ArrowBuffer[] buffers = new ArrowBuffer[1];
+ buffers[0] = ImportValidityBuffer(cArray);
+
+ return buffers;
+ }
+
private ArrowBuffer[] ImportFixedWidthBuffers(CArrowArray* cArray,
int bitWidth)
{
if (cArray->n_buffers != 2)
{
- throw new InvalidOperationException("Arrays of fixed-width
type are expected to have exactly two children");
+ throw new InvalidOperationException("Arrays of fixed-width
type are expected to have exactly two buffers");
}
// validity, data
diff --git a/csharp/src/Apache.Arrow/C/CArrowSchemaExporter.cs
b/csharp/src/Apache.Arrow/C/CArrowSchemaExporter.cs
index 696212eda3..66142da331 100644
--- a/csharp/src/Apache.Arrow/C/CArrowSchemaExporter.cs
+++ b/csharp/src/Apache.Arrow/C/CArrowSchemaExporter.cs
@@ -167,6 +167,8 @@ namespace Apache.Arrow.C
return String.Format("ts{0}:{1}",
FormatTimeUnit(timestampType.Unit), timestampType.Timezone);
// Nested
case ListType _: return "+l";
+ case FixedSizeListType fixedListType:
+ return $"+w:{fixedListType.ListSize}";
case StructType _: return "+s";
// Dictionary
case DictionaryType dictionaryType:
diff --git a/csharp/src/Apache.Arrow/C/CArrowSchemaImporter.cs
b/csharp/src/Apache.Arrow/C/CArrowSchemaImporter.cs
index ec613d59d2..2a750d5e82 100644
--- a/csharp/src/Apache.Arrow/C/CArrowSchemaImporter.cs
+++ b/csharp/src/Apache.Arrow/C/CArrowSchemaImporter.cs
@@ -200,6 +200,27 @@ namespace Apache.Arrow.C
return new StructType(childFields);
}
+ else if (format.StartsWith("+w:"))
+ {
+ // Fixed-width list
+ int width = Int32.Parse(format.Substring(3));
+
+ if (_cSchema->n_children != 1)
+ {
+ throw new InvalidDataException("Expected fixed-length
list type to have exactly one child.");
+ }
+ ImportedArrowSchema childSchema;
+ if (_cSchema->GetChild(0) == null)
+ {
+ throw new InvalidDataException("Expected fixed-length
list type child to be non-null.");
+ }
+ childSchema = new
ImportedArrowSchema(_cSchema->GetChild(0), isRoot: false);
+
+ Field childField = childSchema.GetAsField();
+
+ return new FixedSizeListType(childField, width);
+ }
+
// TODO: Map type and large list type
// Decimals
diff --git a/csharp/src/Apache.Arrow/Field.cs b/csharp/src/Apache.Arrow/Field.cs
index 6e507b642f..562b9587bd 100644
--- a/csharp/src/Apache.Arrow/Field.cs
+++ b/csharp/src/Apache.Arrow/Field.cs
@@ -35,24 +35,24 @@ namespace Apache.Arrow
public Field(string name, IArrowType dataType, bool nullable,
IEnumerable<KeyValuePair<string, string>> metadata = default)
- : this(name, dataType, nullable)
+ : this(name, dataType, nullable, false)
{
Metadata = metadata?.ToDictionary(kv => kv.Key, kv => kv.Value);
}
internal Field(string name, IArrowType dataType, bool nullable,
- IReadOnlyDictionary<string, string> metadata, bool copyCollections)
- : this(name, dataType, nullable)
+ IReadOnlyDictionary<string, string> metadata, bool
copyCollections, bool allowBlankName)
+ : this(name, dataType, nullable, allowBlankName)
{
Debug.Assert(copyCollections == false, "This internal constructor
is to not copy the collections.");
Metadata = metadata;
}
- private Field(string name, IArrowType dataType, bool nullable)
+ private Field(string name, IArrowType dataType, bool nullable, bool
allowBlankName)
{
- if (string.IsNullOrWhiteSpace(name))
+ if (name == null || (!allowBlankName &&
string.IsNullOrWhiteSpace(name)))
{
throw new ArgumentNullException(nameof(name));
}
diff --git a/csharp/src/Apache.Arrow/Interfaces/IArrowArrayBuilder.cs
b/csharp/src/Apache.Arrow/Interfaces/IArrowArrayBuilder.cs
index 12f9ec1fd3..b18ff4581a 100644
--- a/csharp/src/Apache.Arrow/Interfaces/IArrowArrayBuilder.cs
+++ b/csharp/src/Apache.Arrow/Interfaces/IArrowArrayBuilder.cs
@@ -37,6 +37,7 @@ namespace Apache.Arrow
TBuilder Reserve(int capacity);
TBuilder Resize(int length);
TBuilder Clear();
+ TBuilder AppendNull();
}
@@ -47,8 +48,7 @@ namespace Apache.Arrow
TBuilder Append(T value);
TBuilder Append(ReadOnlySpan<T> span);
TBuilder AppendRange(IEnumerable<T> values);
- TBuilder AppendNull();
TBuilder Swap(int i, int j);
TBuilder Set(int index, T value);
}
-}
\ No newline at end of file
+}
diff --git a/csharp/src/Apache.Arrow/Ipc/ArrowReaderImplementation.cs
b/csharp/src/Apache.Arrow/Ipc/ArrowReaderImplementation.cs
index 6261696207..c9c1b21673 100644
--- a/csharp/src/Apache.Arrow/Ipc/ArrowReaderImplementation.cs
+++ b/csharp/src/Apache.Arrow/Ipc/ArrowReaderImplementation.cs
@@ -257,7 +257,7 @@ namespace Apache.Arrow.Ipc
}
ArrowBuffer[] arrowBuff;
- if (field.DataType.TypeId == ArrowTypeId.Struct)
+ if (field.DataType.TypeId == ArrowTypeId.Struct ||
field.DataType.TypeId == ArrowTypeId.FixedSizeList)
{
arrowBuff = new[] { nullArrowBuffer };
}
diff --git a/csharp/src/Apache.Arrow/Ipc/ArrowStreamWriter.cs
b/csharp/src/Apache.Arrow/Ipc/ArrowStreamWriter.cs
index ca567b67e9..a5d8db3f50 100644
--- a/csharp/src/Apache.Arrow/Ipc/ArrowStreamWriter.cs
+++ b/csharp/src/Apache.Arrow/Ipc/ArrowStreamWriter.cs
@@ -50,6 +50,7 @@ namespace Apache.Arrow.Ipc
IArrowArrayVisitor<Time32Array>,
IArrowArrayVisitor<Time64Array>,
IArrowArrayVisitor<ListArray>,
+ IArrowArrayVisitor<FixedSizeListArray>,
IArrowArrayVisitor<StringArray>,
IArrowArrayVisitor<BinaryArray>,
IArrowArrayVisitor<FixedSizeBinaryArray>,
@@ -111,6 +112,13 @@ namespace Apache.Arrow.Ipc
array.Values.Accept(this);
}
+ public void Visit(FixedSizeListArray array)
+ {
+ _buffers.Add(CreateBuffer(array.NullBitmapBuffer));
+
+ array.Values.Accept(this);
+ }
+
public void Visit(StringArray array) => Visit(array as
BinaryArray);
public void Visit(BinaryArray array)
diff --git a/csharp/src/Apache.Arrow/Ipc/ArrowTypeFlatbufferBuilder.cs
b/csharp/src/Apache.Arrow/Ipc/ArrowTypeFlatbufferBuilder.cs
index befd87c2e0..203aa72d93 100644
--- a/csharp/src/Apache.Arrow/Ipc/ArrowTypeFlatbufferBuilder.cs
+++ b/csharp/src/Apache.Arrow/Ipc/ArrowTypeFlatbufferBuilder.cs
@@ -60,6 +60,7 @@ namespace Apache.Arrow.Ipc
IArrowTypeVisitor<BinaryType>,
IArrowTypeVisitor<TimestampType>,
IArrowTypeVisitor<ListType>,
+ IArrowTypeVisitor<FixedSizeListType>,
IArrowTypeVisitor<UnionType>,
IArrowTypeVisitor<StructType>,
IArrowTypeVisitor<Decimal128Type>,
@@ -110,6 +111,13 @@ namespace Apache.Arrow.Ipc
Flatbuf.List.EndList(Builder));
}
+ public void Visit(FixedSizeListType type)
+ {
+ Result = FieldType.Build(
+ Flatbuf.Type.FixedSizeList,
+ Flatbuf.FixedSizeList.CreateFixedSizeList(Builder,
type.ListSize));
+ }
+
public void Visit(UnionType type)
{
throw new NotImplementedException();
diff --git a/csharp/src/Apache.Arrow/Ipc/MessageSerializer.cs
b/csharp/src/Apache.Arrow/Ipc/MessageSerializer.cs
index f5031fbfb5..8ca69b6116 100644
--- a/csharp/src/Apache.Arrow/Ipc/MessageSerializer.cs
+++ b/csharp/src/Apache.Arrow/Ipc/MessageSerializer.cs
@@ -59,7 +59,7 @@ namespace Apache.Arrow.Ipc
for (int i = 0; i < schema.FieldsLength; i++)
{
Flatbuf.Field field = schema.Fields(i).GetValueOrDefault();
- fields.Add(FieldFromFlatbuffer(field, ref dictionaryMemo));
+ fields.Add(FieldFromFlatbuffer(field, ref dictionaryMemo,
allowBlankName: false));
}
Dictionary<string, string> metadata = schema.CustomMetadataLength
> 0 ? new Dictionary<string, string>() : null;
@@ -73,13 +73,14 @@ namespace Apache.Arrow.Ipc
return new Schema(fields, metadata, copyCollections: false);
}
- private static Field FieldFromFlatbuffer(Flatbuf.Field flatbufField,
ref DictionaryMemo dictionaryMemo)
+ private static Field FieldFromFlatbuffer(Flatbuf.Field flatbufField,
ref DictionaryMemo dictionaryMemo, bool allowBlankName)
{
+ bool allowBlankNameChild = flatbufField.ChildrenLength == 1 &&
flatbufField.TypeType == Flatbuf.Type.FixedSizeList;
Field[] childFields = flatbufField.ChildrenLength > 0 ? new
Field[flatbufField.ChildrenLength] : null;
for (int i = 0; i < flatbufField.ChildrenLength; i++)
{
Flatbuf.Field? childFlatbufField = flatbufField.Children(i);
- childFields[i] = FieldFromFlatbuffer(childFlatbufField.Value,
ref dictionaryMemo);
+ childFields[i] = FieldFromFlatbuffer(childFlatbufField.Value,
ref dictionaryMemo, allowBlankNameChild);
}
Flatbuf.DictionaryEncoding? dictionaryEncoding =
flatbufField.Dictionary;
@@ -103,7 +104,7 @@ namespace Apache.Arrow.Ipc
metadata[keyValue.Key] = keyValue.Value;
}
- var arrowField = new Field(flatbufField.Name, type,
flatbufField.Nullable, metadata, copyCollections: false);
+ var arrowField = new Field(flatbufField.Name, type,
flatbufField.Nullable, metadata, copyCollections: false, allowBlankName);
if (dictionaryEncoding.HasValue)
{
@@ -192,6 +193,13 @@ namespace Apache.Arrow.Ipc
throw new InvalidDataException($"List type must have
exactly one child.");
}
return new Types.ListType(childFields[0]);
+ case Flatbuf.Type.FixedSizeList:
+ if (childFields == null || childFields.Length != 1)
+ {
+ throw new InvalidDataException($"Fixed-size list type
must have exactly one child.");
+ }
+ Flatbuf.FixedSizeList fixedSizeListMetadata =
field.Type<Flatbuf.FixedSizeList>().Value;
+ return new Types.FixedSizeListType(childFields[0],
fixedSizeListMetadata.ListSize);
case Flatbuf.Type.Struct_:
Debug.Assert(childFields != null);
return new Types.StructType(childFields);
diff --git a/csharp/src/Apache.Arrow/Types/IArrowType.cs
b/csharp/src/Apache.Arrow/Types/FixedSizeListType.cs
similarity index 50%
copy from csharp/src/Apache.Arrow/Types/IArrowType.cs
copy to csharp/src/Apache.Arrow/Types/FixedSizeListType.cs
index 15c9a02445..5a4c70bf3a 100644
--- a/csharp/src/Apache.Arrow/Types/IArrowType.cs
+++ b/csharp/src/Apache.Arrow/Types/FixedSizeListType.cs
@@ -13,51 +13,32 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+using System;
namespace Apache.Arrow.Types
{
- public enum ArrowTypeId
+ public sealed class FixedSizeListType : NestedType
{
- Null,
- Boolean,
- UInt8,
- Int8,
- UInt16,
- Int16,
- UInt32,
- Int32,
- UInt64,
- Int64,
- HalfFloat,
- Float,
- Double,
- String,
- Binary,
- FixedSizedBinary,
- Date32,
- Date64,
- Timestamp,
- Time32,
- Time64,
- Interval,
- Decimal128,
- Decimal256,
- List,
- Struct,
- Union,
- Dictionary,
- Map
- }
+ public override ArrowTypeId TypeId => ArrowTypeId.FixedSizeList;
+ public override string Name => "fixed_size_list";
+ public int ListSize { get; }
- public interface IArrowType
- {
- ArrowTypeId TypeId { get; }
+ public Field ValueField => Fields[0];
+
+ public IArrowType ValueDataType => Fields[0].DataType;
+
+ public FixedSizeListType(Field valueField, int listSize)
+ : base(valueField)
+ {
+ if (listSize <= 0)
+ throw new ArgumentOutOfRangeException(nameof(listSize));
+
+ ListSize = listSize;
+ }
- string Name { get; }
-
- void Accept(IArrowTypeVisitor visitor);
+ public FixedSizeListType(IArrowType valueDataType, int listSize)
+ : this(new Field("item", valueDataType, true), listSize) { }
- bool IsFixedWidth { get; }
-
+ public override void Accept(IArrowTypeVisitor visitor) => Accept(this,
visitor);
}
}
diff --git a/csharp/src/Apache.Arrow/Types/IArrowType.cs
b/csharp/src/Apache.Arrow/Types/IArrowType.cs
index 15c9a02445..447db47329 100644
--- a/csharp/src/Apache.Arrow/Types/IArrowType.cs
+++ b/csharp/src/Apache.Arrow/Types/IArrowType.cs
@@ -46,7 +46,8 @@ namespace Apache.Arrow.Types
Struct,
Union,
Dictionary,
- Map
+ Map,
+ FixedSizeList,
}
public interface IArrowType
diff --git a/csharp/test/Apache.Arrow.IntegrationTest/IntegrationCommand.cs
b/csharp/test/Apache.Arrow.IntegrationTest/IntegrationCommand.cs
index 5b1ed98993..abf7451e5e 100644
--- a/csharp/test/Apache.Arrow.IntegrationTest/IntegrationCommand.cs
+++ b/csharp/test/Apache.Arrow.IntegrationTest/IntegrationCommand.cs
@@ -149,8 +149,20 @@ namespace Apache.Arrow.IntegrationTest
private static void CreateField(Field.Builder builder, JsonField
jsonField)
{
+ Field[] children = null;
+ if (jsonField.Children?.Count > 0)
+ {
+ children = new Field[jsonField.Children.Count];
+ for (int i = 0; i < jsonField.Children.Count; i++)
+ {
+ Field.Builder field = new Field.Builder();
+ CreateField(field, jsonField.Children[i]);
+ children[i] = field.Build();
+ }
+ }
+
builder.Name(jsonField.Name)
- .DataType(ToArrowType(jsonField.Type))
+ .DataType(ToArrowType(jsonField.Type, children))
.Nullable(jsonField.Nullable);
if (jsonField.Metadata != null)
@@ -159,7 +171,7 @@ namespace Apache.Arrow.IntegrationTest
}
}
- private static IArrowType ToArrowType(JsonArrowType type)
+ private static IArrowType ToArrowType(JsonArrowType type, Field[]
children)
{
return type.Name switch
{
@@ -173,6 +185,9 @@ namespace Apache.Arrow.IntegrationTest
"date" => ToDateArrowType(type),
"time" => ToTimeArrowType(type),
"timestamp" => ToTimestampArrowType(type),
+ "list" => ToListArrowType(type, children),
+ "fixedsizelist" => ToFixedSizeListArrowType(type, children),
+ "struct" => ToStructArrowType(type, children),
"null" => NullType.Default,
_ => throw new NotSupportedException($"JsonArrowType not
supported: {type.Name}")
};
@@ -251,6 +266,21 @@ namespace Apache.Arrow.IntegrationTest
};
}
+ private static IArrowType ToListArrowType(JsonArrowType type, Field[]
children)
+ {
+ return new ListType(children[0]);
+ }
+
+ private static IArrowType ToFixedSizeListArrowType(JsonArrowType type,
Field[] children)
+ {
+ return new FixedSizeListType(children[0], type.ListSize);
+ }
+
+ private static IArrowType ToStructArrowType(JsonArrowType type,
Field[] children)
+ {
+ return new StructType(children);
+ }
+
private class ArrayCreator :
IArrowTypeVisitor<BooleanType>,
IArrowTypeVisitor<Int8Type>,
@@ -274,10 +304,11 @@ namespace Apache.Arrow.IntegrationTest
IArrowTypeVisitor<BinaryType>,
IArrowTypeVisitor<FixedSizeBinaryType>,
IArrowTypeVisitor<ListType>,
+ IArrowTypeVisitor<FixedSizeListType>,
IArrowTypeVisitor<StructType>,
IArrowTypeVisitor<NullType>
{
- private JsonFieldData JsonFieldData { get; }
+ private JsonFieldData JsonFieldData { get; set; }
public IArrowArray Array { get; private set; }
public ArrayCreator(JsonFieldData jsonFieldData)
@@ -478,12 +509,51 @@ namespace Apache.Arrow.IntegrationTest
public void Visit(ListType type)
{
- throw new NotImplementedException();
+ ArrowBuffer validityBuffer = GetValidityBuffer(out int
nullCount);
+ ArrowBuffer offsetBuffer = GetOffsetBuffer();
+
+ var data = JsonFieldData;
+ JsonFieldData = data.Children[0];
+ type.ValueDataType.Accept(this);
+ JsonFieldData = data;
+
+ ArrayData arrayData = new ArrayData(type, JsonFieldData.Count,
nullCount, 0,
+ new[] { validityBuffer, offsetBuffer }, new[] { Array.Data
});
+ Array = new ListArray(arrayData);
+ }
+
+ public void Visit(FixedSizeListType type)
+ {
+ ArrowBuffer validityBuffer = GetValidityBuffer(out int
nullCount);
+
+ var data = JsonFieldData;
+ JsonFieldData = data.Children[0];
+ type.ValueDataType.Accept(this);
+ JsonFieldData = data;
+
+ ArrayData arrayData = new ArrayData(type, JsonFieldData.Count,
nullCount, 0,
+ new[] { validityBuffer }, new[] { Array.Data });
+ Array = new FixedSizeListArray(arrayData);
}
public void Visit(StructType type)
{
- throw new NotImplementedException();
+ ArrowBuffer validityBuffer = GetValidityBuffer(out int
nullCount);
+
+ ArrayData[] children = new ArrayData[type.Fields.Count];
+
+ var data = JsonFieldData;
+ for (int i = 0; i < children.Length; i++)
+ {
+ JsonFieldData = data.Children[i];
+ type.Fields[i].DataType.Accept(this);
+ children[i] = Array.Data;
+ }
+ JsonFieldData = data;
+
+ ArrayData arrayData = new ArrayData(type, JsonFieldData.Count,
nullCount, 0,
+ new[] { validityBuffer }, children);
+ Array = new StructArray(arrayData);
}
private static byte[] ConvertHexStringToByteArray(string hexString)
diff --git a/csharp/test/Apache.Arrow.IntegrationTest/JsonFile.cs
b/csharp/test/Apache.Arrow.IntegrationTest/JsonFile.cs
index f074afc010..f0f63d3e19 100644
--- a/csharp/test/Apache.Arrow.IntegrationTest/JsonFile.cs
+++ b/csharp/test/Apache.Arrow.IntegrationTest/JsonFile.cs
@@ -68,6 +68,9 @@ namespace Apache.Arrow.IntegrationTest
// FixedSizeBinary fields
public int ByteWidth { get; set; }
+ // FixedSizeList fields
+ public int ListSize { get; set; }
+
[JsonExtensionData]
public Dictionary<string, JsonElement> ExtensionData { get; set; }
}
diff --git a/csharp/test/Apache.Arrow.Tests/ArrayBuilderTests.cs
b/csharp/test/Apache.Arrow.Tests/ArrayBuilderTests.cs
index 2568e5e8bd..f789d6a3d3 100644
--- a/csharp/test/Apache.Arrow.Tests/ArrayBuilderTests.cs
+++ b/csharp/test/Apache.Arrow.Tests/ArrayBuilderTests.cs
@@ -203,6 +203,132 @@ namespace Apache.Arrow.Tests
((Int64Array)childList3.GetSlicedValues(0)).ToList());
}
+ [Fact]
+ public void FixedSizeListArrayBuilder()
+ {
+ var listBuilder = new
FixedSizeListArray.Builder(StringType.Default, 4);
+ var valueBuilder = listBuilder.ValueBuilder as StringArray.Builder;
+ Assert.NotNull(valueBuilder);
+ listBuilder.Append();
+
+ Assert.Throws<ArgumentOutOfRangeException>(listBuilder.Append);
+
+ valueBuilder.Append("1");
+ valueBuilder.Append("2");
+ valueBuilder.Append("3");
+ valueBuilder.Append("4");
+ listBuilder.AppendNull();
+ listBuilder.Append();
+ valueBuilder.Append("5").Append("6").Append("7").Append("8");
+ listBuilder.Append();
+
valueBuilder.Append("444").AppendNull().Append("555").Append("666");
+
+ var list = listBuilder.Build();
+
+ Assert.Equal(
+ new List<string> { "1", "2", "3", "4" },
+ ConvertStringArrayToList(list.GetSlicedValues(0) as
StringArray));
+ Assert.Null(list.GetSlicedValues(1));
+ Assert.Equal(
+ new List<string> { "5", "6", "7", "8" },
+ ConvertStringArrayToList(list.GetSlicedValues(2) as
StringArray));
+ Assert.Equal(
+ new List<string> { "444", null, "555", "666" },
+ ConvertStringArrayToList(list.GetSlicedValues(3) as
StringArray));
+
+ Assert.Throws<ArgumentOutOfRangeException>(() =>
list.GetSlicedValues(-1));
+ Assert.Throws<ArgumentOutOfRangeException>(() =>
list.GetSlicedValues(4));
+
+ listBuilder.Resize(3);
+ var truncatedList = listBuilder.Build();
+
+ Assert.Equal(
+ new List<string> { "5", "6", "7", "8" },
+ ConvertStringArrayToList(truncatedList.GetSlicedValues(2) as
StringArray));
+
+ Assert.Throws<ArgumentOutOfRangeException>(() =>
truncatedList.GetSlicedValues(-1));
+ Assert.Throws<ArgumentOutOfRangeException>(() =>
truncatedList.GetSlicedValues(3));
+
+ listBuilder.Clear();
+ var emptyList = listBuilder.Build();
+
+ Assert.Equal(0, emptyList.Length);
+
+ List<string> ConvertStringArrayToList(StringArray array)
+ {
+ var length = array.Length;
+ var resultList = new List<string>(length);
+ for (var index = 0; index < length; index++)
+ {
+ resultList.Add(array.GetString(index));
+ }
+ return resultList;
+ }
+ }
+
+ [Fact]
+ public void FixedSizeListArrayBuilderValidityBuffer()
+ {
+ FixedSizeListArray.Builder builder = new
FixedSizeListArray.Builder(Int64Type.Default, 2).Append();
+ ((Int64Array.Builder)builder.ValueBuilder).Append(1).Append(2);
+ FixedSizeListArray listArray = builder.AppendNull().Build();
+ Assert.False(listArray.IsValid(2));
+ }
+
+ [Fact]
+ public void NestedFixedSizeListArrayBuilder()
+ {
+ var childListType = new FixedSizeListType(Int64Type.Default, 2);
+ var parentListBuilder = new
FixedSizeListArray.Builder(childListType, 3);
+ var childListBuilder = parentListBuilder.ValueBuilder as
FixedSizeListArray.Builder;
+ Assert.NotNull(childListBuilder);
+ var valueBuilder = childListBuilder.ValueBuilder as
Int64Array.Builder;
+ Assert.NotNull(valueBuilder);
+
+ parentListBuilder.Append();
+ childListBuilder.Append();
+ valueBuilder.Append(1);
+ valueBuilder.Append(2);
+ childListBuilder.Append();
+ valueBuilder.Append(3).Append(4);
+ childListBuilder.AppendNull();
+ parentListBuilder.Append();
+ childListBuilder.Append();
+ valueBuilder.Append(4).Append(5);
+ childListBuilder.Append();
+ valueBuilder.Append(6).Append(7);
+ childListBuilder.Append();
+ valueBuilder.Append(8).Append(9);
+ parentListBuilder.Append();
+ childListBuilder.Append();
+ valueBuilder.Append(10).Append(11);
+ childListBuilder.AppendNull();
+ childListBuilder.Append();
+ valueBuilder.Append(12).AppendNull();
+
+ var parentList = parentListBuilder.Build();
+
+ var childList1 = (FixedSizeListArray)parentList.GetSlicedValues(0);
+ var childList2 = (FixedSizeListArray)parentList.GetSlicedValues(1);
+ var childList3 = (FixedSizeListArray)parentList.GetSlicedValues(2);
+
+ Assert.Equal(3, childList1.Length);
+ Assert.Equal(3, childList2.Length);
+ Assert.Equal(3, childList3.Length);
+ Assert.Equal(
+ new List<long?> { 1, 2 },
+ ((Int64Array)childList1.GetSlicedValues(0)).ToList());
+ Assert.Equal(
+ new List<long?> { 3, 4 },
+ ((Int64Array)childList1.GetSlicedValues(1)).ToList());
+ Assert.Equal(
+ new List<long?> { 4, 5 },
+ ((Int64Array)childList2.GetSlicedValues(0)).ToList());
+ Assert.Equal(
+ new List<long?> { 12, null },
+
((Int64Array)childList3.GetSlicedValues(2)).ToList(includeNulls: true));
+ }
+
public class TimestampArrayBuilder
{
[Fact]
diff --git a/csharp/test/Apache.Arrow.Tests/ArrayTypeComparer.cs
b/csharp/test/Apache.Arrow.Tests/ArrayTypeComparer.cs
index f75111b66d..77584aefb1 100644
--- a/csharp/test/Apache.Arrow.Tests/ArrayTypeComparer.cs
+++ b/csharp/test/Apache.Arrow.Tests/ArrayTypeComparer.cs
@@ -27,6 +27,7 @@ namespace Apache.Arrow.Tests
IArrowTypeVisitor<Time64Type>,
IArrowTypeVisitor<FixedSizeBinaryType>,
IArrowTypeVisitor<ListType>,
+ IArrowTypeVisitor<FixedSizeListType>,
IArrowTypeVisitor<StructType>
{
private readonly IArrowType _expectedType;
@@ -95,6 +96,16 @@ namespace Apache.Arrow.Tests
CompareNested(expectedType, actualType);
}
+ public void Visit(FixedSizeListType actualType)
+ {
+ Assert.IsAssignableFrom<FixedSizeListType>(_expectedType);
+ var expectedType = (FixedSizeListType)_expectedType;
+
+ Assert.Equal(expectedType.ListSize, actualType.ListSize);
+
+ CompareNested(expectedType, actualType);
+ }
+
public void Visit(StructType actualType)
{
Assert.IsAssignableFrom<StructType>(_expectedType);
diff --git a/csharp/test/Apache.Arrow.Tests/ArrowArrayConcatenatorTests.cs
b/csharp/test/Apache.Arrow.Tests/ArrowArrayConcatenatorTests.cs
index 6b3277ed57..36cffe7eb4 100644
--- a/csharp/test/Apache.Arrow.Tests/ArrowArrayConcatenatorTests.cs
+++ b/csharp/test/Apache.Arrow.Tests/ArrowArrayConcatenatorTests.cs
@@ -76,6 +76,7 @@ namespace Apache.Arrow.Tests
new
Field.Builder().Name("Strings").DataType(StringType.Default).Nullable(true).Build(),
new
Field.Builder().Name("Ints").DataType(Int32Type.Default).Nullable(true).Build()
}),
+ new FixedSizeListType(Int32Type.Default, 1),
};
foreach (IArrowType type in targetTypes)
@@ -117,6 +118,7 @@ namespace Apache.Arrow.Tests
IArrowTypeVisitor<Date64Type>,
IArrowTypeVisitor<TimestampType>,
IArrowTypeVisitor<ListType>,
+ IArrowTypeVisitor<FixedSizeListType>,
IArrowTypeVisitor<StructType>
{
@@ -324,6 +326,42 @@ namespace Apache.Arrow.Tests
ExpectedArray = resultBuilder.Build();
}
+ public void Visit(FixedSizeListType type)
+ {
+ FixedSizeListArray.Builder resultBuilder = new
FixedSizeListArray.Builder(type.ValueDataType,
type.ListSize).Reserve(_baseDataTotalElementCount);
+ //Todo : Support various types
+ Int32Array.Builder resultValueBuilder =
(Int32Array.Builder)resultBuilder.ValueBuilder.Reserve(_baseDataTotalElementCount);
+
+ for (int i = 0; i < _baseDataListCount; i++)
+ {
+ List<int?> dataList = _baseData[i];
+
+ FixedSizeListArray.Builder builder = new
FixedSizeListArray.Builder(type.ValueField,
type.ListSize).Reserve(dataList.Count);
+ Int32Array.Builder valueBuilder =
(Int32Array.Builder)builder.ValueBuilder.Reserve(dataList.Count);
+
+ foreach (int? value in dataList)
+ {
+ if (value.HasValue)
+ {
+ builder.Append();
+ resultBuilder.Append();
+
+ valueBuilder.Append(value.Value);
+ resultValueBuilder.Append(value.Value);
+ }
+ else
+ {
+ builder.AppendNull();
+ resultBuilder.AppendNull();
+ }
+ }
+
+ TestTargetArrayList.Add(builder.Build());
+ }
+
+ ExpectedArray = resultBuilder.Build();
+ }
+
public void Visit(StructType type)
{
// TODO: Make data from type fields.
diff --git a/csharp/test/Apache.Arrow.Tests/ArrowReaderVerifier.cs
b/csharp/test/Apache.Arrow.Tests/ArrowReaderVerifier.cs
index 543b446bba..e588eab51e 100644
--- a/csharp/test/Apache.Arrow.Tests/ArrowReaderVerifier.cs
+++ b/csharp/test/Apache.Arrow.Tests/ArrowReaderVerifier.cs
@@ -86,6 +86,7 @@ namespace Apache.Arrow.Tests
IArrowArrayVisitor<Time32Array>,
IArrowArrayVisitor<Time64Array>,
IArrowArrayVisitor<ListArray>,
+ IArrowArrayVisitor<FixedSizeListArray>,
IArrowArrayVisitor<StringArray>,
IArrowArrayVisitor<FixedSizeBinaryArray>,
IArrowArrayVisitor<BinaryArray>,
@@ -126,6 +127,7 @@ namespace Apache.Arrow.Tests
public void Visit(Time32Array array) => CompareArrays(array);
public void Visit(Time64Array array) => CompareArrays(array);
public void Visit(ListArray array) => CompareArrays(array);
+ public void Visit(FixedSizeListArray array) =>
CompareArrays(array);
public void Visit(FixedSizeBinaryArray array) =>
CompareArrays(array);
public void Visit(Decimal128Array array) => CompareArrays(array);
public void Visit(Decimal256Array array) => CompareArrays(array);
@@ -304,13 +306,29 @@ namespace Apache.Arrow.Tests
}
else
{
- int offsetsLength = (expectedArray.Length + 1) * 4;
+ int offsetsLength = (expectedArray.Length + 1) *
sizeof(int);
Assert.True(expectedArray.ValueOffsetsBuffer.Span.Slice(0,
offsetsLength).SequenceEqual(actualArray.ValueOffsetsBuffer.Span.Slice(0,
offsetsLength)));
}
actualArray.Values.Accept(new
ArrayComparer(expectedArray.Values, _strictCompare));
}
+ private void CompareArrays(FixedSizeListArray actualArray)
+ {
+ Assert.IsAssignableFrom<FixedSizeListArray>(_expectedArray);
+ FixedSizeListArray expectedArray =
(FixedSizeListArray)_expectedArray;
+
+ actualArray.Data.DataType.Accept(_arrayTypeComparer);
+
+ Assert.Equal(expectedArray.Length, actualArray.Length);
+ Assert.Equal(expectedArray.NullCount, actualArray.NullCount);
+ Assert.Equal(expectedArray.Offset, actualArray.Offset);
+
+ CompareValidityBuffer(expectedArray.NullCount,
_expectedArray.Length, expectedArray.NullBitmapBuffer,
actualArray.NullBitmapBuffer);
+
+ actualArray.Values.Accept(new
ArrayComparer(expectedArray.Values, _strictCompare));
+ }
+
private void CompareValidityBuffer(int nullCount, int arrayLength,
ArrowBuffer expectedValidityBuffer, ArrowBuffer actualValidityBuffer)
{
if (_strictCompare)
diff --git a/csharp/test/Apache.Arrow.Tests/CDataInterfacePythonTests.cs
b/csharp/test/Apache.Arrow.Tests/CDataInterfacePythonTests.cs
index 4c53b98e3d..29b1b9e7db 100644
--- a/csharp/test/Apache.Arrow.Tests/CDataInterfacePythonTests.cs
+++ b/csharp/test/Apache.Arrow.Tests/CDataInterfacePythonTests.cs
@@ -106,6 +106,8 @@ namespace Apache.Arrow.Tests
.Field(f => f.Name("list_string").DataType(new
ListType(StringType.Default)).Nullable(false))
.Field(f => f.Name("list_list_i32").DataType(new
ListType(new ListType(Int32Type.Default))).Nullable(false))
+ .Field(f => f.Name("fixed_length_list_i64").DataType(new
FixedSizeListType(Int64Type.Default, 10)).Nullable(true))
+
.Field(f => f.Name("dict_string").DataType(new
DictionaryType(Int32Type.Default, StringType.Default, false)).Nullable(false))
.Field(f => f.Name("dict_string_ordered").DataType(new
DictionaryType(Int32Type.Default, StringType.Default, true)).Nullable(false))
.Field(f => f.Name("list_dict_string").DataType(new
ListType(new DictionaryType(Int32Type.Default, StringType.Default,
false))).Nullable(false))
@@ -164,6 +166,8 @@ namespace Apache.Arrow.Tests
yield return pa.field("list_string", pa.list_(pa.utf8()),
false);
yield return pa.field("list_list_i32",
pa.list_(pa.list_(pa.int32())), false);
+ yield return pa.field("fixed_length_list_i64",
pa.list_(pa.int64(), 10), true);
+
yield return pa.field("dict_string", pa.dictionary(pa.int32(),
pa.utf8(), false), false);
yield return pa.field("dict_string_ordered",
pa.dictionary(pa.int32(), pa.utf8(), true), false);
yield return pa.field("list_dict_string",
pa.list_(pa.dictionary(pa.int32(), pa.utf8(), false)), false);
@@ -492,8 +496,11 @@ namespace Apache.Arrow.Tests
pa.array(List(1, 0, 1, 1, null)),
pa.array(List("foo", "bar"))
),
+ pa.FixedSizeListArray.from_arrays(
+ pa.array(List(1, 2, 3, 4, null, 6, 7, null, null,
null)),
+ 2),
}),
- new[] { "col1", "col2", "col3", "col4", "col5", "col6",
"col7" });
+ new[] { "col1", "col2", "col3", "col4", "col5", "col6",
"col7", "col8" });
dynamic batch = table.to_batches()[0];
@@ -554,6 +561,13 @@ namespace Apache.Arrow.Tests
Assert.Equal(2, col7b.Length);
Assert.Equal("foo", col7b.GetString(0));
Assert.Equal("bar", col7b.GetString(1));
+
+ FixedSizeListArray col8 =
(FixedSizeListArray)recordBatch.Column("col8");
+ Assert.Equal(5, col8.Length);
+ Int64Array col8a = (Int64Array)col8.Values;
+ Assert.Equal(new long[] { 1, 2, 3, 4, 0, 6, 7, 0, 0, 0 },
col8a.Values.ToArray());
+ Assert.True(col8a.IsValid(3));
+ Assert.False(col8a.IsValid(9));
}
[SkippableFact]
diff --git a/csharp/test/Apache.Arrow.Tests/TableTests.cs
b/csharp/test/Apache.Arrow.Tests/TableTests.cs
index 45a14cc256..b4c4b1faed 100644
--- a/csharp/test/Apache.Arrow.Tests/TableTests.cs
+++ b/csharp/test/Apache.Arrow.Tests/TableTests.cs
@@ -60,7 +60,7 @@ namespace Apache.Arrow.Tests
Table table1 = Table.TableFromRecordBatches(recordBatch1.Schema,
recordBatches);
Assert.Equal(20, table1.RowCount);
- Assert.Equal(23, table1.ColumnCount);
+ Assert.Equal(24, table1.ColumnCount);
FixedSizeBinaryType type = new FixedSizeBinaryType(17);
Field newField1 = new Field(type.Name, type, false);
diff --git a/csharp/test/Apache.Arrow.Tests/TestData.cs
b/csharp/test/Apache.Arrow.Tests/TestData.cs
index 96c6fafee2..41507311f6 100644
--- a/csharp/test/Apache.Arrow.Tests/TestData.cs
+++ b/csharp/test/Apache.Arrow.Tests/TestData.cs
@@ -59,6 +59,7 @@ namespace Apache.Arrow.Tests
{
builder.Field(CreateField(new
DictionaryType(Int32Type.Default, StringType.Default, false), i));
builder.Field(CreateField(new FixedSizeBinaryType(16), i));
+ builder.Field(CreateField(new
FixedSizeListType(Int32Type.Default, 3), i));
}
//builder.Field(CreateField(HalfFloatType.Default));
@@ -122,6 +123,7 @@ namespace Apache.Arrow.Tests
IArrowTypeVisitor<TimestampType>,
IArrowTypeVisitor<StringType>,
IArrowTypeVisitor<ListType>,
+ IArrowTypeVisitor<FixedSizeListType>,
IArrowTypeVisitor<StructType>,
IArrowTypeVisitor<Decimal128Type>,
IArrowTypeVisitor<Decimal256Type>,
@@ -270,6 +272,32 @@ namespace Apache.Arrow.Tests
Array = builder.Build();
}
+ public void Visit(FixedSizeListType type)
+ {
+ var builder = new FixedSizeListArray.Builder(type.ValueField,
type.ListSize).Reserve(Length);
+
+ //Todo : Support various types
+ var valueBuilder = (Int32Array.Builder)builder.ValueBuilder;
+
+ for (var i = 0; i < Length; i++)
+ {
+ if (type.Fields[0].IsNullable && (i % 3) == 0)
+ {
+ builder.AppendNull();
+ }
+ else
+ {
+ builder.Append();
+ for (var j = 0; j < type.ListSize; j++)
+ {
+ valueBuilder.Append(i * type.ListSize + j);
+ }
+ }
+ }
+
+ Array = builder.Build();
+ }
+
public void Visit(StructType type)
{
IArrowArray[] childArrays = new IArrowArray[type.Fields.Count];
diff --git a/dev/archery/archery/integration/datagen.py
b/dev/archery/archery/integration/datagen.py
index 11cb02a9f4..99aeda5248 100644
--- a/dev/archery/archery/integration/datagen.py
+++ b/dev/archery/archery/integration/datagen.py
@@ -1702,11 +1702,9 @@ def get_generated_json_files(tempdir=None):
.skip_category('Java') # TODO(ARROW-8715)
.skip_category('JS'), # TODO(ARROW-8716)
- generate_nested_case()
- .skip_category('C#'),
+ generate_nested_case(),
- generate_recursive_nested_case()
- .skip_category('C#'),
+ generate_recursive_nested_case(),
generate_nested_large_offsets_case()
.skip_category('C#')
diff --git a/docs/source/status.rst b/docs/source/status.rst
index fbbe1cede9..6d7f674932 100644
--- a/docs/source/status.rst
+++ b/docs/source/status.rst
@@ -73,7 +73,7 @@ Data Types
| Data type | C++ | Java | Go | JavaScript | C# | Rust |
Julia | Swift |
| (nested) | | | | | | |
| |
+===================+=======+=======+=======+============+=======+=======+=======+=======+
-| Fixed Size List | ✓ | ✓ | ✓ | ✓ | | ✓ | ✓
| |
+| Fixed Size List | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓
| |
+-------------------+-------+-------+-------+------------+-------+-------+-------+-------+
| List | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓
| |
+-------------------+-------+-------+-------+------------+-------+-------+-------+-------+
