This is an automated email from the ASF dual-hosted git repository.
curth pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-dotnet.git
The following commit(s) were added to refs/heads/main by this push:
new 1acbca6 Add missing builders and concatenator support for "large"
array types. (#302)
1acbca6 is described below
commit 1acbca65efb0fe949b665c34e82a08a27ef2d7ff
Author: Curt Hagenlocher <[email protected]>
AuthorDate: Mon Mar 30 17:48:21 2026 -0700
Add missing builders and concatenator support for "large" array types.
(#302)
## What's Changed
When support for largebinary, largeutf8 and largelist were added,
corresponding support for builders and concatenators were not added.
---
src/Apache.Arrow/Arrays/ArrayDataConcatenator.cs | 93 +++++++++++
.../Arrays/ArrowArrayBuilderFactory.cs | 6 +
src/Apache.Arrow/Arrays/LargeBinaryArray.cs | 170 +++++++++++++++++++++
src/Apache.Arrow/Arrays/LargeListArray.cs | 85 +++++++++++
src/Apache.Arrow/Arrays/LargeStringArray.cs | 31 ++++
src/Apache.Arrow/Arrays/ListArray.cs | 1 +
src/Apache.Arrow/Arrays/ListViewArray.cs | 2 +-
src/Apache.Arrow/Arrays/MapArray.cs | 1 +
test/Apache.Arrow.Tests/ArrayBuilderTests.cs | 57 +++++++
.../ArrowArrayConcatenatorTests.cs | 33 ++++
.../LargeBinaryArrayBuilderTests.cs | 120 +++++++++++++++
.../LargeListArrayBuilderTests.cs | 131 ++++++++++++++++
.../LargeStringArrayBuilderTests.cs | 84 ++++++++++
13 files changed, 813 insertions(+), 1 deletion(-)
diff --git a/src/Apache.Arrow/Arrays/ArrayDataConcatenator.cs
b/src/Apache.Arrow/Arrays/ArrayDataConcatenator.cs
index 609ebb9..a835435 100644
--- a/src/Apache.Arrow/Arrays/ArrayDataConcatenator.cs
+++ b/src/Apache.Arrow/Arrays/ArrayDataConcatenator.cs
@@ -55,6 +55,9 @@ namespace Apache.Arrow
IArrowTypeVisitor<FixedSizeListType>,
IArrowTypeVisitor<StructType>,
IArrowTypeVisitor<UnionType>,
+ IArrowTypeVisitor<LargeBinaryType>,
+ IArrowTypeVisitor<LargeStringType>,
+ IArrowTypeVisitor<LargeListType>,
IArrowTypeVisitor<MapType>
{
public ArrayData Result { get; private set; }
@@ -101,6 +104,12 @@ namespace Apache.Arrow
public void Visit(StringViewType type) =>
ConcatenateBinaryViewArrayData(type);
+ public void Visit(LargeBinaryType type) =>
ConcatenateLargeVariableBinaryArrayData(type);
+
+ public void Visit(LargeStringType type) =>
ConcatenateLargeVariableBinaryArrayData(type);
+
+ public void Visit(LargeListType type) =>
ConcatenateLargeLists(type);
+
public void Visit(ListType type) => ConcatenateLists(type);
public void Visit(ListViewType type)
@@ -310,6 +319,90 @@ namespace Apache.Arrow
Result = new ArrayData(type, _totalLength, _totalNullCount, 0,
new ArrowBuffer[] { validityBuffer, offsetBuffer }, new[] { combinedChild });
}
+ private void ConcatenateLargeVariableBinaryArrayData(IArrowType
type)
+ {
+ CheckData(type, 3);
+ ArrowBuffer validityBuffer = ConcatenateValidityBuffer();
+ ArrowBuffer offsetBuffer = ConcatenateLargeOffsetBuffer();
+ ArrowBuffer valueBuffer =
ConcatenateLargeVariableBinaryValueBuffer();
+
+ Result = new ArrayData(type, _totalLength, _totalNullCount, 0,
new ArrowBuffer[] { validityBuffer, offsetBuffer, valueBuffer });
+ }
+
+ private void ConcatenateLargeLists(LargeListType type)
+ {
+ CheckData(type, 2);
+ ArrowBuffer validityBuffer = ConcatenateValidityBuffer();
+ ArrowBuffer offsetBuffer = ConcatenateLargeOffsetBuffer();
+
+ var children = new List<ArrayData>(_arrayDataList.Count);
+ foreach (ArrayData arrayData in _arrayDataList)
+ {
+ if (arrayData.Length == 0)
+ {
+ continue;
+ }
+
+ var child = arrayData.Children[0];
+ ReadOnlySpan<long> offsets =
arrayData.Buffers[1].Span.CastTo<long>().Slice(arrayData.Offset,
arrayData.Length + 1);
+ var firstOffset = offsets[0];
+ var lastOffset = offsets[arrayData.Length];
+ if (firstOffset != 0 || lastOffset != child.Length)
+ {
+ child = child.Slice(checked((int)firstOffset),
checked((int)(lastOffset - firstOffset)));
+ }
+
+ children.Add(child);
+ }
+
+ ArrayData combinedChild = Concatenate(children, _allocator);
+
+ Result = new ArrayData(type, _totalLength, _totalNullCount, 0,
new ArrowBuffer[] { validityBuffer, offsetBuffer }, new[] { combinedChild });
+ }
+
+ private ArrowBuffer ConcatenateLargeOffsetBuffer()
+ {
+ var builder = new ArrowBuffer.Builder<long>(_totalLength + 1);
+ long baseOffset = 0;
+
+ foreach (ArrayData arrayData in _arrayDataList)
+ {
+ if (arrayData.Length == 0)
+ {
+ continue;
+ }
+
+ ReadOnlySpan<long> span =
arrayData.Buffers[1].Span.CastTo<long>().Slice(arrayData.Offset,
arrayData.Length + 1);
+ var firstOffset = span[0];
+
+ foreach (long offset in span.Slice(0, arrayData.Length))
+ {
+ builder.Append(baseOffset + offset - firstOffset);
+ }
+
+ baseOffset += span[arrayData.Length] - firstOffset;
+ }
+
+ builder.Append(baseOffset);
+
+ return builder.Build(_allocator);
+ }
+
+ private ArrowBuffer ConcatenateLargeVariableBinaryValueBuffer()
+ {
+ var builder = new ArrowBuffer.Builder<byte>();
+
+ foreach (ArrayData arrayData in _arrayDataList)
+ {
+ var offsets =
arrayData.Buffers[1].Span.CastTo<long>().Slice(arrayData.Offset,
arrayData.Length + 1);
+ var firstOffset = checked((int)offsets[0]);
+ var lastOffset = checked((int)offsets[arrayData.Length]);
+
builder.Append(arrayData.Buffers[2].Span.Slice(firstOffset, lastOffset -
firstOffset));
+ }
+
+ return builder.Build(_allocator);
+ }
+
private ArrowBuffer ConcatenateValidityBuffer()
{
if (_totalNullCount == 0)
diff --git a/src/Apache.Arrow/Arrays/ArrowArrayBuilderFactory.cs
b/src/Apache.Arrow/Arrays/ArrowArrayBuilderFactory.cs
index f0afb93..f4f551c 100644
--- a/src/Apache.Arrow/Arrays/ArrowArrayBuilderFactory.cs
+++ b/src/Apache.Arrow/Arrays/ArrowArrayBuilderFactory.cs
@@ -95,6 +95,12 @@ namespace Apache.Arrow
IntervalUnit.MonthDayNanosecond => new
MonthDayNanosecondIntervalArray.Builder(),
_ => throw new
ArgumentOutOfRangeException($"unsupported interval unit <{intervalType.Unit}>")
};
+ case ArrowTypeId.LargeBinary:
+ return new LargeBinaryArray.Builder();
+ case ArrowTypeId.LargeString:
+ return new LargeStringArray.Builder();
+ case ArrowTypeId.LargeList:
+ return new LargeListArray.Builder(dataType as
LargeListType);
case ArrowTypeId.Map:
return new MapArray.Builder(dataType as MapType);
case ArrowTypeId.Struct:
diff --git a/src/Apache.Arrow/Arrays/LargeBinaryArray.cs
b/src/Apache.Arrow/Arrays/LargeBinaryArray.cs
index 1219466..09e1ad8 100644
--- a/src/Apache.Arrow/Arrays/LargeBinaryArray.cs
+++ b/src/Apache.Arrow/Arrays/LargeBinaryArray.cs
@@ -17,12 +17,182 @@ using System;
using System.Collections;
using System.Collections.Generic;
using System.Runtime.CompilerServices;
+using Apache.Arrow.Memory;
using Apache.Arrow.Types;
namespace Apache.Arrow;
public class LargeBinaryArray : Array, IReadOnlyList<byte[]>,
ICollection<byte[]>
{
+ public class Builder : BuilderBase<LargeBinaryArray, Builder>
+ {
+ public Builder() : base(LargeBinaryType.Default) { }
+ public Builder(IArrowType dataType) : base(dataType) { }
+
+ protected override LargeBinaryArray Build(ArrayData data)
+ {
+ return new LargeBinaryArray(data);
+ }
+ }
+
+ public abstract class BuilderBase<TArray, TBuilder> :
IArrowArrayBuilder<byte, TArray, TBuilder>
+ where TArray : IArrowArray
+ where TBuilder : class, IArrowArrayBuilder<byte, TArray, TBuilder>
+ {
+ protected IArrowType DataType { get; }
+ protected TBuilder Instance => this as TBuilder;
+ protected ArrowBuffer.Builder<long> ValueOffsets { get; }
+ protected ArrowBuffer.Builder<byte> ValueBuffer { get; }
+ protected ArrowBuffer.BitmapBuilder ValidityBuffer { get; }
+ protected long Offset { get; set; }
+ protected int NullCount => this.ValidityBuffer.UnsetBitCount;
+
+ protected BuilderBase(IArrowType dataType)
+ {
+ DataType = dataType;
+ ValueOffsets = new ArrowBuffer.Builder<long>();
+ ValueBuffer = new ArrowBuffer.Builder<byte>();
+ ValidityBuffer = new ArrowBuffer.BitmapBuilder();
+ ValueOffsets.Append(this.Offset);
+ }
+
+ protected abstract TArray Build(ArrayData data);
+
+ public int Length => ValueOffsets.Length - 1;
+
+ public TArray Build(MemoryAllocator allocator = default)
+ {
+ var bufs = new[]
+ {
+ NullCount > 0 ? ValidityBuffer.Build(allocator) :
ArrowBuffer.Empty,
+ ValueOffsets.Build(allocator),
+ ValueBuffer.Build(allocator),
+ };
+ var data = new ArrayData(
+ DataType,
+ length: Length,
+ NullCount,
+ offset: 0,
+ bufs);
+
+ return Build(data);
+ }
+
+ public TBuilder AppendNull()
+ {
+ ValidityBuffer.Append(false);
+ ValueOffsets.Append(Offset);
+ return Instance;
+ }
+
+ public TBuilder Append(byte value)
+ {
+ ValueBuffer.Append(value);
+ ValidityBuffer.Append(true);
+ Offset++;
+ ValueOffsets.Append(Offset);
+ return Instance;
+ }
+
+ public TBuilder Append(ReadOnlySpan<byte> span)
+ {
+ ValueBuffer.Append(span);
+ ValidityBuffer.Append(true);
+ Offset += span.Length;
+ ValueOffsets.Append(Offset);
+ return Instance;
+ }
+
+ public TBuilder Append(IEnumerable<byte> value)
+ {
+ if (value == null)
+ {
+ return AppendNull();
+ }
+
+ long priorLength = ValueBuffer.Length;
+ ValueBuffer.AppendRange(value);
+ long valueLength = ValueBuffer.Length - priorLength;
+ Offset += valueLength;
+ ValidityBuffer.Append(true);
+ ValueOffsets.Append(Offset);
+ return Instance;
+ }
+
+ public TBuilder AppendRange(IEnumerable<byte> values)
+ {
+ if (values == null)
+ {
+ throw new ArgumentNullException(nameof(values));
+ }
+
+ foreach (byte b in values)
+ {
+ Append(b);
+ }
+
+ return Instance;
+ }
+
+ public TBuilder AppendRange(IEnumerable<byte[]> values)
+ {
+ if (values == null)
+ {
+ throw new ArgumentNullException(nameof(values));
+ }
+
+ foreach (byte[] arr in values)
+ {
+ if (arr == null)
+ {
+ AppendNull();
+ }
+ else
+ {
+ Append((ReadOnlySpan<byte>)arr);
+ }
+ }
+
+ return Instance;
+ }
+
+ public TBuilder Reserve(int capacity)
+ {
+ ValueOffsets.Reserve(capacity + 1);
+ ValueBuffer.Reserve(capacity);
+ ValidityBuffer.Reserve(capacity);
+ return Instance;
+ }
+
+ public TBuilder Resize(int length)
+ {
+ ValueOffsets.Resize(length + 1);
+ ValueBuffer.Resize(length);
+ ValidityBuffer.Resize(length);
+ return Instance;
+ }
+
+ public TBuilder Swap(int i, int j)
+ {
+ throw new NotImplementedException();
+ }
+
+ public TBuilder Set(int index, byte value)
+ {
+ throw new NotImplementedException();
+ }
+
+ public TBuilder Clear()
+ {
+ ValueOffsets.Clear();
+ ValueBuffer.Clear();
+ ValidityBuffer.Clear();
+ Offset = 0;
+ ValueOffsets.Append(Offset);
+ return Instance;
+ }
+ }
+
public LargeBinaryArray(ArrayData data)
: base(data)
{
diff --git a/src/Apache.Arrow/Arrays/LargeListArray.cs
b/src/Apache.Arrow/Arrays/LargeListArray.cs
index 6e37aa4..f4cef97 100644
--- a/src/Apache.Arrow/Arrays/LargeListArray.cs
+++ b/src/Apache.Arrow/Arrays/LargeListArray.cs
@@ -14,12 +14,97 @@
// limitations under the License.
using System;
+using Apache.Arrow.Memory;
using Apache.Arrow.Types;
namespace Apache.Arrow
{
public class LargeListArray : Array
{
+ public class Builder : IArrowArrayBuilder<LargeListArray, Builder>
+ {
+ public IArrowArrayBuilder<IArrowArray,
IArrowArrayBuilder<IArrowArray>> ValueBuilder { get; }
+
+ public int Length => ValueOffsetsBufferBuilder.Length;
+
+ private ArrowBuffer.Builder<long> ValueOffsetsBufferBuilder { get;
}
+
+ private ArrowBuffer.BitmapBuilder ValidityBufferBuilder { get; }
+
+ public int NullCount { get; protected set; }
+
+ private IArrowType DataType { get; }
+
+ public Builder(IArrowType valueDataType) : this(new
LargeListType(valueDataType))
+ {
+ }
+
+ public Builder(Field valueField) : this(new
LargeListType(valueField))
+ {
+ }
+
+ internal Builder(LargeListType dataType)
+ {
+ ValueBuilder =
ArrowArrayBuilderFactory.Build(dataType.ValueDataType);
+ ValueOffsetsBufferBuilder = new ArrowBuffer.Builder<long>();
+ ValidityBufferBuilder = new ArrowBuffer.BitmapBuilder();
+ DataType = dataType;
+ }
+
+ public Builder Append()
+ {
+ ValueOffsetsBufferBuilder.Append(ValueBuilder.Length);
+ ValidityBufferBuilder.Append(true);
+
+ return this;
+ }
+
+ public Builder AppendNull()
+ {
+ ValueOffsetsBufferBuilder.Append(ValueBuilder.Length);
+ ValidityBufferBuilder.Append(false);
+ NullCount++;
+
+ return this;
+ }
+
+ public LargeListArray Build(MemoryAllocator allocator = default)
+ {
+ ValueOffsetsBufferBuilder.Append(ValueBuilder.Length);
+
+ ArrowBuffer validityBuffer = NullCount > 0
+ ?
ValidityBufferBuilder.Build(allocator)
+ : ArrowBuffer.Empty;
+
+ return new LargeListArray(DataType, Length - 1,
+ ValueOffsetsBufferBuilder.Build(allocator),
ValueBuilder.Build(allocator),
+ validityBuffer, NullCount, 0);
+ }
+
+ public Builder Reserve(int capacity)
+ {
+ ValueOffsetsBufferBuilder.Reserve(capacity + 1);
+ ValidityBufferBuilder.Reserve(capacity);
+ return this;
+ }
+
+ public Builder Resize(int length)
+ {
+ ValueOffsetsBufferBuilder.Resize(length + 1);
+ ValidityBufferBuilder.Resize(length);
+ return this;
+ }
+
+ public Builder Clear()
+ {
+ ValueOffsetsBufferBuilder.Clear();
+ ValueBuilder.Clear();
+ ValidityBufferBuilder.Clear();
+ NullCount = 0;
+ return this;
+ }
+ }
+
public IArrowArray Values { get; }
public ArrowBuffer ValueOffsetsBuffer => Data.Buffers[1];
diff --git a/src/Apache.Arrow/Arrays/LargeStringArray.cs
b/src/Apache.Arrow/Arrays/LargeStringArray.cs
index a12bae6..06752c2 100644
--- a/src/Apache.Arrow/Arrays/LargeStringArray.cs
+++ b/src/Apache.Arrow/Arrays/LargeStringArray.cs
@@ -26,6 +26,37 @@ public class LargeStringArray : LargeBinaryArray,
IReadOnlyList<string>, ICollec
{
public static readonly Encoding DefaultEncoding =
StringArray.DefaultEncoding;
+ public new class Builder : BuilderBase<LargeStringArray, Builder>
+ {
+ public Builder() : base(LargeStringType.Default) { }
+
+ protected override LargeStringArray Build(ArrayData data)
+ {
+ return new LargeStringArray(data);
+ }
+
+ public Builder Append(string value, Encoding encoding = null)
+ {
+ if (value == null)
+ {
+ return AppendNull();
+ }
+ encoding = encoding ?? DefaultEncoding;
+ byte[] span = encoding.GetBytes(value);
+ return Append(span.AsSpan());
+ }
+
+ public Builder AppendRange(IEnumerable<string> values, Encoding
encoding = null)
+ {
+ foreach (string value in values)
+ {
+ Append(value, encoding);
+ }
+
+ return this;
+ }
+ }
+
public LargeStringArray(ArrayData data)
: base(ArrowTypeId.LargeString, data) { }
diff --git a/src/Apache.Arrow/Arrays/ListArray.cs
b/src/Apache.Arrow/Arrays/ListArray.cs
index 4d2ff96..dd27e68 100644
--- a/src/Apache.Arrow/Arrays/ListArray.cs
+++ b/src/Apache.Arrow/Arrays/ListArray.cs
@@ -107,6 +107,7 @@ namespace Apache.Arrow
ValueOffsetsBufferBuilder.Clear();
ValueBuilder.Clear();
ValidityBufferBuilder.Clear();
+ NullCount = 0;
return this;
}
diff --git a/src/Apache.Arrow/Arrays/ListViewArray.cs
b/src/Apache.Arrow/Arrays/ListViewArray.cs
index 081385d..7bbb292 100644
--- a/src/Apache.Arrow/Arrays/ListViewArray.cs
+++ b/src/Apache.Arrow/Arrays/ListViewArray.cs
@@ -132,9 +132,9 @@ namespace Apache.Arrow
SizesBufferBuilder.Clear();
ValueBuilder.Clear();
ValidityBufferBuilder.Clear();
+ NullCount = 0;
return this;
}
-
}
public IArrowArray Values { get; }
diff --git a/src/Apache.Arrow/Arrays/MapArray.cs
b/src/Apache.Arrow/Arrays/MapArray.cs
index 341bb24..a488168 100644
--- a/src/Apache.Arrow/Arrays/MapArray.cs
+++ b/src/Apache.Arrow/Arrays/MapArray.cs
@@ -106,6 +106,7 @@ namespace Apache.Arrow
KeyBuilder.Clear();
ValueBuilder.Clear();
ValidityBufferBuilder.Clear();
+ NullCount = 0;
return this;
}
diff --git a/test/Apache.Arrow.Tests/ArrayBuilderTests.cs
b/test/Apache.Arrow.Tests/ArrayBuilderTests.cs
index 7588c34..2db71b2 100644
--- a/test/Apache.Arrow.Tests/ArrayBuilderTests.cs
+++ b/test/Apache.Arrow.Tests/ArrayBuilderTests.cs
@@ -138,6 +138,7 @@ namespace Apache.Arrow.Tests
var emptyList = listBuilder.Build();
Assert.Equal(0, emptyList.Length);
+ Assert.Equal(0, emptyList.NullCount);
List<string> ConvertStringArrayToList(StringArray array)
{
@@ -329,6 +330,62 @@ namespace Apache.Arrow.Tests
((Int64Array)childList3.GetSlicedValues(2)).ToList(includeNulls: true));
}
+ [Fact]
+ public void ListViewArrayBuilderClearResetsNullCount()
+ {
+ var builder = new ListViewArray.Builder(Int64Type.Default);
+ var valueBuilder = (Int64Array.Builder)builder.ValueBuilder;
+
+ builder.AppendNull();
+ builder.Append();
+ valueBuilder.Append(1);
+
+ var list = builder.Build();
+ Assert.Equal(2, list.Length);
+ Assert.Equal(1, list.NullCount);
+
+ builder.Clear();
+ builder.Append();
+ valueBuilder.Append(10);
+ var clearedList = builder.Build();
+
+ Assert.Equal(0, clearedList.NullCount);
+ }
+
+ [Fact]
+ public void MapArrayBuilderClearResetsNullCount()
+ {
+ var mapType = new MapType(
+ new
Field.Builder().Name("key").DataType(StringType.Default).Nullable(false).Build(),
+ new
Field.Builder().Name("value").DataType(Int32Type.Default).Nullable(true).Build(),
+ keySorted: false);
+
+ var builder = new MapArray.Builder(mapType);
+ var keyBuilder = (StringArray.Builder)builder.KeyBuilder;
+ var valueBuilder = (Int32Array.Builder)builder.ValueBuilder;
+
+ builder.Append();
+ keyBuilder.Append("a");
+ valueBuilder.Append(1);
+ builder.AppendNull();
+ builder.Append();
+ keyBuilder.Append("b");
+ valueBuilder.Append(2);
+
+ var map = builder.Build();
+ Assert.Equal(3, map.Length);
+ Assert.Equal(1, map.NullCount);
+
+ builder.Clear();
+ builder.Append();
+ keyBuilder.Append("x");
+ valueBuilder.Append(99);
+ var clearedMap = builder.Build();
+
+ Assert.Equal(1, clearedMap.Length);
+ Assert.Equal(0, clearedMap.NullCount);
+ }
+
public class TimestampArrayBuilder
{
[Fact]
diff --git a/test/Apache.Arrow.Tests/ArrowArrayConcatenatorTests.cs
b/test/Apache.Arrow.Tests/ArrowArrayConcatenatorTests.cs
index 758c426..f9589ef 100644
--- a/test/Apache.Arrow.Tests/ArrowArrayConcatenatorTests.cs
+++ b/test/Apache.Arrow.Tests/ArrowArrayConcatenatorTests.cs
@@ -84,6 +84,9 @@ namespace Apache.Arrow.Tests
new Decimal64Type(14, 4),
new Decimal128Type(14, 10),
new Decimal256Type(14, 10),
+ LargeBinaryType.Default,
+ LargeStringType.Default,
+ new LargeListType(Int64Type.Default),
new ListType(Int64Type.Default),
new ListViewType(Int64Type.Default),
new StructType(new List<Field>{
@@ -154,6 +157,9 @@ namespace Apache.Arrow.Tests
IArrowTypeVisitor<FixedSizeListType>,
IArrowTypeVisitor<StructType>,
IArrowTypeVisitor<UnionType>,
+ IArrowTypeVisitor<LargeBinaryType>,
+ IArrowTypeVisitor<LargeStringType>,
+ IArrowTypeVisitor<LargeListType>,
IArrowTypeVisitor<MapType>
{
private readonly List<List<int?>> _baseData;
@@ -496,6 +502,33 @@ namespace Apache.Arrow.Tests
new[] { stringResultBuilder.Build().Data,
intResultBuilder.Build().Data }));
}
+ public void Visit(LargeBinaryType type) =>
+ GenerateTestData<LargeBinaryArray,
LargeBinaryArray.Builder>(type, (builder, x) =>
+ {
+ if (x % 2 == 0)
+ {
+ builder.Append((byte)x);
+ }
+ else
+ {
+ builder.Append(new byte[] { (byte)x, (byte)(x + 1)
}.AsSpan());
+ }
+ });
+
+ public void Visit(LargeStringType type) =>
+ GenerateTestData<LargeStringArray,
LargeStringArray.Builder>(type, (builder, x) => builder.Append(x.ToString()));
+
+ public void Visit(LargeListType type) =>
+ GenerateTestData<LargeListArray, LargeListArray.Builder>(type,
(builder, x) =>
+ {
+ builder.Append();
+ ((Int64Array.Builder)builder.ValueBuilder).Append(x);
+ }, initAction: (builder, length) =>
+ {
+ builder.Reserve(length);
+ builder.ValueBuilder.Reserve(length);
+ });
+
public void Visit(MapType type) =>
GenerateTestData<MapArray, MapArray.Builder>(type, (builder,
x) =>
{
diff --git a/test/Apache.Arrow.Tests/LargeBinaryArrayBuilderTests.cs
b/test/Apache.Arrow.Tests/LargeBinaryArrayBuilderTests.cs
new file mode 100644
index 0000000..7172c93
--- /dev/null
+++ b/test/Apache.Arrow.Tests/LargeBinaryArrayBuilderTests.cs
@@ -0,0 +1,120 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+using System;
+using System.Linq;
+using Xunit;
+
+namespace Apache.Arrow.Tests
+{
+ public class LargeBinaryArrayBuilderTests
+ {
+ [Fact]
+ public void AppendByteSpanBuildsCorrectArray()
+ {
+ var builder = new LargeBinaryArray.Builder();
+ builder.Append(new byte[] { 1, 2, 3 }.AsSpan());
+ builder.Append(new byte[] { 4, 5 }.AsSpan());
+ builder.AppendNull();
+ builder.Append(ReadOnlySpan<byte>.Empty);
+
+ var array = builder.Build();
+
+ Assert.Equal(4, array.Length);
+ Assert.Equal(new byte[] { 1, 2, 3 }, array.GetBytes(0).ToArray());
+ Assert.Equal(new byte[] { 4, 5 }, array.GetBytes(1).ToArray());
+ Assert.True(array.IsNull(2));
+ Assert.Empty(array.GetBytes(3).ToArray());
+ }
+
+ [Fact]
+ public void AppendSingleByteBuildsCorrectArray()
+ {
+ var builder = new LargeBinaryArray.Builder();
+ builder.Append((byte)10);
+ builder.Append((byte)20);
+
+ var array = builder.Build();
+
+ Assert.Equal(2, array.Length);
+ Assert.Equal(new byte[] { 10 }, array.GetBytes(0).ToArray());
+ Assert.Equal(new byte[] { 20 }, array.GetBytes(1).ToArray());
+ }
+
+ [Fact]
+ public void AppendRangeByteArraysBuildsCorrectArray()
+ {
+ var builder = new LargeBinaryArray.Builder();
+ builder.AppendRange(new byte[][]
+ {
+ new byte[] { 1, 2, 3 },
+ null,
+ new byte[] { 4, 5 },
+ });
+
+ var array = builder.Build();
+
+ Assert.Equal(3, array.Length);
+ Assert.Equal(new byte[] { 1, 2, 3 }, array.GetBytes(0).ToArray());
+ Assert.True(array.IsNull(1));
+ Assert.Equal(new byte[] { 4, 5 }, array.GetBytes(2).ToArray());
+ }
+
+ [Fact]
+ public void ClearResetsBuilder()
+ {
+ var builder = new LargeBinaryArray.Builder();
+ builder.Append(new byte[] { 1, 2, 3 }.AsSpan());
+ builder.Clear();
+ builder.Append(new byte[] { 4, 5 }.AsSpan());
+
+ var array = builder.Build();
+
+ Assert.Equal(1, array.Length);
+ Assert.Equal(new byte[] { 4, 5 }, array.GetBytes(0).ToArray());
+ }
+
+ [Fact]
+ public void LengthTracksAppendedItems()
+ {
+ var builder = new LargeBinaryArray.Builder();
+ Assert.Equal(0, builder.Length);
+
+ builder.Append(new byte[] { 1 }.AsSpan());
+ Assert.Equal(1, builder.Length);
+
+ builder.AppendNull();
+ Assert.Equal(2, builder.Length);
+
+ builder.Append((byte)5);
+ Assert.Equal(3, builder.Length);
+ }
+
+ [Fact]
+ public void NullCountIsCorrect()
+ {
+ var builder = new LargeBinaryArray.Builder();
+ builder.Append(new byte[] { 1 }.AsSpan());
+ builder.AppendNull();
+ builder.Append(new byte[] { 2 }.AsSpan());
+ builder.AppendNull();
+
+ var array = builder.Build();
+
+ Assert.Equal(4, array.Length);
+ Assert.Equal(2, array.NullCount);
+ }
+ }
+}
diff --git a/test/Apache.Arrow.Tests/LargeListArrayBuilderTests.cs
b/test/Apache.Arrow.Tests/LargeListArrayBuilderTests.cs
new file mode 100644
index 0000000..3fd448a
--- /dev/null
+++ b/test/Apache.Arrow.Tests/LargeListArrayBuilderTests.cs
@@ -0,0 +1,131 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+using Apache.Arrow.Types;
+using Xunit;
+
+namespace Apache.Arrow.Tests
+{
+ public class LargeListArrayBuilderTests
+ {
+ [Fact]
+ public void AppendBuildsCorrectArray()
+ {
+ var builder = new LargeListArray.Builder(Int32Type.Default);
+ var valueBuilder = (Int32Array.Builder)builder.ValueBuilder;
+
+ builder.Append();
+ valueBuilder.Append(1);
+ valueBuilder.Append(2);
+
+ builder.Append();
+ valueBuilder.Append(3);
+
+ builder.AppendNull();
+
+ builder.Append();
+
+ var array = builder.Build();
+
+ Assert.Equal(4, array.Length);
+
+ var slice0 = (Int32Array)array.GetSlicedValues(0);
+ Assert.Equal(2, slice0.Length);
+ Assert.Equal(1, slice0.GetValue(0));
+ Assert.Equal(2, slice0.GetValue(1));
+
+ var slice1 = (Int32Array)array.GetSlicedValues(1);
+ Assert.Equal(1, slice1.Length);
+ Assert.Equal(3, slice1.GetValue(0));
+
+ Assert.Null(array.GetSlicedValues(2));
+
+ var slice3 = (Int32Array)array.GetSlicedValues(3);
+ Assert.Equal(0, slice3.Length);
+ }
+
+ [Fact]
+ public void NullCountIsCorrect()
+ {
+ var builder = new LargeListArray.Builder(Int32Type.Default);
+ var valueBuilder = (Int32Array.Builder)builder.ValueBuilder;
+
+ builder.Append();
+ valueBuilder.Append(1);
+
+ builder.AppendNull();
+ builder.AppendNull();
+
+ builder.Append();
+ valueBuilder.Append(2);
+
+ var array = builder.Build();
+
+ Assert.Equal(4, array.Length);
+ Assert.Equal(2, array.NullCount);
+ }
+
+ [Fact]
+ public void ClearResetsBuilder()
+ {
+ var builder = new LargeListArray.Builder(Int32Type.Default);
+ var valueBuilder = (Int32Array.Builder)builder.ValueBuilder;
+
+ builder.Append();
+ valueBuilder.Append(1);
+ builder.AppendNull();
+
+ builder.Clear();
+
+ builder.Append();
+ valueBuilder.Append(99);
+
+ var array = builder.Build();
+
+ Assert.Equal(1, array.Length);
+ Assert.Equal(0, array.NullCount);
+ var slice = (Int32Array)array.GetSlicedValues(0);
+ Assert.Equal(1, slice.Length);
+ Assert.Equal(99, slice.GetValue(0));
+ }
+
+ [Fact]
+ public void NestedStringListBuildsCorrectly()
+ {
+ var builder = new LargeListArray.Builder(StringType.Default);
+ var valueBuilder = (StringArray.Builder)builder.ValueBuilder;
+
+ builder.Append();
+ valueBuilder.Append("hello");
+ valueBuilder.Append("world");
+
+ builder.Append();
+ valueBuilder.Append("foo");
+
+ var array = builder.Build();
+
+ Assert.Equal(2, array.Length);
+
+ var slice0 = (StringArray)array.GetSlicedValues(0);
+ Assert.Equal(2, slice0.Length);
+ Assert.Equal("hello", slice0.GetString(0));
+ Assert.Equal("world", slice0.GetString(1));
+
+ var slice1 = (StringArray)array.GetSlicedValues(1);
+ Assert.Equal(1, slice1.Length);
+ Assert.Equal("foo", slice1.GetString(0));
+ }
+ }
+}
diff --git a/test/Apache.Arrow.Tests/LargeStringArrayBuilderTests.cs
b/test/Apache.Arrow.Tests/LargeStringArrayBuilderTests.cs
new file mode 100644
index 0000000..e50a04b
--- /dev/null
+++ b/test/Apache.Arrow.Tests/LargeStringArrayBuilderTests.cs
@@ -0,0 +1,84 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+using Xunit;
+
+namespace Apache.Arrow.Tests
+{
+ public class LargeStringArrayBuilderTests
+ {
+ [Fact]
+ public void AppendStringBuildsCorrectArray()
+ {
+ var builder = new LargeStringArray.Builder();
+ builder.Append("hello");
+ builder.Append("world");
+ builder.AppendNull();
+ builder.Append("");
+
+ var array = builder.Build();
+
+ Assert.Equal(4, array.Length);
+ Assert.Equal("hello", array.GetString(0));
+ Assert.Equal("world", array.GetString(1));
+ Assert.Null(array.GetString(2));
+ Assert.Equal("", array.GetString(3));
+ }
+
+ [Fact]
+ public void AppendRangeBuildsCorrectArray()
+ {
+ var builder = new LargeStringArray.Builder();
+ builder.AppendRange(new[] { "foo", null, "bar", "baz" });
+
+ var array = builder.Build();
+
+ Assert.Equal(4, array.Length);
+ Assert.Equal("foo", array.GetString(0));
+ Assert.Null(array.GetString(1));
+ Assert.Equal("bar", array.GetString(2));
+ Assert.Equal("baz", array.GetString(3));
+ }
+
+ [Fact]
+ public void ClearResetsBuilder()
+ {
+ var builder = new LargeStringArray.Builder();
+ builder.Append("hello");
+ builder.Clear();
+ builder.Append("world");
+
+ var array = builder.Build();
+
+ Assert.Equal(1, array.Length);
+ Assert.Equal("world", array.GetString(0));
+ }
+
+ [Fact]
+ public void NullCountIsCorrect()
+ {
+ var builder = new LargeStringArray.Builder();
+ builder.Append("a");
+ builder.AppendNull();
+ builder.Append("b");
+ builder.AppendNull();
+
+ var array = builder.Build();
+
+ Assert.Equal(4, array.Length);
+ Assert.Equal(2, array.NullCount);
+ }
+ }
+}
