This is an automated email from the ASF dual-hosted git repository.
curth pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new 63c91ff32b GH-41164: [C#] Fix concatenation of sliced arrays (#41245)
63c91ff32b is described below
commit 63c91ff32b8547f0bfd6ff827d7a6901d9e7ca5c
Author: Adam Reeve <[email protected]>
AuthorDate: Wed Apr 17 16:03:38 2024 +1200
GH-41164: [C#] Fix concatenation of sliced arrays (#41245)
### Rationale for this change
Makes array concatenation work correctly when the input arrays have been
sliced.
### What changes are included in this PR?
* Updates the array concatenation tests so that the `TestDataGenerator` can
generate test cases with sliced input arrays. To avoid too much duplicated
logic, I've added a new `GenerateTestData<TArray, TArrayBuilder>` method that
works with builders that are not `IArrowArrayBuilder<T, TArray,
TArrayBuilder>`, and simplified a lot of the data generation by using this new
method. Only struct and union array test data generation still needs to
duplicate the logic in `GenerateTestData`.
* Fixes `ArrayDataConcatenator` logic to handle sliced input arrays
### Are these changes tested?
Yes, I've added a new test for this.
### Are there any user-facing changes?
Yes, this is a user-facing bug fix.
* GitHub Issue: #41164
Authored-by: Adam Reeve <[email protected]>
Signed-off-by: Curt Hagenlocher <[email protected]>
---
.../Apache.Arrow/Arrays/ArrayDataConcatenator.cs | 197 +++++--
.../Arrays/ArrowArrayBuilderFactory.cs | 13 +-
.../ArrowArrayBuilderFactoryReflector.cs | 32 -
.../ArrowArrayConcatenatorTests.cs | 653 ++++++++-------------
4 files changed, 430 insertions(+), 465 deletions(-)
diff --git a/csharp/src/Apache.Arrow/Arrays/ArrayDataConcatenator.cs
b/csharp/src/Apache.Arrow/Arrays/ArrayDataConcatenator.cs
index 347d0d76ba..fe2543b70a 100644
--- a/csharp/src/Apache.Arrow/Arrays/ArrayDataConcatenator.cs
+++ b/csharp/src/Apache.Arrow/Arrays/ArrayDataConcatenator.cs
@@ -107,38 +107,76 @@ namespace Apache.Arrow
{
CheckData(type, 3);
ArrowBuffer validityBuffer = ConcatenateValidityBuffer();
+ ArrowBuffer sizesBuffer =
ConcatenateFixedWidthTypeValueBuffer(2, Int32Type.Default);
+ var children = new List<ArrayData>(_arrayDataList.Count);
var offsetsBuilder = new
ArrowBuffer.Builder<int>(_totalLength);
int baseOffset = 0;
foreach (ArrayData arrayData in _arrayDataList)
{
- if (arrayData.Length > 0)
+ if (arrayData.Length == 0)
{
- ReadOnlySpan<int> span =
arrayData.Buffers[1].Span.CastTo<int>().Slice(0, arrayData.Length);
- foreach (int offset in span)
- {
- offsetsBuilder.Append(baseOffset + offset);
- }
+ continue;
+ }
+
+ var child = arrayData.Children[0];
+ ReadOnlySpan<int> offsets =
arrayData.Buffers[1].Span.CastTo<int>().Slice(arrayData.Offset,
arrayData.Length);
+ ReadOnlySpan<int> sizes =
arrayData.Buffers[2].Span.CastTo<int>().Slice(arrayData.Offset,
arrayData.Length);
+ var minOffset = offsets[0];
+ var maxEnd = 0;
+
+ for (int i = 0; i < arrayData.Length; ++i)
+ {
+ minOffset = Math.Min(minOffset, offsets[i]);
+ maxEnd = Math.Max(maxEnd, offsets[i] + sizes[i]);
}
- baseOffset += arrayData.Children[0].Length;
+ foreach (int offset in offsets)
+ {
+ offsetsBuilder.Append(baseOffset + offset - minOffset);
+ }
+
+ var childLength = maxEnd - minOffset;
+ if (minOffset != 0 || childLength != child.Length)
+ {
+ child = child.Slice(minOffset, childLength);
+ }
+
+ baseOffset += childLength;
+ children.Add(child);
}
ArrowBuffer offsetBuffer = offsetsBuilder.Build(_allocator);
- ArrowBuffer sizesBuffer =
ConcatenateFixedWidthTypeValueBuffer(2, Int32Type.Default);
- ArrayData child = Concatenate(SelectChildren(0), _allocator);
+ ArrayData combinedChild = Concatenate(children, _allocator);
- Result = new ArrayData(type, _totalLength, _totalNullCount, 0,
new ArrowBuffer[] { validityBuffer, offsetBuffer, sizesBuffer }, new[] { child
});
+ Result = new ArrayData(type, _totalLength, _totalNullCount, 0,
new ArrowBuffer[] { validityBuffer, offsetBuffer, sizesBuffer }, new[] {
combinedChild });
}
public void Visit(FixedSizeListType type)
{
CheckData(type, 1);
+ var listSize = type.ListSize;
ArrowBuffer validityBuffer = ConcatenateValidityBuffer();
- ArrayData child = Concatenate(SelectChildren(0), _allocator);
- Result = new ArrayData(type, _totalLength, _totalNullCount, 0,
new ArrowBuffer[] { validityBuffer }, new[] { child });
+ var children = new List<ArrayData>(_arrayDataList.Count);
+
+ foreach (ArrayData arrayData in _arrayDataList)
+ {
+ var offset = arrayData.Offset;
+ var length = arrayData.Length;
+ var child = arrayData.Children[0];
+ if (offset != 0 || child.Length != length * listSize)
+ {
+ child = child.Slice(offset * listSize, length *
listSize);
+ }
+
+ children.Add(child);
+ }
+
+ ArrayData combinedChild = Concatenate(children, _allocator);
+
+ Result = new ArrayData(type, _totalLength, _totalNullCount, 0,
new ArrowBuffer[] { validityBuffer }, new[] { combinedChild });
}
public void Visit(StructType type)
@@ -149,7 +187,7 @@ namespace Apache.Arrow
for (int i = 0; i < type.Fields.Count; i++)
{
- children.Add(Concatenate(SelectChildren(i), _allocator));
+ children.Add(Concatenate(SelectSlicedChildren(i),
_allocator));
}
Result = new ArrayData(type, _totalLength, _totalNullCount, 0,
new ArrowBuffer[] { validityBuffer }, children);
@@ -169,7 +207,11 @@ namespace Apache.Arrow
for (int i = 0; i < type.Fields.Count; i++)
{
- children.Add(Concatenate(SelectChildren(i), _allocator));
+ // For dense mode, the offsets aren't adjusted so are into
the non-sliced child arrays
+ var fieldChildren = type.Mode == UnionMode.Sparse
+ ? SelectSlicedChildren(i)
+ : SelectChildren(i);
+ children.Add(Concatenate(fieldChildren, _allocator));
}
ArrowBuffer[] buffers = new ArrowBuffer[bufferCount];
@@ -242,9 +284,30 @@ namespace Apache.Arrow
CheckData(type, 2);
ArrowBuffer validityBuffer = ConcatenateValidityBuffer();
ArrowBuffer offsetBuffer = ConcatenateOffsetBuffer();
- ArrayData child = Concatenate(SelectChildren(0), _allocator);
- Result = new ArrayData(type, _totalLength, _totalNullCount, 0,
new ArrowBuffer[] { validityBuffer, offsetBuffer }, new[] { child });
+ var children = new List<ArrayData>(_arrayDataList.Count);
+ foreach (ArrayData arrayData in _arrayDataList)
+ {
+ if (arrayData.Length == 0)
+ {
+ continue;
+ }
+
+ var child = arrayData.Children[0];
+ ReadOnlySpan<int> offsets =
arrayData.Buffers[1].Span.CastTo<int>().Slice(arrayData.Offset,
arrayData.Length + 1);
+ var firstOffset = offsets[0];
+ var lastOffset = offsets[arrayData.Length];
+ if (firstOffset != 0 || lastOffset != child.Length)
+ {
+ child = child.Slice(firstOffset, lastOffset -
firstOffset);
+ }
+
+ children.Add(child);
+ }
+
+ ArrayData combinedChild = Concatenate(children, _allocator);
+
+ Result = new ArrayData(type, _totalLength, _totalNullCount, 0,
new ArrowBuffer[] { validityBuffer, offsetBuffer }, new[] { combinedChild });
}
private ArrowBuffer ConcatenateValidityBuffer()
@@ -254,7 +317,43 @@ namespace Apache.Arrow
return ArrowBuffer.Empty;
}
- return ConcatenateBitmapBuffer(0);
+ var builder = new ArrowBuffer.BitmapBuilder(_totalLength);
+
+ foreach (ArrayData arrayData in _arrayDataList)
+ {
+ int length = arrayData.Length;
+ int offset = arrayData.Offset;
+ ReadOnlySpan<byte> span = arrayData.Buffers[0].Span;
+
+ if (length > 0 && span.Length == 0)
+ {
+ if (arrayData.NullCount == 0)
+ {
+ builder.AppendRange(true , length);
+ }
+ else if (arrayData.NullCount == length)
+ {
+ builder.AppendRange(false , length);
+ }
+ else
+ {
+ throw new Exception("Array has no validity buffer
and null count != 0 or length");
+ }
+ }
+ else if (offset == 0)
+ {
+ builder.Append(span, length);
+ }
+ else
+ {
+ for (int i = 0; i < length; ++i)
+ {
+ builder.Append(BitUtility.GetBit(span, offset +
i));
+ }
+ }
+ }
+
+ return builder.Build(_allocator);
}
private ArrowBuffer ConcatenateBitmapBuffer(int bufferIndex)
@@ -264,9 +363,20 @@ namespace Apache.Arrow
foreach (ArrayData arrayData in _arrayDataList)
{
int length = arrayData.Length;
+ int offset = arrayData.Offset;
ReadOnlySpan<byte> span =
arrayData.Buffers[bufferIndex].Span;
- builder.Append(span, length);
+ if (offset == 0)
+ {
+ builder.Append(span, length);
+ }
+ else
+ {
+ for (int i = 0; i < length; ++i)
+ {
+ builder.Append(BitUtility.GetBit(span, offset +
i));
+ }
+ }
}
return builder.Build(_allocator);
@@ -279,10 +389,10 @@ namespace Apache.Arrow
foreach (ArrayData arrayData in _arrayDataList)
{
- int length = arrayData.Length;
- int byteLength = length * typeByteWidth;
+ int byteLength = arrayData.Length * typeByteWidth;
+ int byteOffset = arrayData.Offset * typeByteWidth;
-
builder.Append(arrayData.Buffers[bufferIndex].Span.Slice(0, byteLength));
+
builder.Append(arrayData.Buffers[bufferIndex].Span.Slice(byteOffset,
byteLength));
}
return builder.Build(_allocator);
@@ -294,8 +404,10 @@ namespace Apache.Arrow
foreach (ArrayData arrayData in _arrayDataList)
{
- int lastOffset =
arrayData.Buffers[1].Span.CastTo<int>()[arrayData.Length];
- builder.Append(arrayData.Buffers[2].Span.Slice(0,
lastOffset));
+ var offsets =
arrayData.Buffers[1].Span.CastTo<int>().Slice(arrayData.Offset,
arrayData.Length + 1);
+ var firstOffset = offsets[0];
+ var lastOffset = offsets[arrayData.Length];
+
builder.Append(arrayData.Buffers[2].Span.Slice(firstOffset, lastOffset -
firstOffset));
}
return builder.Build(_allocator);
@@ -306,8 +418,6 @@ namespace Apache.Arrow
var builder = new ArrowBuffer.Builder<int>(_totalLength + 1);
int baseOffset = 0;
- builder.Append(0);
-
foreach (ArrayData arrayData in _arrayDataList)
{
if (arrayData.Length == 0)
@@ -315,19 +425,20 @@ namespace Apache.Arrow
continue;
}
- // The first offset is always 0.
- // It should be skipped because it duplicate to the last
offset of builder.
- ReadOnlySpan<int> span =
arrayData.Buffers[1].Span.CastTo<int>().Slice(1, arrayData.Length);
+ ReadOnlySpan<int> span =
arrayData.Buffers[1].Span.CastTo<int>().Slice(arrayData.Offset,
arrayData.Length + 1);
+ // First offset may be non-zero for sliced arrays
+ var firstOffset = span[0];
- foreach (int offset in span)
+ foreach (int offset in span.Slice(0, arrayData.Length))
{
- builder.Append(baseOffset + offset);
+ builder.Append(baseOffset + offset - firstOffset);
}
- // The next offset must start from the current last offset.
- baseOffset += span[arrayData.Length - 1];
+ baseOffset += span[arrayData.Length] - firstOffset;
}
+ builder.Append(baseOffset);
+
return builder.Build(_allocator);
}
@@ -342,7 +453,7 @@ namespace Apache.Arrow
continue;
}
- ReadOnlySpan<BinaryView> span =
arrayData.Buffers[1].Span.CastTo<BinaryView>().Slice(0, arrayData.Length);
+ ReadOnlySpan<BinaryView> span =
arrayData.Buffers[1].Span.CastTo<BinaryView>().Slice(arrayData.Offset,
arrayData.Length);
foreach (BinaryView view in span)
{
if (view.Length > BinaryView.MaxInlineLength)
@@ -412,6 +523,26 @@ namespace Apache.Arrow
return children;
}
+
+ private List<ArrayData> SelectSlicedChildren(int index)
+ {
+ var children = new List<ArrayData>(_arrayDataList.Count);
+
+ foreach (ArrayData arrayData in _arrayDataList)
+ {
+ var offset = arrayData.Offset;
+ var length = arrayData.Length;
+ var child = arrayData.Children[index];
+ if (offset != 0 || child.Length != length)
+ {
+ child = child.Slice(offset, length);
+ }
+
+ children.Add(child);
+ }
+
+ return children;
+ }
}
}
}
diff --git a/csharp/src/Apache.Arrow/Arrays/ArrowArrayBuilderFactory.cs
b/csharp/src/Apache.Arrow/Arrays/ArrowArrayBuilderFactory.cs
index f836710208..cb7164aa14 100644
--- a/csharp/src/Apache.Arrow/Arrays/ArrowArrayBuilderFactory.cs
+++ b/csharp/src/Apache.Arrow/Arrays/ArrowArrayBuilderFactory.cs
@@ -82,12 +82,21 @@ namespace Apache.Arrow
return new Decimal128Array.Builder(dataType as
Decimal128Type);
case ArrowTypeId.Decimal256:
return new Decimal256Array.Builder(dataType as
Decimal256Type);
+ case ArrowTypeId.Interval:
+ var intervalType = (IntervalType)dataType;
+ return intervalType.Unit switch
+ {
+ IntervalUnit.YearMonth => new
YearMonthIntervalArray.Builder(),
+ IntervalUnit.DayTime => new
DayTimeIntervalArray.Builder(),
+ IntervalUnit.MonthDayNanosecond => new
MonthDayNanosecondIntervalArray.Builder(),
+ _ => throw new
ArgumentOutOfRangeException($"unsupported interval unit <{intervalType.Unit}>")
+ };
+ case ArrowTypeId.Map:
+ return new MapArray.Builder(dataType as MapType);
case ArrowTypeId.Struct:
case ArrowTypeId.Union:
case ArrowTypeId.Dictionary:
case ArrowTypeId.FixedSizedBinary:
- case ArrowTypeId.Interval:
- case ArrowTypeId.Map:
default:
throw new NotSupportedException($"An ArrowArrayBuilder
cannot be built for type {dataType.TypeId}.");
}
diff --git
a/csharp/test/Apache.Arrow.Tests/ArrowArrayBuilderFactoryReflector.cs
b/csharp/test/Apache.Arrow.Tests/ArrowArrayBuilderFactoryReflector.cs
deleted file mode 100644
index 69894ab3cd..0000000000
--- a/csharp/test/Apache.Arrow.Tests/ArrowArrayBuilderFactoryReflector.cs
+++ /dev/null
@@ -1,32 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more
-// contributor license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright ownership.
-// The ASF licenses this file to You under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance with
-// the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-using System;
-using System.Reflection;
-using Apache.Arrow.Types;
-
-namespace Apache.Arrow.Tests
-{
- static class ArrayArrayBuilderFactoryReflector
- {
- private static readonly MethodInfo s_buildInfo =
typeof(ArrayData).Assembly.GetType("Apache.Arrow.ArrowArrayBuilderFactory")
- .GetMethod("Build", BindingFlags.Static | BindingFlags.NonPublic);
-
- internal static IArrowArrayBuilder<IArrowArray,
IArrowArrayBuilder<IArrowArray>> InvokeBuild(IArrowType dataType)
- {
- return s_buildInfo.Invoke(null, new object[] { dataType }) as
IArrowArrayBuilder<IArrowArray, IArrowArrayBuilder<IArrowArray>>;
- }
- }
-}
diff --git a/csharp/test/Apache.Arrow.Tests/ArrowArrayConcatenatorTests.cs
b/csharp/test/Apache.Arrow.Tests/ArrowArrayConcatenatorTests.cs
index a1f6b1b8d8..2437d3d94c 100644
--- a/csharp/test/Apache.Arrow.Tests/ArrowArrayConcatenatorTests.cs
+++ b/csharp/test/Apache.Arrow.Tests/ArrowArrayConcatenatorTests.cs
@@ -34,6 +34,16 @@ namespace Apache.Arrow.Tests
}
}
+ [Fact]
+ public void TestConcatenateSlices()
+ {
+ foreach ((List<IArrowArray> testTargetArrayList, IArrowArray
expectedArray) in GenerateTestData(slicedArrays: true))
+ {
+ IArrowArray actualArray =
ArrowArrayConcatenator.Concatenate(testTargetArrayList);
+ ArrowReaderVerifier.CompareArrays(expectedArray, actualArray,
strictCompare: false);
+ }
+ }
+
[Fact]
public void TestNullOrEmpty()
{
@@ -49,7 +59,7 @@ namespace Apache.Arrow.Tests
ArrowReaderVerifier.CompareArrays(array, actualArray);
}
- private static IEnumerable<Tuple<List<IArrowArray>, IArrowArray>>
GenerateTestData()
+ private static IEnumerable<Tuple<List<IArrowArray>, IArrowArray>>
GenerateTestData(bool slicedArrays = false)
{
var targetTypes = new List<IArrowType>() {
BooleanType.Default,
@@ -78,7 +88,7 @@ namespace Apache.Arrow.Tests
new
Field.Builder().Name("Strings").DataType(StringType.Default).Nullable(true).Build(),
new
Field.Builder().Name("Ints").DataType(Int32Type.Default).Nullable(true).Build()
}),
- new FixedSizeListType(Int32Type.Default, 1),
+ new FixedSizeListType(Int32Type.Default, 2),
new UnionType(
new List<Field>{
new
Field.Builder().Name("Strings").DataType(StringType.Default).Nullable(true).Build(),
@@ -106,7 +116,7 @@ namespace Apache.Arrow.Tests
foreach (IArrowType type in targetTypes)
{
- var creator = new TestDataGenerator();
+ var creator = new TestDataGenerator(slicedArrays);
type.Accept(creator);
yield return Tuple.Create(creator.TestTargetArrayList,
creator.ExpectedArray);
}
@@ -142,26 +152,44 @@ namespace Apache.Arrow.Tests
IArrowTypeVisitor<UnionType>,
IArrowTypeVisitor<MapType>
{
+ private readonly List<List<int?>> _baseData;
- private List<List<int?>> _baseData;
+ private readonly int _baseDataListCount;
- private int _baseDataListCount;
+ private readonly int _resultTotalElementCount;
- private int _baseDataTotalElementCount;
+ private readonly List<(int Offset, int Length)> _sliceParameters =
null;
public List<IArrowArray> TestTargetArrayList { get; }
public IArrowArray ExpectedArray { get; private set; }
- public TestDataGenerator()
+ public TestDataGenerator(bool slicedArrays)
{
_baseData = new List<List<int?>> {
- new List<int?> { 1, 2, 3 },
- new List<int?> { 100, 101, null },
- new List<int?> { 11, null, 12 },
+ new List<int?> { 1, 2, 3, 4, 5, 6 },
+ new List<int?> { 100, 101, null, 102, null, 103 },
+ new List<int?> { null, null },
+ new List<int?> { },
+ new List<int?> { 11, null, 12, 13, 14 },
};
+ if (slicedArrays)
+ {
+ _sliceParameters = new List<(int, int)>
+ {
+ (2, 3),
+ (0, 5),
+ (0, 2),
+ (0, 0),
+ (1, 4),
+ };
+ }
+
_baseDataListCount = _baseData.Count;
- _baseDataTotalElementCount = _baseData.Sum(_ => _.Count);
+ _resultTotalElementCount = slicedArrays
+ ? _sliceParameters.Sum(p => p.Length)
+ : _baseData.Sum(baseList => baseList.Count);
+
TestTargetArrayList = new
List<IArrowArray>(_baseDataListCount);
}
@@ -179,166 +207,35 @@ namespace Apache.Arrow.Tests
public void Visit(Date32Type type) => GenerateTestData<DateTime,
Date32Array, Date32Array.Builder>(type, x => DateTime.MinValue.AddDays(x));
public void Visit(Date64Type type) => GenerateTestData<DateTime,
Date64Array, Date64Array.Builder>(type, x => DateTime.MinValue.AddDays(x));
- public void Visit(Decimal128Type type)
- {
- Decimal128Array.Builder resultBuilder = new
Decimal128Array.Builder(type).Reserve(_baseDataTotalElementCount);
+ public void Visit(Decimal128Type type) =>
GenerateTestData<Decimal128Array, Decimal128Array.Builder>(type, (builder, x)
=> builder.Append(x));
- for (int i = 0; i < _baseDataListCount; i++)
- {
- List<int?> dataList = _baseData[i];
- Decimal128Array.Builder builder = new
Decimal128Array.Builder(type).Reserve(dataList.Count);
- foreach (decimal? value in dataList)
- {
- if (value.HasValue)
- {
- builder.Append(value.Value);
- resultBuilder.Append(value.Value);
- }
- else
- {
- builder.AppendNull();
- resultBuilder.AppendNull();
- }
- }
- TestTargetArrayList.Add(builder.Build());
- }
-
- ExpectedArray = resultBuilder.Build();
- }
-
- public void Visit(Decimal256Type type)
- {
- Decimal256Array.Builder resultBuilder = new
Decimal256Array.Builder(type).Reserve(_baseDataTotalElementCount);
-
- for (int i = 0; i < _baseDataListCount; i++)
- {
- List<int?> dataList = _baseData[i];
- Decimal256Array.Builder builder = new
Decimal256Array.Builder(type).Reserve(dataList.Count);
- foreach (decimal? value in dataList)
- {
- if (value.HasValue)
- {
- builder.Append(value.Value);
- resultBuilder.Append(value.Value);
- }
- else
- {
- builder.AppendNull();
- resultBuilder.AppendNull();
- }
- }
- TestTargetArrayList.Add(builder.Build());
- }
-
- ExpectedArray = resultBuilder.Build();
- }
+ public void Visit(Decimal256Type type) =>
GenerateTestData<Decimal256Array, Decimal256Array.Builder>(type, (builder, x)
=> builder.Append(x));
public void Visit(TimestampType type)
{
- TimestampArray.Builder resultBuilder = new
TimestampArray.Builder().Reserve(_baseDataTotalElementCount);
DateTimeOffset basis = DateTimeOffset.UtcNow;
-
- for (int i = 0; i < _baseDataListCount; i++)
- {
- List<int?> dataList = _baseData[i];
- TimestampArray.Builder builder = new
TimestampArray.Builder().Reserve(dataList.Count);
- foreach (int? value in dataList)
- {
- if (value.HasValue)
- {
- DateTimeOffset dateValue =
basis.AddMilliseconds(value.Value);
- builder.Append(dateValue);
- resultBuilder.Append(dateValue);
- }
- else
- {
- builder.AppendNull();
- resultBuilder.AppendNull();
- }
- }
- TestTargetArrayList.Add(builder.Build());
- }
-
- ExpectedArray = resultBuilder.Build();
+ GenerateTestData<TimestampArray, TimestampArray.Builder>(type,
(builder, x) => builder.Append(basis.AddMilliseconds(x)));
}
- public void Visit(DurationType type)
- {
- DurationArray.Builder resultBuilder = new
DurationArray.Builder(type).Reserve(_baseDataTotalElementCount);
-
- for (int i = 0; i < _baseDataListCount; i++)
- {
- List<int?> dataList = _baseData[i];
- DurationArray.Builder builder = new
DurationArray.Builder(type).Reserve(dataList.Count);
- foreach (int? value in dataList)
- {
- if (value.HasValue)
- {
- builder.Append(value.Value);
- resultBuilder.Append(value.Value);
- }
- else
- {
- builder.AppendNull();
- resultBuilder.AppendNull();
- }
- }
- TestTargetArrayList.Add(builder.Build());
- }
-
- ExpectedArray = resultBuilder.Build();
- }
+ public void Visit(DurationType type) => GenerateTestData<long,
DurationArray, DurationArray.Builder>(type, x => (long)x);
public void Visit(IntervalType type)
{
switch (type.Unit)
{
case IntervalUnit.YearMonth:
- YearMonthIntervalArray.Builder yearMonthBuilder = new
YearMonthIntervalArray.Builder().Reserve(_baseDataTotalElementCount);
- foreach (List<int?> dataList in _baseData)
- {
- YearMonthIntervalArray.Builder yearMonthBuilder1 =
new YearMonthIntervalArray.Builder().Reserve(dataList.Count);
- foreach (int? value in dataList)
- {
- YearMonthInterval? ymi = value != null ? new
YearMonthInterval(value.Value) : null;
- yearMonthBuilder.Append(ymi);
- yearMonthBuilder1.Append(ymi);
- }
- TestTargetArrayList.Add(yearMonthBuilder1.Build());
- }
- ExpectedArray = yearMonthBuilder.Build();
+ GenerateTestData<YearMonthInterval,
YearMonthIntervalArray, YearMonthIntervalArray.Builder>(
+ type, x => new YearMonthInterval(x));
break;
case IntervalUnit.DayTime:
- DayTimeIntervalArray.Builder dayTimeBuilder = new
DayTimeIntervalArray.Builder().Reserve(_baseDataTotalElementCount);
- foreach (List<int?> dataList in _baseData)
- {
- DayTimeIntervalArray.Builder dayTimeBuilder1 = new
DayTimeIntervalArray.Builder().Reserve(dataList.Count);
- foreach (int? value in dataList)
- {
- DayTimeInterval? dti = value != null ? new
DayTimeInterval(100 - 50 * value.Value, 100 * value.Value) : null;
- dayTimeBuilder.Append(dti);
- dayTimeBuilder1.Append(dti);
- }
- TestTargetArrayList.Add(dayTimeBuilder1.Build());
- }
- ExpectedArray = dayTimeBuilder.Build();
+ GenerateTestData<DayTimeInterval,
DayTimeIntervalArray, DayTimeIntervalArray.Builder>(
+ type, x => new DayTimeInterval(100 - 50 * x, 100 *
x));
break;
case IntervalUnit.MonthDayNanosecond:
- MonthDayNanosecondIntervalArray.Builder
monthDayNanoBuilder = new
MonthDayNanosecondIntervalArray.Builder().Reserve(_baseDataTotalElementCount);
- foreach (List<int?> dataList in _baseData)
- {
- MonthDayNanosecondIntervalArray.Builder
monthDayNanoBuilder1 = new
MonthDayNanosecondIntervalArray.Builder().Reserve(dataList.Count);
- foreach (int? value in dataList)
- {
- MonthDayNanosecondInterval? mdni = value !=
null ? new MonthDayNanosecondInterval(value.Value, 5 - value.Value, 100 *
value.Value) : null;
- monthDayNanoBuilder.Append(mdni);
- monthDayNanoBuilder1.Append(mdni);
- }
-
TestTargetArrayList.Add(monthDayNanoBuilder1.Build());
- }
- ExpectedArray = monthDayNanoBuilder.Build();
+ GenerateTestData<MonthDayNanosecondInterval,
MonthDayNanosecondIntervalArray, MonthDayNanosecondIntervalArray.Builder>(
+ type, x => new MonthDayNanosecondInterval(x, 5 -
x, 100 * x));
break;
default:
@@ -346,245 +243,154 @@ namespace Apache.Arrow.Tests
}
}
- public void Visit(BinaryType type)
- {
- BinaryArray.Builder resultBuilder = new
BinaryArray.Builder().Reserve(_baseDataTotalElementCount);
-
- for (int i = 0; i < _baseDataListCount; i++)
+ public void Visit(BinaryType type) =>
+ GenerateTestData<BinaryArray, BinaryArray.Builder>(type,
(builder, x) =>
{
- List<int?> dataList = _baseData[i];
- BinaryArray.Builder builder = new
BinaryArray.Builder().Reserve(dataList.Count);
-
- foreach (byte? value in dataList)
+ if (x % 2 == 0)
{
- if (value.HasValue)
- {
- builder.Append(value.Value);
- resultBuilder.Append(value.Value);
- }
- else
- {
- builder.AppendNull();
- resultBuilder.AppendNull();
- }
+ builder.Append((byte)x);
}
- TestTargetArrayList.Add(builder.Build());
- }
-
- ExpectedArray = resultBuilder.Build();
- }
-
- public void Visit(BinaryViewType type)
- {
- BinaryViewArray.Builder resultBuilder = new
BinaryViewArray.Builder().Reserve(_baseDataTotalElementCount);
-
- for (int i = 0; i < _baseDataListCount; i++)
- {
- List<int?> dataList = _baseData[i];
- BinaryViewArray.Builder builder = new
BinaryViewArray.Builder().Reserve(dataList.Count);
-
- foreach (byte? value in dataList)
+ else
{
- if (value.HasValue)
- {
- builder.Append(value.Value);
- resultBuilder.Append(value.Value);
- }
- else
- {
- builder.AppendNull();
- resultBuilder.AppendNull();
- }
+ builder.Append(new byte[] {(byte)x, (byte)(x +
1)}.AsSpan());
}
- TestTargetArrayList.Add(builder.Build());
- }
+ });
- ExpectedArray = resultBuilder.Build();
- }
-
- public void Visit(StringType type)
- {
- StringArray.Builder resultBuilder = new
StringArray.Builder().Reserve(_baseDataTotalElementCount);
-
- for (int i = 0; i < _baseDataListCount; i++)
+ public void Visit(BinaryViewType type) =>
+ GenerateTestData<BinaryViewArray,
BinaryViewArray.Builder>(type, (builder, x) =>
{
- List<int?> dataList = _baseData[i];
- StringArray.Builder builder = new
StringArray.Builder().Reserve(dataList.Count);
-
- foreach (string value in dataList.Select(_ => _.ToString()
?? null))
+ if (x % 2 == 0)
{
- builder.Append(value);
- resultBuilder.Append(value);
+ builder.Append((byte)x);
}
- TestTargetArrayList.Add(builder.Build());
- }
-
- ExpectedArray = resultBuilder.Build();
- }
-
- public void Visit(StringViewType type)
- {
- StringViewArray.Builder resultBuilder = new
StringViewArray.Builder().Reserve(_baseDataTotalElementCount);
-
- for (int i = 0; i < _baseDataListCount; i++)
- {
- List<int?> dataList = _baseData[i];
- StringViewArray.Builder builder = new
StringViewArray.Builder().Reserve(dataList.Count);
-
- foreach (string value in dataList.Select(_ => _.ToString()
?? null))
+ else
{
- builder.Append(value);
- resultBuilder.Append(value);
+ builder.Append(new byte[] {(byte)x, (byte)(x +
1)}.AsSpan());
}
- TestTargetArrayList.Add(builder.Build());
- }
+ });
- ExpectedArray = resultBuilder.Build();
- }
+ public void Visit(StringType type) =>
+ GenerateTestData<StringArray, StringArray.Builder>(type,
(builder, x) => builder.Append(x.ToString()));
- public void Visit(ListType type)
- {
- ListArray.Builder resultBuilder = new
ListArray.Builder(type.ValueDataType).Reserve(_baseDataTotalElementCount);
- Int64Array.Builder resultValueBuilder =
(Int64Array.Builder)resultBuilder.ValueBuilder.Reserve(_baseDataTotalElementCount);
+ public void Visit(StringViewType type) =>
+ GenerateTestData<StringViewArray,
StringViewArray.Builder>(type, (builder, x) => builder.Append(x.ToString()));
- for (int i = 0; i < _baseDataListCount; i++)
+ public void Visit(ListType type) =>
+ GenerateTestData<ListArray, ListArray.Builder>(type, (builder,
x) =>
{
- List<int?> dataList = _baseData[i];
-
- ListArray.Builder builder = new
ListArray.Builder(type.ValueField).Reserve(dataList.Count);
- Int64Array.Builder valueBuilder =
(Int64Array.Builder)builder.ValueBuilder.Reserve(dataList.Count);
-
- foreach (long? value in dataList)
- {
- if (value.HasValue)
- {
- builder.Append();
- resultBuilder.Append();
-
- valueBuilder.Append(value.Value);
- resultValueBuilder.Append(value.Value);
- }
- else
- {
- builder.AppendNull();
- resultBuilder.AppendNull();
- }
- }
-
- TestTargetArrayList.Add(builder.Build());
- }
-
- ExpectedArray = resultBuilder.Build();
- }
-
- public void Visit(ListViewType type)
- {
- ListViewArray.Builder resultBuilder = new
ListViewArray.Builder(type.ValueDataType).Reserve(_baseDataTotalElementCount);
- Int64Array.Builder resultValueBuilder =
(Int64Array.Builder)resultBuilder.ValueBuilder.Reserve(_baseDataTotalElementCount);
-
- for (int i = 0; i < _baseDataListCount; i++)
+ builder.Append();
+ ((Int64Array.Builder)builder.ValueBuilder).Append(x);
+ }, initAction: (builder, length) =>
{
- List<int?> dataList = _baseData[i];
+ builder.Reserve(length);
+ builder.ValueBuilder.Reserve(length);
+ });
- ListViewArray.Builder builder = new
ListViewArray.Builder(type.ValueField).Reserve(dataList.Count);
- Int64Array.Builder valueBuilder =
(Int64Array.Builder)builder.ValueBuilder.Reserve(dataList.Count);
+ public void Visit(ListViewType type) =>
+ GenerateTestData<ListViewArray, ListViewArray.Builder>(type,
(builder, x) =>
+ {
+ builder.Append();
+ ((Int64Array.Builder)builder.ValueBuilder).Append(x);
+ }, initAction: (builder, length) =>
+ {
+ builder.Reserve(length);
+ builder.ValueBuilder.Reserve(length);
+ });
- foreach (long? value in dataList)
+ public void Visit(FixedSizeListType type) =>
+ GenerateTestData<FixedSizeListArray,
FixedSizeListArray.Builder>(type, (builder, x) =>
+ {
+ builder.Append();
+ var valueBuilder =
(Int32Array.Builder)builder.ValueBuilder;
+ for (int i = 0; i < type.ListSize; ++i)
{
- if (value.HasValue)
- {
- builder.Append();
- resultBuilder.Append();
-
- valueBuilder.Append(value.Value);
- resultValueBuilder.Append(value.Value);
- }
- else
- {
- builder.AppendNull();
- resultBuilder.AppendNull();
- }
+ valueBuilder.Append(x);
}
+ }, initAction: (builder, length) =>
+ {
+ builder.Reserve(length);
+ builder.ValueBuilder.Reserve(length * type.ListSize);
+ });
- TestTargetArrayList.Add(builder.Build());
- }
-
- ExpectedArray = resultBuilder.Build();
- }
-
- public void Visit(FixedSizeListType type)
+ public void Visit(StructType type)
{
- FixedSizeListArray.Builder resultBuilder = new
FixedSizeListArray.Builder(type.ValueDataType,
type.ListSize).Reserve(_baseDataTotalElementCount);
- Int32Array.Builder resultValueBuilder =
(Int32Array.Builder)resultBuilder.ValueBuilder.Reserve(_baseDataTotalElementCount);
+ // TODO: Make data from type fields.
- for (int i = 0; i < _baseDataListCount; i++)
+ // The following can be improved with a Builder class for
StructArray.
+ StringArray.Builder resultStringBuilder = new
StringArray.Builder().Reserve(_resultTotalElementCount);
+ Int32Array.Builder resultInt32Builder = new
Int32Array.Builder().Reserve(_resultTotalElementCount);
+ ArrowBuffer.BitmapBuilder resultNullBitmapBuilder = new
ArrowBuffer.BitmapBuilder().Reserve(_resultTotalElementCount);
+ int resultNullCount = 0;
+
+ for (int i = 0; i < _baseData.Count; i++)
{
List<int?> dataList = _baseData[i];
+ StringArray.Builder stringBuilder = new
StringArray.Builder().Reserve(dataList.Count);
+ Int32Array.Builder int32Builder = new
Int32Array.Builder().Reserve(dataList.Count);
+ ArrowBuffer.BitmapBuilder nullBitmapBuilder = new
ArrowBuffer.BitmapBuilder().Reserve(dataList.Count);
+ int nullCount = 0;
- FixedSizeListArray.Builder builder = new
FixedSizeListArray.Builder(type.ValueField,
type.ListSize).Reserve(dataList.Count);
- Int32Array.Builder valueBuilder =
(Int32Array.Builder)builder.ValueBuilder.Reserve(dataList.Count);
-
- foreach (int? value in dataList)
+ for (int j = 0; j < dataList.Count; ++j)
{
+ var value = dataList[j];
if (value.HasValue)
{
- builder.Append();
- resultBuilder.Append();
+ nullBitmapBuilder.Append(true);
+ stringBuilder.Append(value.Value.ToString());
+ int32Builder.Append(value.Value);
- valueBuilder.Append(value.Value);
- resultValueBuilder.Append(value.Value);
+ if (IncludeInResult(i, j))
+ {
+ resultNullBitmapBuilder.Append(true);
+
resultStringBuilder.Append(value.Value.ToString());
+ resultInt32Builder.Append(value.Value);
+ }
}
else
{
- builder.AppendNull();
- resultBuilder.AppendNull();
+ nullCount++;
+ nullBitmapBuilder.Append(false);
+ stringBuilder.Append("");
+ int32Builder.Append(0);
+
+ if (IncludeInResult(i, j))
+ {
+ resultNullCount++;
+ resultNullBitmapBuilder.Append(false);
+ resultStringBuilder.Append("");
+ resultInt32Builder.Append(0);
+ }
}
}
- TestTargetArrayList.Add(builder.Build());
- }
-
- ExpectedArray = resultBuilder.Build();
- }
-
- public void Visit(StructType type)
- {
- // TODO: Make data from type fields.
-
- // The following can be improved with a Builder class for
StructArray.
- StringArray.Builder resultStringBuilder = new
StringArray.Builder();
- Int32Array.Builder resultInt32Builder = new
Int32Array.Builder();
- ArrowBuffer nullBitmapBuffer = new
ArrowBuffer.BitmapBuilder().Append(true).Append(true).Append(false).Build();
-
- for (int i = 0; i < 3; i++)
- {
-
resultStringBuilder.Append("joe").AppendNull().AppendNull().Append("mark");
-
resultInt32Builder.Append(1).Append(2).AppendNull().Append(4);
- StringArray stringArray = new
StringArray.Builder().Append("joe").AppendNull().AppendNull().Append("mark").Build();
- Int32Array intArray = new
Int32Array.Builder().Append(1).Append(2).AppendNull().Append(4).Build();
- List<Array> arrays = new List<Array>
+ var arrays = new List<Array>
{
- stringArray,
- intArray
+ stringBuilder.Build(),
+ int32Builder.Build(),
};
- TestTargetArrayList.Add(new StructArray(type, 3, arrays,
nullBitmapBuffer, 1));
+ TestTargetArrayList.Add(SliceTargetArray(
+ new StructArray(type, dataList.Count, arrays,
nullBitmapBuilder.Build(), nullCount), i));
}
- StringArray resultStringArray = resultStringBuilder.Build();
- Int32Array resultInt32Array = resultInt32Builder.Build();
+ var resultArrays = new List<Array>
+ {
+ resultStringBuilder.Build(),
+ resultInt32Builder.Build(),
+ };
- ExpectedArray = new StructArray(type, 9, new List<Array> {
resultStringArray, resultInt32Array }, nullBitmapBuffer, 3);
+ ExpectedArray = new StructArray(
+ type, _resultTotalElementCount, resultArrays,
resultNullBitmapBuilder.Build(), resultNullCount);
}
public void Visit(UnionType type)
{
bool isDense = type.Mode == UnionMode.Dense;
- StringArray.Builder stringResultBuilder = new
StringArray.Builder().Reserve(_baseDataTotalElementCount);
- Int32Array.Builder intResultBuilder = new
Int32Array.Builder().Reserve(_baseDataTotalElementCount);
- ArrowBuffer.Builder<byte> typeResultBuilder = new
ArrowBuffer.Builder<byte>().Reserve(_baseDataTotalElementCount);
- ArrowBuffer.Builder<int> offsetResultBuilder = new
ArrowBuffer.Builder<int>().Reserve(_baseDataTotalElementCount);
+ StringArray.Builder stringResultBuilder = new
StringArray.Builder().Reserve(_resultTotalElementCount);
+ Int32Array.Builder intResultBuilder = new
Int32Array.Builder().Reserve(_resultTotalElementCount);
+ ArrowBuffer.Builder<byte> typeResultBuilder = new
ArrowBuffer.Builder<byte>().Reserve(_resultTotalElementCount);
+ ArrowBuffer.Builder<int> offsetResultBuilder = new
ArrowBuffer.Builder<int>().Reserve(_resultTotalElementCount);
int resultNullCount = 0;
for (int i = 0; i < _baseDataListCount; i++)
@@ -598,41 +404,59 @@ namespace Apache.Arrow.Tests
for (int j = 0; j < dataList.Count; j++)
{
+ bool includeInResult = IncludeInResult(i, j);
byte index = (byte)Math.Min(j % 3, 1);
int? intValue = (index == 1) ? dataList[j] : null;
string stringValue = (index == 1) ? null :
dataList[j]?.ToString();
typeBuilder.Append(index);
- typeResultBuilder.Append(index);
+ if (includeInResult)
+ {
+ typeResultBuilder.Append(index);
+ }
if (isDense)
{
if (index == 0)
{
offsetBuilder.Append(stringBuilder.Length);
-
offsetResultBuilder.Append(stringResultBuilder.Length);
stringBuilder.Append(stringValue);
+ if (includeInResult)
+ {
+
offsetResultBuilder.Append(stringResultBuilder.Length);
+ }
+ // For dense mode, concatenation doesn't slice
the child arrays, so always
+ // add the value to the result.
stringResultBuilder.Append(stringValue);
}
else
{
offsetBuilder.Append(intBuilder.Length);
-
offsetResultBuilder.Append(intResultBuilder.Length);
intBuilder.Append(intValue);
+ if (includeInResult)
+ {
+
offsetResultBuilder.Append(intResultBuilder.Length);
+ }
intResultBuilder.Append(intValue);
}
}
else
{
stringBuilder.Append(stringValue);
- stringResultBuilder.Append(stringValue);
intBuilder.Append(intValue);
- intResultBuilder.Append(intValue);
+ if (includeInResult)
+ {
+ stringResultBuilder.Append(stringValue);
+ intResultBuilder.Append(intValue);
+ }
}
if (dataList[j] == null)
{
nullCount++;
- resultNullCount++;
+ if (includeInResult)
+ {
+ resultNullCount++;
+ }
}
}
@@ -645,9 +469,11 @@ namespace Apache.Arrow.Tests
{
buffers = new[] { typeBuilder.Build() };
}
- TestTargetArrayList.Add(UnionArray.Create(new ArrayData(
+
+ var unionArray = UnionArray.Create(new ArrayData(
type, dataList.Count, nullCount, 0, buffers,
- new[] { stringBuilder.Build().Data,
intBuilder.Build().Data })));
+ new[] { stringBuilder.Build().Data,
intBuilder.Build().Data }));
+ TestTargetArrayList.Add(SliceTargetArray(unionArray, i));
}
ArrowBuffer[] resultBuffers;
@@ -659,50 +485,27 @@ namespace Apache.Arrow.Tests
{
resultBuffers = new[] { typeResultBuilder.Build() };
}
+
ExpectedArray = UnionArray.Create(new ArrayData(
- type, _baseDataTotalElementCount, resultNullCount, 0,
resultBuffers,
+ type, _resultTotalElementCount, resultNullCount, 0,
resultBuffers,
new[] { stringResultBuilder.Build().Data,
intResultBuilder.Build().Data }));
}
- public void Visit(MapType type)
- {
- MapArray.Builder resultBuilder = new
MapArray.Builder(type).Reserve(_baseDataTotalElementCount);
- StringArray.Builder resultKeyBuilder =
(StringArray.Builder)resultBuilder.KeyBuilder.Reserve(_baseDataTotalElementCount);
- Int32Array.Builder resultValueBuilder =
(Int32Array.Builder)resultBuilder.ValueBuilder.Reserve(_baseDataTotalElementCount);
- ArrowBuffer nullBitmapBuilder = new
ArrowBuffer.BitmapBuilder().Append(true).Append(true).Append(false).Build();
-
- for (int i = 0; i < _baseData.Count; i++)
+ public void Visit(MapType type) =>
+ GenerateTestData<MapArray, MapArray.Builder>(type, (builder,
x) =>
{
- List<int?> dataList = _baseData[i];
-
- MapArray.Builder builder = new
MapArray.Builder(type).Reserve(dataList.Count);
- StringArray.Builder keyBuilder =
(StringArray.Builder)builder.KeyBuilder.Reserve(dataList.Count);
- Int32Array.Builder valueBuilder =
(Int32Array.Builder)builder.ValueBuilder.Reserve(dataList.Count);
-
- foreach (int? value in dataList)
- {
- if (value.HasValue)
- {
- builder.Append();
- resultBuilder.Append();
-
- keyBuilder.Append(value.Value.ToString());
- valueBuilder.Append(value.Value);
- resultKeyBuilder.Append(value.Value.ToString());
- resultValueBuilder.Append(value.Value);
- }
- else
- {
- builder.AppendNull();
- resultBuilder.AppendNull();
- }
- }
-
- TestTargetArrayList.Add(builder.Build());
- }
+ var keyBuilder = (StringArray.Builder)builder.KeyBuilder;
+ var valueBuilder =
(Int32Array.Builder)builder.ValueBuilder;
- ExpectedArray = resultBuilder.Build();
- }
+ builder.Append();
+ keyBuilder.Append(x.ToString());
+ valueBuilder.Append(x);
+ }, initAction: (builder, length) =>
+ {
+ builder.Reserve(length);
+ builder.KeyBuilder.Reserve(length);
+ builder.ValueBuilder.Reserve(length);
+ });
public void Visit(IArrowType type)
{
@@ -713,33 +516,87 @@ namespace Apache.Arrow.Tests
where TArrayBuilder : IArrowArrayBuilder<T, TArray,
TArrayBuilder>
where TArray : IArrowArray
{
- var resultBuilder = (IArrowArrayBuilder<T, TArray,
TArrayBuilder>)ArrayArrayBuilderFactoryReflector.InvokeBuild(type);
- resultBuilder.Reserve(_baseDataTotalElementCount);
+ GenerateTestData<TArray, TArrayBuilder>(type, (builder, x) =>
builder.Append(generator(x)));
+ }
+
+ private void GenerateTestData<TArray, TArrayBuilder>(
+ IArrowType type, Action<TArrayBuilder, int> buildAction,
Action<TArrayBuilder, int> initAction=null)
+ where TArrayBuilder : IArrowArrayBuilder<TArray, TArrayBuilder>
+ where TArray : IArrowArray
+ {
+ var resultBuilder =
(TArrayBuilder)ArrowArrayBuilderFactory.Build(type);
+ if (initAction != null)
+ {
+ initAction(resultBuilder, _resultTotalElementCount);
+ }
+ else
+ {
+ resultBuilder.Reserve(_resultTotalElementCount);
+ }
for (int i = 0; i < _baseDataListCount; i++)
{
List<int?> dataList = _baseData[i];
- var builder = (IArrowArrayBuilder<T, TArray,
TArrayBuilder>)ArrayArrayBuilderFactoryReflector.InvokeBuild(type);
- builder.Reserve(dataList.Count);
+ var builder =
(TArrayBuilder)ArrowArrayBuilderFactory.Build(type);
+ if (initAction != null)
+ {
+ initAction(builder, dataList.Count);
+ }
+ else
+ {
+ builder.Reserve(dataList.Count);
+ }
- foreach (int? value in dataList)
+ for (int j = 0; j < dataList.Count; ++j)
{
+ var value = dataList[j];
if (value.HasValue)
{
- builder.Append(generator(value.Value));
- resultBuilder.Append(generator(value.Value));
+ buildAction(builder, value.Value);
+ if (IncludeInResult(i, j))
+ {
+ buildAction(resultBuilder, value.Value);
+ }
}
else
{
builder.AppendNull();
- resultBuilder.AppendNull();
+ if (IncludeInResult(i, j))
+ {
+ resultBuilder.AppendNull();
+ }
}
}
- TestTargetArrayList.Add(builder.Build(default));
+
+
TestTargetArrayList.Add(SliceTargetArray(builder.Build(default), i));
}
ExpectedArray = resultBuilder.Build(default);
}
+
+ private bool IncludeInResult(int listIndex, int itemIndex)
+ {
+ if (_sliceParameters == null)
+ {
+ // Unsliced arrays, all values are expected in the result
+ return true;
+ }
+
+ var sliceParameters = _sliceParameters[listIndex];
+ return itemIndex >= sliceParameters.Offset &&
+ itemIndex < (sliceParameters.Offset +
sliceParameters.Length);
+ }
+
+ private IArrowArray SliceTargetArray(IArrowArray array, int
targetIndex)
+ {
+ if (_sliceParameters == null)
+ {
+ return array;
+ }
+
+ return ArrowArrayFactory.Slice(
+ array, _sliceParameters[targetIndex].Offset,
_sliceParameters[targetIndex].Length);
+ }
}
}
}
