CurtHagenlocher commented on code in PR #504:
URL: https://github.com/apache/arrow-dotnet/pull/504#discussion_r4581328343
##########
src/Apache.Arrow/Arrays/BinaryArray.cs:
##########
@@ -63,11 +74,16 @@ public abstract class BuilderBase<TArray, TBuilder> :
IArrowArrayBuilder<byte, T
private int _availableValueBufferByteCount;
protected BuilderBase(IArrowType dataType)
+ : this(dataType, null)
+ {
+ }
+
+ protected BuilderBase(IArrowType dataType, MemoryAllocator
allocator)
{
DataType = dataType;
- ValueOffsets = new ArrowBuffer.Builder<int>();
- ValueBuffer = new ArrowBuffer.Builder<byte>();
- ValidityBuffer = new ArrowBuffer.BitmapBuilder();
+ ValueOffsets = new ArrowBuffer.Builder<int>(0, allocator);
+ ValueBuffer = new ArrowBuffer.Builder<byte>(0, allocator);
+ ValidityBuffer = new ArrowBuffer.BitmapBuilder(0, allocator);
Review Comment:
Agreed, fixed in 34dc888. `Build` now releases the buffers already built if
a later one, or the array's construction, throws. I fixed the same pattern in
every builder that builds more than one part: the primitive (the base of all
numeric builders), Boolean, binary, large binary, fixed-size binary, run-end
encoded, timestamp-with-offset and variant builders. A new test builds each one
into an allocator that refuses its n-th allocation, as a budget-enforcing
allocator would, for every n a successful build makes, and checks that neither
allocator has anything left outstanding. It failed for all eight before the fix.
##########
src/Apache.Arrow/Arrays/BooleanArray.cs:
##########
@@ -33,9 +33,22 @@ public class Builder : IArrowArrayBuilder<bool,
BooleanArray, Builder>
public int NullCount => ValidityBuffer.UnsetBitCount;
public Builder()
+ : this(null)
{
- ValueBuffer = new ArrowBuffer.BitmapBuilder();
- ValidityBuffer = new ArrowBuffer.BitmapBuilder();
+ }
+
+ /// <summary>
+ /// Creates a builder whose working memory comes from <paramref
name="allocator"/>.
+ /// </summary>
+ /// <param name="allocator">
+ /// Allocator for the builder's working memory, which usually
becomes the built array's
+ /// memory. When <c>null</c>, the builder works in managed memory
and copies
+ /// what it builds into <see cref="MemoryAllocator.Default"/>, as
builders always have.
+ /// </param>
+ public Builder(MemoryAllocator allocator)
+ {
+ ValueBuffer = new ArrowBuffer.BitmapBuilder(0, allocator);
+ ValidityBuffer = new ArrowBuffer.BitmapBuilder(0, allocator);
Review Comment:
Agreed, fixed in 34dc888. `Build` now releases the validity buffer if
building the values or constructing the array throws. I fixed the same pattern
in every builder that builds more than one part: the primitive (the base of all
numeric builders), Boolean, binary, large binary, fixed-size binary, run-end
encoded, timestamp-with-offset and variant builders. A new test builds each one
into an allocator that refuses its n-th allocation, as a budget-enforcing
allocator would, for every n a successful build makes, and checks that neither
allocator has anything left outstanding. It failed for all eight before the fix.
##########
src/Apache.Arrow/Arrays/FixedSizeBinaryArray.cs:
##########
@@ -98,11 +98,39 @@ public abstract class BuilderBase<TArray, TBuilder> :
IArrowArrayBuilder<byte[],
protected abstract TArray Build(ArrayData data);
protected BuilderBase(IArrowType dataType, int byteWidth)
+ : this(dataType, byteWidth, null)
+ {
+ }
+
+ protected BuilderBase(IArrowType dataType, int byteWidth,
MemoryAllocator allocator)
{
DataType = dataType;
ByteWidth = byteWidth;
- ValueBuffer = new ArrowBuffer.Builder<byte>();
- ValidityBuffer = new ArrowBuffer.BitmapBuilder();
+ ValueBuffer = new ArrowBuffer.Builder<byte>(0, allocator);
+ ValidityBuffer = new ArrowBuffer.BitmapBuilder(0, allocator);
Review Comment:
Agreed, fixed in 34dc888. `Build` now releases the validity buffer if
building the values or constructing the array throws. I fixed the same pattern
in every builder that builds more than one part: the primitive (the base of all
numeric builders), Boolean, binary, large binary, fixed-size binary, run-end
encoded, timestamp-with-offset and variant builders. A new test builds each one
into an allocator that refuses its n-th allocation, as a budget-enforcing
allocator would, for every n a successful build makes, and checks that neither
allocator has anything left outstanding. It failed for all eight before the fix.
##########
src/Apache.Arrow/Arrays/LargeBinaryArray.cs:
##########
@@ -48,18 +59,47 @@ public abstract class BuilderBase<TArray, TBuilder> :
IArrowArrayBuilder<byte, T
protected int NullCount => this.ValidityBuffer.UnsetBitCount;
protected BuilderBase(IArrowType dataType)
+ : this(dataType, null)
+ {
+ }
+
+ protected BuilderBase(IArrowType dataType, MemoryAllocator allocator)
{
DataType = dataType;
- ValueOffsets = new ArrowBuffer.Builder<long>();
- ValueBuffer = new ArrowBuffer.Builder<byte>();
- ValidityBuffer = new ArrowBuffer.BitmapBuilder();
+ ValueOffsets = new ArrowBuffer.Builder<long>(0, allocator);
+ ValueBuffer = new ArrowBuffer.Builder<byte>(0, allocator);
+ ValidityBuffer = new ArrowBuffer.BitmapBuilder(0, allocator);
Review Comment:
Agreed, fixed in 34dc888. `Build` now releases the buffers already built if
a later one, or the array's construction, throws. I fixed the same pattern in
every builder that builds more than one part: the primitive (the base of all
numeric builders), Boolean, binary, large binary, fixed-size binary, run-end
encoded, timestamp-with-offset and variant builders. A new test builds each one
into an allocator that refuses its n-th allocation, as a budget-enforcing
allocator would, for every n a successful build makes, and checks that neither
allocator has anything left outstanding. It failed for all eight before the fix.
##########
src/Apache.Arrow/Arrays/TimestampWithOffsetArray.cs:
##########
@@ -213,11 +213,25 @@ public Builder(TimeUnit unit = TimeUnit.Microsecond)
}
public Builder(TimestampWithOffsetType type)
+ : this(type, null)
+ {
+ }
+
+ /// <summary>
+ /// Creates a builder whose working memory comes from <paramref
name="allocator"/>.
+ /// </summary>
+ /// <param name="type">The timestamp-with-offset type to
build.</param>
+ /// <param name="allocator">
+ /// Allocator for the builder's working memory, which usually
becomes the built array's
+ /// memory. When <c>null</c>, the builder works in managed memory
and copies
+ /// what it builds into <see cref="MemoryAllocator.Default"/>, as
builders always have.
+ /// </param>
+ public Builder(TimestampWithOffsetType type, MemoryAllocator
allocator)
{
_type = type ?? throw new ArgumentNullException(nameof(type));
- _timestampBuilder = new TimestampArray.Builder(type.Unit,
"UTC");
- _offsetBuilder = new Int16Array.Builder();
- _validityBuilder = new ArrowBuffer.BitmapBuilder();
+ _timestampBuilder = new TimestampArray.Builder(new
TimestampType(type.Unit, "UTC"), 0, allocator);
+ _offsetBuilder = new Int16Array.Builder(0, allocator);
+ _validityBuilder = new ArrowBuffer.BitmapBuilder(0, allocator);
Review Comment:
Agreed, fixed in 34dc888. `Build` now releases the child arrays and validity
buffer already built if a later step throws. I fixed the same pattern in every
builder that builds more than one part: the primitive (the base of all numeric
builders), Boolean, binary, large binary, fixed-size binary, run-end encoded,
timestamp-with-offset and variant builders. A new test builds each one into an
allocator that refuses its n-th allocation, as a budget-enforcing allocator
would, for every n a successful build makes, and checks that neither allocator
has anything left outstanding. It failed for all eight before the fix.
##########
src/Apache.Arrow/Arrays/VariantArray.cs:
##########
@@ -390,6 +390,26 @@ public class Builder
private static readonly Lazy<(byte[] Metadata, byte[] Value)>
NullPlaceholder =
new Lazy<(byte[], byte[])>(() => new
VariantBuilder().Encode(VariantValue.Null));
+ public Builder()
+ : this(null)
+ {
+ }
+
+ /// <summary>
+ /// Creates a builder whose working memory comes from <paramref
name="allocator"/>.
+ /// </summary>
+ /// <param name="allocator">
+ /// Allocator for the builder's working memory, which usually
becomes the built array's
+ /// memory. When <c>null</c>, the builder works in managed memory
and copies
+ /// what it builds into <see cref="MemoryAllocator.Default"/>, as
builders always have.
+ /// </param>
+ public Builder(MemoryAllocator allocator)
+ {
+ _metadataBuilder = new BinaryArray.Builder(BinaryType.Default,
allocator);
+ _valueBuilder = new BinaryArray.Builder(BinaryType.Default,
allocator);
+ _validityBuilder = new ArrowBuffer.BitmapBuilder(0, allocator);
Review Comment:
Agreed, fixed in 34dc888. `Build` now releases the child arrays and validity
buffer already built if a later step throws. I fixed the same pattern in every
builder that builds more than one part: the primitive (the base of all numeric
builders), Boolean, binary, large binary, fixed-size binary, run-end encoded,
timestamp-with-offset and variant builders. A new test builds each one into an
allocator that refuses its n-th allocation, as a budget-enforcing allocator
would, for every n a successful build makes, and checks that neither allocator
has anything left outstanding. It failed for all eight before the fix.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]