ARROW-1111: [JAVA] Make aligning buffers optional, and allow -1 for unknown null count
Closes #747 Project: http://git-wip-us.apache.org/repos/asf/arrow/repo Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/68b2cc61 Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/68b2cc61 Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/68b2cc61 Branch: refs/heads/master Commit: 68b2cc6112848e373d6b140c9a02958dcb5267a5 Parents: 1ba2c87 Author: Steven Phillips <ste...@dremio.com> Authored: Tue Jun 6 15:19:00 2017 -0700 Committer: Steven Phillips <ste...@dremio.com> Committed: Thu Jun 8 14:58:09 2017 -0700 ---------------------------------------------------------------------- .../java/org/apache/arrow/vector/VectorUnloader.java | 12 ++++++++++-- .../apache/arrow/vector/schema/ArrowRecordBatch.java | 8 ++++++-- 2 files changed, 16 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/arrow/blob/68b2cc61/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java index 8e9ff6d..9965245 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java @@ -29,9 +29,17 @@ import org.apache.arrow.vector.schema.ArrowVectorType; public class VectorUnloader { private final VectorSchemaRoot root; + private final boolean includeNullCount; + private final boolean alignBuffers; public VectorUnloader(VectorSchemaRoot root) { + this(root, true, true); + } + + public VectorUnloader(VectorSchemaRoot root, boolean includeNullCount, boolean alignBuffers) { this.root = root; + this.includeNullCount = includeNullCount; + this.alignBuffers = alignBuffers; } public ArrowRecordBatch getRecordBatch() { @@ -40,12 +48,12 @@ public class VectorUnloader { for (FieldVector vector : root.getFieldVectors()) { appendNodes(vector, nodes, buffers); } - return new ArrowRecordBatch(root.getRowCount(), nodes, buffers); + return new ArrowRecordBatch(root.getRowCount(), nodes, buffers, alignBuffers); } private void appendNodes(FieldVector vector, List<ArrowFieldNode> nodes, List<ArrowBuf> buffers) { Accessor accessor = vector.getAccessor(); - nodes.add(new ArrowFieldNode(accessor.getValueCount(), accessor.getNullCount())); + nodes.add(new ArrowFieldNode(accessor.getValueCount(), includeNullCount ? accessor.getNullCount() : -1)); List<ArrowBuf> fieldBuffers = vector.getFieldBuffers(); List<ArrowVectorType> expectedBuffers = vector.getField().getTypeLayout().getVectorTypes(); if (fieldBuffers.size() != expectedBuffers.size()) { http://git-wip-us.apache.org/repos/asf/arrow/blob/68b2cc61/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java index 6ef514e..6a716fa 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java @@ -48,12 +48,16 @@ public class ArrowRecordBatch implements ArrowMessage { private boolean closed = false; + public ArrowRecordBatch(int length, List<ArrowFieldNode> nodes, List<ArrowBuf> buffers) { + this(length, nodes, buffers, true); + } + /** * @param length how many rows in this batch * @param nodes field level info * @param buffers will be retained until this recordBatch is closed */ - public ArrowRecordBatch(int length, List<ArrowFieldNode> nodes, List<ArrowBuf> buffers) { + public ArrowRecordBatch(int length, List<ArrowFieldNode> nodes, List<ArrowBuf> buffers, boolean alignBuffers) { super(); this.length = length; this.nodes = nodes; @@ -66,7 +70,7 @@ public class ArrowRecordBatch implements ArrowMessage { arrowBuffers.add(new ArrowBuffer(0, offset, size)); LOGGER.debug(String.format("Buffer in RecordBatch at %d, length: %d", offset, size)); offset += size; - if (offset % 8 != 0) { // align on 8 byte boundaries + if (alignBuffers && offset % 8 != 0) { // align on 8 byte boundaries offset += 8 - (offset % 8); } }
