lidavidm commented on code in PR #41732:
URL: https://github.com/apache/arrow/pull/41732#discussion_r1609062291
##########
java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowRecordBatch.java:
##########
@@ -76,12 +78,13 @@ public ArrowRecordBatch(
* @param nodes field level info
* @param buffers will be retained until this recordBatch is closed
* @param bodyCompression compression info.
+ * @param variadicBufferCounts the number of buffers in each variadic
section.
* @param alignBuffers Whether to align buffers to an 8 byte boundary.
*/
public ArrowRecordBatch(
Review Comment:
Create a new constructor and delegate. Don't edit any existing signatures.
##########
java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowRecordBatch.java:
##########
@@ -248,7 +288,8 @@ public void close() {
@Override
public String toString() {
return "ArrowRecordBatch [length=" + length + ", nodes=" + nodes + ",
#buffers=" + buffers.size() +
- ", buffersLayout=" + buffersLayout + ", closed=" + closed + "]";
+ ", #variadicBufferCounts=" + variadicBufferCounts.size() + ",
buffersLayout=" + buffersLayout +
Review Comment:
This needs a null check.
##########
java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java:
##########
@@ -80,19 +80,29 @@ public VectorUnloader(
public ArrowRecordBatch getRecordBatch() {
List<ArrowFieldNode> nodes = new ArrayList<>();
List<ArrowBuf> buffers = new ArrayList<>();
+ List<Long> variadicBufferCounts = new ArrayList<>();
for (FieldVector vector : root.getFieldVectors()) {
- appendNodes(vector, nodes, buffers);
+ appendNodes(vector, nodes, buffers, variadicBufferCounts);
}
// Do NOT retain buffers in ArrowRecordBatch constructor since we have
already retained them.
return new ArrowRecordBatch(
- root.getRowCount(), nodes, buffers,
CompressionUtil.createBodyCompression(codec), alignBuffers,
- /*retainBuffers*/ false);
+ root.getRowCount(), nodes, buffers,
CompressionUtil.createBodyCompression(codec),
+ variadicBufferCounts, alignBuffers, /*retainBuffers*/ false);
}
- private void appendNodes(FieldVector vector, List<ArrowFieldNode> nodes,
List<ArrowBuf> buffers) {
+ private long getVariadicBufferCount(FieldVector vector) {
+ if (vector instanceof BaseVariableWidthViewVector) {
+ return ((BaseVariableWidthViewVector) vector).getDataBuffers().size();
+ }
+ return 0L;
+ }
+
+ private void appendNodes(FieldVector vector, List<ArrowFieldNode> nodes,
List<ArrowBuf> buffers,
+ List<Long> variadicBufferCounts) {
nodes.add(new ArrowFieldNode(vector.getValueCount(), includeNullCount ?
vector.getNullCount() : -1));
List<ArrowBuf> fieldBuffers = vector.getFieldBuffers();
- int expectedBufferCount =
TypeLayout.getTypeBufferCount(vector.getField().getType());
+ int expectedBufferCount =
TypeLayout.getTypeBufferCount(vector.getField().getType(), vector);
Review Comment:
If we have a separate method for variadic buffer count, there should be no
need to pass the vector to get the fixed buffer count here.
##########
java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java:
##########
@@ -95,10 +100,16 @@ private void loadBuffers(
Field field,
Iterator<ArrowBuf> buffers,
Iterator<ArrowFieldNode> nodes,
- CompressionCodec codec) {
+ CompressionCodec codec,
+ Iterator<Long> variadicBufferCounts) {
checkArgument(nodes.hasNext(), "no more field nodes for field %s and
vector %s", field, vector);
ArrowFieldNode fieldNode = nodes.next();
- int bufferLayoutCount = TypeLayout.getTypeBufferCount(field.getType());
+ // variadicBufferLayoutCount will be 0 for vectors of type except
BaseVariableWidthViewVector
+ long variadicBufferLayoutCount = 0;
+ if (variadicBufferCounts != null) {
+ variadicBufferLayoutCount = variadicBufferCounts.next();
+ }
+ int bufferLayoutCount = (int) (variadicBufferLayoutCount +
TypeLayout.getTypeBufferCount(field.getType(), vector));
Review Comment:
When we're loading the vector, the vector shouldn't be relevant for the
buffer count.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
