This is an automated email from the ASF dual-hosted git repository.
lidavidm pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new b33f040640 GH-41272: [Java] LargeListViewVector Implementation (#43516)
b33f040640 is described below
commit b33f040640c7ccb3e6a8406e4d3158608c597025
Author: Vibhatha Lakmal Abeykoon <[email protected]>
AuthorDate: Fri Aug 9 08:12:10 2024 +0530
GH-41272: [Java] LargeListViewVector Implementation (#43516)
### Rationale for this change
This PR includes the initial integration of `LargeListViewVector`.
### What changes are included in this PR?
This PR includes the core functions associated with the
`LargeListViewVector`.
### Are these changes tested?
Yes.
### Are there any user-facing changes?
No
* GitHub Issue: #41272
Authored-by: Vibhatha Abeykoon <[email protected]>
Signed-off-by: David Li <[email protected]>
---
.../jdbc/binder/ColumnBinderArrowTypeVisitor.java | 5 +
.../apache/arrow/c/BufferImportTypeVisitor.java | 6 +
.../driver/jdbc/utils/AvaticaParameterBinder.java | 5 +
.../arrow/driver/jdbc/utils/ConvertUtils.java | 6 +
java/vector/src/main/codegen/data/ArrowTypes.tdd | 5 +
.../src/main/codegen/templates/ComplexCopier.java | 1 +
.../codegen/templates/PromotableViewWriter.java | 13 +-
.../main/codegen/templates/PromotableWriter.java | 47 +
.../main/codegen/templates/UnionListWriter.java | 48 +-
.../src/main/codegen/templates/UnionReader.java | 2 +-
.../java/org/apache/arrow/vector/BufferLayout.java | 5 +
.../java/org/apache/arrow/vector/TypeLayout.java | 16 +
.../complex/BaseLargeRepeatedValueViewVector.java | 406 +++++
.../arrow/vector/complex/LargeListViewVector.java | 875 +++++++++++
.../complex/impl/UnionLargeListViewReader.java | 116 ++
.../apache/arrow/vector/extension/OpaqueType.java | 5 +
.../java/org/apache/arrow/vector/types/Types.java | 21 +
.../arrow/vector/TestLargeListViewVector.java | 1649 ++++++++++++++++++++
18 files changed, 3225 insertions(+), 6 deletions(-)
diff --git
a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/ColumnBinderArrowTypeVisitor.java
b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/ColumnBinderArrowTypeVisitor.java
index 30b2305f3f..cb8e43035d 100644
---
a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/ColumnBinderArrowTypeVisitor.java
+++
b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/ColumnBinderArrowTypeVisitor.java
@@ -284,4 +284,9 @@ public class ColumnBinderArrowTypeVisitor implements
ArrowType.ArrowTypeVisitor<
public ColumnBinder visit(ArrowType.ListView type) {
throw new UnsupportedOperationException("No column binder implemented for
type " + type);
}
+
+ @Override
+ public ColumnBinder visit(ArrowType.LargeListView type) {
+ throw new UnsupportedOperationException("No column binder implemented for
type " + type);
+ }
}
diff --git
a/java/c/src/main/java/org/apache/arrow/c/BufferImportTypeVisitor.java
b/java/c/src/main/java/org/apache/arrow/c/BufferImportTypeVisitor.java
index 5f262d3dc3..633ecd43bd 100644
--- a/java/c/src/main/java/org/apache/arrow/c/BufferImportTypeVisitor.java
+++ b/java/c/src/main/java/org/apache/arrow/c/BufferImportTypeVisitor.java
@@ -403,4 +403,10 @@ class BufferImportTypeVisitor implements
ArrowType.ArrowTypeVisitor<List<ArrowBu
throw new UnsupportedOperationException(
"Importing buffers for view type: " + type + " not supported");
}
+
+ @Override
+ public List<ArrowBuf> visit(ArrowType.LargeListView type) {
+ throw new UnsupportedOperationException(
+ "Importing buffers for view type: " + type + " not supported");
+ }
}
diff --git
a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/AvaticaParameterBinder.java
b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/AvaticaParameterBinder.java
index 7acffb4bc9..232fa15240 100644
---
a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/AvaticaParameterBinder.java
+++
b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/AvaticaParameterBinder.java
@@ -276,5 +276,10 @@ public class AvaticaParameterBinder {
public Boolean visit(ArrowType.ListView type) {
throw new UnsupportedOperationException("Binding is not yet supported
for type " + type);
}
+
+ @Override
+ public Boolean visit(ArrowType.LargeListView type) {
+ throw new UnsupportedOperationException("Binding is not yet supported
for type " + type);
+ }
}
}
diff --git
a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ConvertUtils.java
b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ConvertUtils.java
index 77b7a88536..ea57aeb774 100644
---
a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ConvertUtils.java
+++
b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ConvertUtils.java
@@ -278,5 +278,11 @@ public final class ConvertUtils {
throw new UnsupportedOperationException(
"AvaticaParameter not yet supported for type " + type);
}
+
+ @Override
+ public AvaticaParameter visit(ArrowType.LargeListView type) {
+ throw new UnsupportedOperationException(
+ "AvaticaParameter not yet supported for type " + type);
+ }
}
}
diff --git a/java/vector/src/main/codegen/data/ArrowTypes.tdd
b/java/vector/src/main/codegen/data/ArrowTypes.tdd
index 72df477979..d0e8ef1e35 100644
--- a/java/vector/src/main/codegen/data/ArrowTypes.tdd
+++ b/java/vector/src/main/codegen/data/ArrowTypes.tdd
@@ -134,6 +134,11 @@
name: "ListView",
fields: [],
complex: true
+ },
+ {
+ name: "LargeListView",
+ fields: [],
+ complex: true
}
]
}
diff --git a/java/vector/src/main/codegen/templates/ComplexCopier.java
b/java/vector/src/main/codegen/templates/ComplexCopier.java
index 1eebba018b..5adad52312 100644
--- a/java/vector/src/main/codegen/templates/ComplexCopier.java
+++ b/java/vector/src/main/codegen/templates/ComplexCopier.java
@@ -53,6 +53,7 @@ public class ComplexCopier {
case LIST:
case LISTVIEW:
case LARGELIST:
+ case LARGELISTVIEW:
case FIXED_SIZE_LIST:
if (reader.isSet()) {
writer.startList();
diff --git a/java/vector/src/main/codegen/templates/PromotableViewWriter.java
b/java/vector/src/main/codegen/templates/PromotableViewWriter.java
index 373abbe4b9..a40901e295 100644
--- a/java/vector/src/main/codegen/templates/PromotableViewWriter.java
+++ b/java/vector/src/main/codegen/templates/PromotableViewWriter.java
@@ -68,6 +68,15 @@ public class PromotableViewWriter extends PromotableWriter {
super(v, listViewVector, nullableStructWriterFactory);
}
+ public PromotableViewWriter(ValueVector v, LargeListViewVector
largeListViewVector) {
+ super(v, largeListViewVector);
+ }
+
+ public PromotableViewWriter(ValueVector v, LargeListViewVector
largeListViewVector,
+ NullableStructWriterFactory nullableStructWriterFactory) {
+ super(v, largeListViewVector, nullableStructWriterFactory);
+ }
+
public PromotableViewWriter(ValueVector v, AbstractStructVector
parentContainer) {
super(v, parentContainer);
}
@@ -103,8 +112,10 @@ public class PromotableViewWriter extends PromotableWriter
{
v = fixedListVector.addOrGetVector(fieldType).getVector();
} else if (listViewVector != null) {
v = listViewVector.addOrGetVector(fieldType).getVector();
- } else {
+ } else if (largeListVector != null) {
v = largeListVector.addOrGetVector(fieldType).getVector();
+ } else {
+ v = largeListViewVector.addOrGetVector(fieldType).getVector();
}
v.allocateNew();
setWriter(v);
diff --git a/java/vector/src/main/codegen/templates/PromotableWriter.java
b/java/vector/src/main/codegen/templates/PromotableWriter.java
index 82bd3c5345..c0e686f317 100644
--- a/java/vector/src/main/codegen/templates/PromotableWriter.java
+++ b/java/vector/src/main/codegen/templates/PromotableWriter.java
@@ -41,6 +41,7 @@ public class PromotableWriter extends
AbstractPromotableFieldWriter {
protected final ListViewVector listViewVector;
protected final FixedSizeListVector fixedListVector;
protected final LargeListVector largeListVector;
+ protected final LargeListViewVector largeListViewVector;
protected final NullableStructWriterFactory nullableStructWriterFactory;
protected int position;
protected static final int MAX_DECIMAL_PRECISION = 38;
@@ -84,6 +85,7 @@ public class PromotableWriter extends
AbstractPromotableFieldWriter {
this.listViewVector = null;
this.fixedListVector = null;
this.largeListVector = null;
+ this.largeListViewVector = null;
this.nullableStructWriterFactory = nullableStructWriterFactory;
init(v);
}
@@ -118,6 +120,26 @@ public class PromotableWriter extends
AbstractPromotableFieldWriter {
this(v, largeListVector,
NullableStructWriterFactory.getNullableStructWriterFactoryInstance());
}
+ /**
+ * Constructs a new instance.
+ *
+ * @param v The vector to initialize the writer with.
+ * @param listViewVector The vector that serves as a parent of v.
+ */
+ public PromotableWriter(ValueVector v, ListViewVector listViewVector) {
+ this(v, listViewVector,
NullableStructWriterFactory.getNullableStructWriterFactoryInstance());
+ }
+
+ /**
+ * Constructs a new instance.
+ *
+ * @param v The vector to initialize the writer with.
+ * @param largeListViewVector The vector that serves as a parent of v.
+ */
+ public PromotableWriter(ValueVector v, LargeListViewVector
largeListViewVector) {
+ this(v, largeListViewVector,
NullableStructWriterFactory.getNullableStructWriterFactoryInstance());
+ }
+
/**
* Constructs a new instance.
*
@@ -134,6 +156,7 @@ public class PromotableWriter extends
AbstractPromotableFieldWriter {
this.parentContainer = null;
this.fixedListVector = null;
this.largeListVector = null;
+ this.largeListViewVector = null;
this.nullableStructWriterFactory = nullableStructWriterFactory;
init(v);
}
@@ -154,6 +177,7 @@ public class PromotableWriter extends
AbstractPromotableFieldWriter {
this.parentContainer = null;
this.fixedListVector = null;
this.largeListVector = null;
+ this.largeListViewVector = null;
this.nullableStructWriterFactory = nullableStructWriterFactory;
init(v);
}
@@ -174,6 +198,7 @@ public class PromotableWriter extends
AbstractPromotableFieldWriter {
this.listVector = null;
this.listViewVector = null;
this.largeListVector = null;
+ this.largeListViewVector = null;
this.nullableStructWriterFactory = nullableStructWriterFactory;
init(v);
}
@@ -194,6 +219,28 @@ public class PromotableWriter extends
AbstractPromotableFieldWriter {
this.parentContainer = null;
this.listVector = null;
this.listViewVector = null;
+ this.largeListViewVector = null;
+ this.nullableStructWriterFactory = nullableStructWriterFactory;
+ init(v);
+ }
+
+ /**
+ * Constructs a new instance.
+ *
+ * @param v The vector to initialize the writer with.
+ * @param largeListViewVector The vector that serves as a parent of v.
+ * @param nullableStructWriterFactory The factory to create the delegate
writer.
+ */
+ public PromotableWriter(
+ ValueVector v,
+ LargeListViewVector largeListViewVector,
+ NullableStructWriterFactory nullableStructWriterFactory) {
+ this.largeListViewVector = largeListViewVector;
+ this.fixedListVector = null;
+ this.parentContainer = null;
+ this.listVector = null;
+ this.listViewVector = null;
+ this.largeListVector = null;
this.nullableStructWriterFactory = nullableStructWriterFactory;
init(v);
}
diff --git a/java/vector/src/main/codegen/templates/UnionListWriter.java
b/java/vector/src/main/codegen/templates/UnionListWriter.java
index e40c70eaff..3962e1d073 100644
--- a/java/vector/src/main/codegen/templates/UnionListWriter.java
+++ b/java/vector/src/main/codegen/templates/UnionListWriter.java
@@ -26,7 +26,7 @@ import java.lang.UnsupportedOperationException;
import java.math.BigDecimal;
<@pp.dropOutputFile />
-<#list ["List", "ListView", "LargeList"] as listName>
+<#list ["List", "ListView", "LargeList", "LargeListView"] as listName>
<@pp.changeOutputFile
name="/org/apache/arrow/vector/complex/impl/Union${listName}Writer.java" />
@@ -53,15 +53,18 @@ public class Union${listName}Writer extends
AbstractFieldWriter {
private boolean inStruct = false;
private boolean listStarted = false;
private String structName;
- <#if listName == "LargeList">
+ <#if listName == "LargeList" || listName == "LargeListView">
private static final long OFFSET_WIDTH = 8;
<#else>
private static final int OFFSET_WIDTH = 4;
</#if>
- <#if listName = "ListView">
+ <#if listName == "ListView">
private static final long SIZE_WIDTH = 4;
</#if>
+ <#if listName == "LargeListView">
+ private static final long SIZE_WIDTH = 8;
+ </#if>
public Union${listName}Writer(${listName}Vector vector) {
this(vector,
NullableStructWriterFactory.getNullableStructWriterFactoryInstance());
@@ -69,7 +72,7 @@ public class Union${listName}Writer extends
AbstractFieldWriter {
public Union${listName}Writer(${listName}Vector vector,
NullableStructWriterFactory nullableStructWriterFactory) {
this.vector = vector;
- <#if listName = "ListView">
+ <#if listName = "ListView" || listName = "LargeListView">
this.writer = new PromotableViewWriter(vector.getDataVector(), vector,
nullableStructWriterFactory);
<#else>
this.writer = new PromotableWriter(vector.getDataVector(), vector,
nullableStructWriterFactory);
@@ -231,12 +234,49 @@ public class Union${listName}Writer extends
AbstractFieldWriter {
listStarted = false;
}
+ @Override
public void startListView() {
vector.startNewValue(idx());
writer.setPosition(vector.getOffsetBuffer().getInt((idx()) *
OFFSET_WIDTH));
listStarted = true;
}
+ @Override
+ public void endListView() {
+ int sizeUptoIdx = 0;
+ for (int i = 0; i < idx(); i++) {
+ sizeUptoIdx += vector.getSizeBuffer().getInt(i * SIZE_WIDTH);
+ }
+ vector.getSizeBuffer().setInt(idx() * SIZE_WIDTH, writer.idx() -
sizeUptoIdx);
+ setPosition(idx() + 1);
+ listStarted = false;
+ }
+ <#elseif listName == "LargeListView">
+ @Override
+ public void startList() {
+ vector.startNewValue(idx());
+ writer.setPosition(vector.getOffsetBuffer().getInt((idx()) *
OFFSET_WIDTH));
+ listStarted = true;
+ }
+
+ @Override
+ public void endList() {
+ int sizeUptoIdx = 0;
+ for (int i = 0; i < idx(); i++) {
+ sizeUptoIdx += vector.getSizeBuffer().getInt(i * SIZE_WIDTH);
+ }
+ vector.getSizeBuffer().setInt(idx() * SIZE_WIDTH, writer.idx() -
sizeUptoIdx);
+ setPosition(idx() + 1);
+ listStarted = false;
+ }
+
+ @Override
+ public void startListView() {
+ vector.startNewValue(idx());
+
writer.setPosition(checkedCastToInt(vector.getOffsetBuffer().getInt((idx()) *
OFFSET_WIDTH)));
+ listStarted = true;
+ }
+
@Override
public void endListView() {
int sizeUptoIdx = 0;
diff --git a/java/vector/src/main/codegen/templates/UnionReader.java
b/java/vector/src/main/codegen/templates/UnionReader.java
index 615ea3a536..d2b2f4bb70 100644
--- a/java/vector/src/main/codegen/templates/UnionReader.java
+++ b/java/vector/src/main/codegen/templates/UnionReader.java
@@ -39,7 +39,7 @@ package org.apache.arrow.vector.complex.impl;
@SuppressWarnings("unused")
public class UnionReader extends AbstractFieldReader {
- private static final int NUM_SUPPORTED_TYPES = 49;
+ private static final int NUM_SUPPORTED_TYPES = 50;
private BaseReader[] readers = new BaseReader[NUM_SUPPORTED_TYPES];
public UnionVector data;
diff --git
a/java/vector/src/main/java/org/apache/arrow/vector/BufferLayout.java
b/java/vector/src/main/java/org/apache/arrow/vector/BufferLayout.java
index d4248c4ef9..6c6fd919ab 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BufferLayout.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BufferLayout.java
@@ -61,6 +61,7 @@ public class BufferLayout {
private static final BufferLayout VALUES_32 = new
BufferLayout(BufferType.DATA, 32);
private static final BufferLayout VALUES_16 = new
BufferLayout(BufferType.DATA, 16);
private static final BufferLayout VALUES_8 = new
BufferLayout(BufferType.DATA, 8);
+ private static final BufferLayout LARGE_SIZE_BUFFER = new
BufferLayout(BufferType.SIZE, 64);
private static final BufferLayout SIZE_BUFFER = new
BufferLayout(BufferType.SIZE, 32);
private static final BufferLayout VIEW_BUFFER = new
BufferLayout(BufferType.VIEWS, 16);
@@ -80,6 +81,10 @@ public class BufferLayout {
return SIZE_BUFFER;
}
+ public static BufferLayout largeSizeBuffer() {
+ return LARGE_SIZE_BUFFER;
+ }
+
/**
* Returns a databuffer for the given bitwidth. Only supports powers of two
between 8 and 128
* inclusive.
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TypeLayout.java
b/java/vector/src/main/java/org/apache/arrow/vector/TypeLayout.java
index b8535532ea..78a3cac020 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/TypeLayout.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/TypeLayout.java
@@ -116,6 +116,16 @@ public class TypeLayout {
return new TypeLayout(vectors);
}
+ @Override
+ public TypeLayout visit(ArrowType.LargeListView type) {
+ List<BufferLayout> vectors =
+ asList(
+ BufferLayout.validityVector(),
+ BufferLayout.largeOffsetBuffer(),
+ BufferLayout.largeSizeBuffer());
+ return new TypeLayout(vectors);
+ }
+
@Override
public TypeLayout visit(ArrowType.LargeList type) {
List<BufferLayout> vectors =
@@ -340,6 +350,12 @@ public class TypeLayout {
return 2;
}
+ @Override
+ public Integer visit(ArrowType.LargeListView type) {
+ // validity buffer + offset buffer + size buffer
+ return 3;
+ }
+
@Override
public Integer visit(FixedSizeList type) {
// validity buffer
diff --git
a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseLargeRepeatedValueViewVector.java
b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseLargeRepeatedValueViewVector.java
new file mode 100644
index 0000000000..26079cbee9
--- /dev/null
+++
b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseLargeRepeatedValueViewVector.java
@@ -0,0 +1,406 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.complex;
+
+import static org.apache.arrow.memory.util.LargeMemoryUtil.capAtMaxInt;
+import static org.apache.arrow.memory.util.LargeMemoryUtil.checkedCastToInt;
+
+import java.util.Collections;
+import java.util.Iterator;
+import org.apache.arrow.memory.ArrowBuf;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.util.CommonUtil;
+import org.apache.arrow.util.Preconditions;
+import org.apache.arrow.vector.AddOrGetResult;
+import org.apache.arrow.vector.BaseFixedWidthVector;
+import org.apache.arrow.vector.BaseValueVector;
+import org.apache.arrow.vector.BaseVariableWidthVector;
+import org.apache.arrow.vector.DensityAwareVector;
+import org.apache.arrow.vector.FieldVector;
+import org.apache.arrow.vector.NullVector;
+import org.apache.arrow.vector.UInt4Vector;
+import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.ZeroVector;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.util.CallBack;
+import org.apache.arrow.vector.util.OversizedAllocationException;
+import org.apache.arrow.vector.util.SchemaChangeRuntimeException;
+
+public abstract class BaseLargeRepeatedValueViewVector extends BaseValueVector
+ implements RepeatedValueVector, FieldVector {
+ public static final FieldVector DEFAULT_DATA_VECTOR = ZeroVector.INSTANCE;
+ public static final String DATA_VECTOR_NAME = "$data$";
+
+ public static final byte OFFSET_WIDTH = 8;
+ public static final byte SIZE_WIDTH = 8;
+ protected ArrowBuf offsetBuffer;
+ protected ArrowBuf sizeBuffer;
+ protected FieldVector vector;
+ protected final CallBack repeatedCallBack;
+ protected int valueCount;
+ protected long offsetAllocationSizeInBytes = INITIAL_VALUE_ALLOCATION *
OFFSET_WIDTH;
+ protected long sizeAllocationSizeInBytes = INITIAL_VALUE_ALLOCATION *
SIZE_WIDTH;
+ private final String name;
+
+ protected String defaultDataVectorName = DATA_VECTOR_NAME;
+
+ protected BaseLargeRepeatedValueViewVector(
+ String name, BufferAllocator allocator, CallBack callBack) {
+ this(name, allocator, DEFAULT_DATA_VECTOR, callBack);
+ }
+
+ protected BaseLargeRepeatedValueViewVector(
+ String name, BufferAllocator allocator, FieldVector vector, CallBack
callBack) {
+ super(allocator);
+ this.name = name;
+ this.offsetBuffer = allocator.getEmpty();
+ this.sizeBuffer = allocator.getEmpty();
+ this.vector = Preconditions.checkNotNull(vector, "data vector cannot be
null");
+ this.repeatedCallBack = callBack;
+ this.valueCount = 0;
+ }
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public boolean allocateNewSafe() {
+ boolean dataAlloc = false;
+ try {
+ allocateBuffers();
+ dataAlloc = vector.allocateNewSafe();
+ } catch (Exception e) {
+ clear();
+ return false;
+ } finally {
+ if (!dataAlloc) {
+ clear();
+ }
+ }
+ return dataAlloc;
+ }
+
+ private void allocateBuffers() {
+ offsetBuffer = allocateBuffers(offsetAllocationSizeInBytes);
+ sizeBuffer = allocateBuffers(sizeAllocationSizeInBytes);
+ }
+
+ private ArrowBuf allocateBuffers(final long size) {
+ final int curSize = (int) size;
+ ArrowBuf buffer = allocator.buffer(curSize);
+ buffer.readerIndex(0);
+ buffer.setZero(0, buffer.capacity());
+ return buffer;
+ }
+
+ @Override
+ public void reAlloc() {
+ reallocateBuffers();
+ vector.reAlloc();
+ }
+
+ protected void reallocateBuffers() {
+ reallocOffsetBuffer();
+ reallocSizeBuffer();
+ }
+
+ private void reallocOffsetBuffer() {
+ final long currentBufferCapacity = offsetBuffer.capacity();
+ long newAllocationSize = currentBufferCapacity * 2;
+ if (newAllocationSize == 0) {
+ if (offsetAllocationSizeInBytes > 0) {
+ newAllocationSize = offsetAllocationSizeInBytes;
+ } else {
+ newAllocationSize = INITIAL_VALUE_ALLOCATION * OFFSET_WIDTH * 2;
+ }
+ }
+
+ newAllocationSize = CommonUtil.nextPowerOfTwo(newAllocationSize);
+ newAllocationSize = Math.min(newAllocationSize, (long) OFFSET_WIDTH *
Integer.MAX_VALUE);
+ assert newAllocationSize >= 1;
+
+ if (newAllocationSize > MAX_ALLOCATION_SIZE || newAllocationSize <=
offsetBuffer.capacity()) {
+ throw new OversizedAllocationException("Unable to expand the buffer");
+ }
+
+ final ArrowBuf newBuf = allocator.buffer(newAllocationSize);
+ newBuf.setBytes(0, offsetBuffer, 0, currentBufferCapacity);
+ newBuf.setZero(currentBufferCapacity, newBuf.capacity() -
currentBufferCapacity);
+ offsetBuffer.getReferenceManager().release(1);
+ offsetBuffer = newBuf;
+ offsetAllocationSizeInBytes = newAllocationSize;
+ }
+
+ private void reallocSizeBuffer() {
+ final long currentBufferCapacity = sizeBuffer.capacity();
+ long newAllocationSize = currentBufferCapacity * 2;
+ if (newAllocationSize == 0) {
+ if (sizeAllocationSizeInBytes > 0) {
+ newAllocationSize = sizeAllocationSizeInBytes;
+ } else {
+ newAllocationSize = INITIAL_VALUE_ALLOCATION * SIZE_WIDTH * 2;
+ }
+ }
+
+ newAllocationSize = CommonUtil.nextPowerOfTwo(newAllocationSize);
+ newAllocationSize = Math.min(newAllocationSize, (long) SIZE_WIDTH *
Integer.MAX_VALUE);
+ assert newAllocationSize >= 1;
+
+ if (newAllocationSize > MAX_ALLOCATION_SIZE || newAllocationSize <=
sizeBuffer.capacity()) {
+ throw new OversizedAllocationException("Unable to expand the buffer");
+ }
+
+ final ArrowBuf newBuf = allocator.buffer(newAllocationSize);
+ newBuf.setBytes(0, sizeBuffer, 0, currentBufferCapacity);
+ newBuf.setZero(currentBufferCapacity, newBuf.capacity() -
currentBufferCapacity);
+ sizeBuffer.getReferenceManager().release(1);
+ sizeBuffer = newBuf;
+ sizeAllocationSizeInBytes = newAllocationSize;
+ }
+
+ @Override
+ public FieldVector getDataVector() {
+ return vector;
+ }
+
+ @Override
+ public void setInitialCapacity(int numRecords) {
+ offsetAllocationSizeInBytes = (long) (numRecords) * OFFSET_WIDTH;
+ sizeAllocationSizeInBytes = (long) (numRecords) * SIZE_WIDTH;
+ if (vector instanceof BaseFixedWidthVector || vector instanceof
BaseVariableWidthVector) {
+ vector.setInitialCapacity(numRecords *
RepeatedValueVector.DEFAULT_REPEAT_PER_RECORD);
+ } else {
+ vector.setInitialCapacity(numRecords);
+ }
+ }
+
+ @Override
+ public void setInitialCapacity(int numRecords, double density) {
+ if ((numRecords * density) >= Integer.MAX_VALUE) {
+ throw new OversizedAllocationException("Requested amount of memory is
more than max allowed");
+ }
+
+ offsetAllocationSizeInBytes = (long) numRecords * OFFSET_WIDTH;
+ sizeAllocationSizeInBytes = (long) numRecords * SIZE_WIDTH;
+
+ int innerValueCapacity = Math.max((int) (numRecords * density), 1);
+
+ if (vector instanceof DensityAwareVector) {
+ ((DensityAwareVector) vector).setInitialCapacity(innerValueCapacity,
density);
+ } else {
+ vector.setInitialCapacity(innerValueCapacity);
+ }
+ }
+
+ /**
+ * Specialized version of setInitialTotalCapacity() for LargeListViewVector.
This is used by some
+ * callers when they want to explicitly control and be conservative about
memory allocated for
+ * inner data vector. This is very useful when we are working with memory
constraints for a query
+ * and have a fixed amount of memory reserved for the record batch. In such
cases, we are likely
+ * to face OOM or related problems when we reserve memory for a record batch
with value count x
+ * and do setInitialCapacity(x) such that each vector allocates only what is
necessary and not the
+ * default amount, but the multiplier forces the memory requirement to go
beyond what was needed.
+ *
+ * @param numRecords value count
+ * @param totalNumberOfElements the total number of elements to allow for in
this vector across
+ * all records.
+ */
+ public void setInitialTotalCapacity(int numRecords, int
totalNumberOfElements) {
+ offsetAllocationSizeInBytes = (long) numRecords * OFFSET_WIDTH;
+ sizeAllocationSizeInBytes = (long) numRecords * SIZE_WIDTH;
+ vector.setInitialCapacity(totalNumberOfElements);
+ }
+
+ @Override
+ public int getValueCapacity() {
+ throw new UnsupportedOperationException(
+ "Get value capacity is not supported in RepeatedValueVector");
+ }
+
+ protected int getOffsetBufferValueCapacity() {
+ return checkedCastToInt(offsetBuffer.capacity() / OFFSET_WIDTH);
+ }
+
+ protected int getSizeBufferValueCapacity() {
+ return capAtMaxInt(sizeBuffer.capacity() / SIZE_WIDTH);
+ }
+
+ @Override
+ public int getBufferSize() {
+ if (valueCount == 0) {
+ return 0;
+ }
+ return (valueCount * OFFSET_WIDTH) + (valueCount * SIZE_WIDTH) +
vector.getBufferSize();
+ }
+
+ @Override
+ public int getBufferSizeFor(int valueCount) {
+ if (valueCount == 0) {
+ return 0;
+ }
+
+ int innerVectorValueCount = 0;
+
+ for (int i = 0; i < valueCount; i++) {
+ innerVectorValueCount += sizeBuffer.getInt(i * SIZE_WIDTH);
+ }
+
+ return (valueCount * OFFSET_WIDTH)
+ + (valueCount * SIZE_WIDTH)
+ + vector.getBufferSizeFor(checkedCastToInt(innerVectorValueCount));
+ }
+
+ @Override
+ public Iterator<ValueVector> iterator() {
+ return Collections.<ValueVector>singleton(getDataVector()).iterator();
+ }
+
+ @Override
+ public void clear() {
+ offsetBuffer = releaseBuffer(offsetBuffer);
+ sizeBuffer = releaseBuffer(sizeBuffer);
+ vector.clear();
+ valueCount = 0;
+ super.clear();
+ }
+
+ @Override
+ public void reset() {
+ offsetBuffer.setZero(0, offsetBuffer.capacity());
+ sizeBuffer.setZero(0, sizeBuffer.capacity());
+ vector.reset();
+ valueCount = 0;
+ }
+
+ @Override
+ public ArrowBuf[] getBuffers(boolean clear) {
+ return new ArrowBuf[0];
+ }
+
+ @Override
+ public int getValueCount() {
+ return valueCount;
+ }
+
+ @Override
+ public void setValueCount(int valueCount) {
+ this.valueCount = valueCount;
+ while (valueCount > getOffsetBufferValueCapacity()) {
+ reallocateBuffers();
+ }
+ final int childValueCount = valueCount == 0 ? 0 : getLengthOfChildVector();
+ vector.setValueCount(childValueCount);
+ }
+
+ protected int getLengthOfChildVector() {
+ int maxOffsetSizeSum = offsetBuffer.getInt(0) + sizeBuffer.getInt(0);
+ int minOffset = offsetBuffer.getInt(0);
+ for (int i = 0; i < valueCount; i++) {
+ int currentOffset = offsetBuffer.getInt((long) i * OFFSET_WIDTH);
+ int currentSize = sizeBuffer.getInt((long) i * SIZE_WIDTH);
+ int currentSum = currentOffset + currentSize;
+
+ maxOffsetSizeSum = Math.max(maxOffsetSizeSum, currentSum);
+ minOffset = Math.min(minOffset, currentOffset);
+ }
+
+ return maxOffsetSizeSum - minOffset;
+ }
+
+ protected int getLengthOfChildVectorByIndex(int index) {
+ int maxOffsetSizeSum = offsetBuffer.getInt(0) + sizeBuffer.getInt(0);
+ int minOffset = offsetBuffer.getInt(0);
+ for (int i = 0; i < index; i++) {
+ int currentOffset = offsetBuffer.getInt((long) i * OFFSET_WIDTH);
+ int currentSize = sizeBuffer.getInt((long) i * SIZE_WIDTH);
+ int currentSum = currentOffset + currentSize;
+
+ maxOffsetSizeSum = Math.max(maxOffsetSizeSum, currentSum);
+ minOffset = Math.min(minOffset, currentOffset);
+ }
+
+ return maxOffsetSizeSum - minOffset;
+ }
+
+ /**
+ * Initialize the data vector (and execute callback) if it hasn't already
been done, returns the
+ * data vector.
+ */
+ public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(FieldType
fieldType) {
+ boolean created = false;
+ if (vector instanceof NullVector) {
+ vector = fieldType.createNewSingleVector(defaultDataVectorName,
allocator, repeatedCallBack);
+ // returned vector must have the same field
+ created = true;
+ if (repeatedCallBack != null
+ &&
+ // not a schema change if changing from ZeroVector to ZeroVector
+ (fieldType.getType().getTypeID() != ArrowType.ArrowTypeID.Null)) {
+ repeatedCallBack.doWork();
+ }
+ }
+
+ if (vector.getField().getType().getTypeID() !=
fieldType.getType().getTypeID()) {
+ final String msg =
+ String.format(
+ "Inner vector type mismatch. Requested type: [%s], actual type:
[%s]",
+ fieldType.getType().getTypeID(),
vector.getField().getType().getTypeID());
+ throw new SchemaChangeRuntimeException(msg);
+ }
+
+ return new AddOrGetResult<>((T) vector, created);
+ }
+
+ protected void replaceDataVector(FieldVector v) {
+ vector.clear();
+ vector = v;
+ }
+
+ public abstract boolean isEmpty(int index);
+
+ /**
+ * Start a new value at the given index.
+ *
+ * @param index the index to start the new value at
+ * @return the offset in the data vector where the new value starts
+ */
+ public int startNewValue(int index) {
+ while (index >= getOffsetBufferValueCapacity()) {
+ reallocOffsetBuffer();
+ }
+ while (index >= getSizeBufferValueCapacity()) {
+ reallocSizeBuffer();
+ }
+
+ if (index > 0) {
+ final int prevOffset = getLengthOfChildVectorByIndex(index);
+ offsetBuffer.setInt((long) index * OFFSET_WIDTH, prevOffset);
+ }
+
+ setValueCount(index + 1);
+ return offsetBuffer.getInt((long) index * OFFSET_WIDTH);
+ }
+
+ @Override
+ @Deprecated
+ public UInt4Vector getOffsetVector() {
+ throw new UnsupportedOperationException("There is no inner offset vector");
+ }
+}
diff --git
a/java/vector/src/main/java/org/apache/arrow/vector/complex/LargeListViewVector.java
b/java/vector/src/main/java/org/apache/arrow/vector/complex/LargeListViewVector.java
new file mode 100644
index 0000000000..1bb24a53fc
--- /dev/null
+++
b/java/vector/src/main/java/org/apache/arrow/vector/complex/LargeListViewVector.java
@@ -0,0 +1,875 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.complex;
+
+import static java.util.Collections.singletonList;
+import static org.apache.arrow.memory.util.LargeMemoryUtil.capAtMaxInt;
+import static org.apache.arrow.memory.util.LargeMemoryUtil.checkedCastToInt;
+import static org.apache.arrow.util.Preconditions.checkArgument;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import org.apache.arrow.memory.ArrowBuf;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.OutOfMemoryException;
+import org.apache.arrow.memory.util.ArrowBufPointer;
+import org.apache.arrow.memory.util.ByteFunctionHelpers;
+import org.apache.arrow.memory.util.CommonUtil;
+import org.apache.arrow.memory.util.hash.ArrowBufHasher;
+import org.apache.arrow.util.Preconditions;
+import org.apache.arrow.vector.AddOrGetResult;
+import org.apache.arrow.vector.BitVectorHelper;
+import org.apache.arrow.vector.BufferBacked;
+import org.apache.arrow.vector.FieldVector;
+import org.apache.arrow.vector.ValueIterableVector;
+import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.compare.VectorVisitor;
+import org.apache.arrow.vector.complex.impl.UnionLargeListViewReader;
+import org.apache.arrow.vector.complex.impl.UnionLargeListViewWriter;
+import org.apache.arrow.vector.complex.impl.UnionListReader;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.util.CallBack;
+import org.apache.arrow.vector.util.JsonStringArrayList;
+import org.apache.arrow.vector.util.OversizedAllocationException;
+import org.apache.arrow.vector.util.TransferPair;
+
+/**
+ * A large list view vector contains lists of a specific type of elements. Its
structure contains 3
+ * elements.
+ *
+ * <ol>
+ * <li>A validity buffer.
+ * <li>An offset buffer, that denotes lists starting positions.
+ * <li>A size buffer, that denotes sizes of the lists.
+ * <li>A child data vector that contains the elements of lists.
+ * </ol>
+ *
+ * This is the LargeListView variant of listview, it has a 64-bit wide offset
+ *
+ * <p>WARNING: Currently Arrow in Java doesn't support 64-bit vectors. This
class follows the
+ * expected behaviour of a LargeList but doesn't actually support allocating a
64-bit vector. It has
+ * little use until 64-bit vectors are supported and should be used with
caution. todo review
+ * checkedCastToInt usage in this class. Once int64 indexed vectors are
supported these checks
+ * aren't needed.
+ */
+public class LargeListViewVector extends BaseLargeRepeatedValueViewVector
+ implements PromotableVector, ValueIterableVector<List<?>> {
+
+ protected ArrowBuf validityBuffer;
+ protected UnionLargeListViewReader reader;
+ private CallBack callBack;
+ protected Field field;
+ protected int validityAllocationSizeInBytes;
+
+ public static LargeListViewVector empty(String name, BufferAllocator
allocator) {
+ return new LargeListViewVector(
+ name, allocator, FieldType.nullable(ArrowType.LargeListView.INSTANCE),
null);
+ }
+
+ /**
+ * Constructs a new instance.
+ *
+ * @param name The name of the instance.
+ * @param allocator The allocator to use for allocating/reallocating buffers.
+ * @param fieldType The type of this list.
+ * @param callBack A schema change callback.
+ */
+ public LargeListViewVector(
+ String name, BufferAllocator allocator, FieldType fieldType, CallBack
callBack) {
+ this(new Field(name, fieldType, null), allocator, callBack);
+ }
+
+ /**
+ * Constructs a new instance.
+ *
+ * @param field The field materialized by this vector.
+ * @param allocator The allocator to use for allocating/reallocating buffers.
+ * @param callBack A schema change callback.
+ */
+ public LargeListViewVector(Field field, BufferAllocator allocator, CallBack
callBack) {
+ super(field.getName(), allocator, callBack);
+ this.validityBuffer = allocator.getEmpty();
+ this.field = field;
+ this.callBack = callBack;
+ this.validityAllocationSizeInBytes =
getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION);
+ }
+
+ @Override
+ public void initializeChildrenFromFields(List<Field> children) {
+ checkArgument(
+ children.size() == 1,
+ "ListViews have one child Field. Found: %s",
+ children.isEmpty() ? "none" : children);
+
+ Field field = children.get(0);
+ AddOrGetResult<FieldVector> addOrGetVector =
addOrGetVector(field.getFieldType());
+ checkArgument(
+ addOrGetVector.isCreated(), "Child vector already existed: %s",
addOrGetVector.getVector());
+
+
addOrGetVector.getVector().initializeChildrenFromFields(field.getChildren());
+ this.field = new Field(this.field.getName(), this.field.getFieldType(),
children);
+ }
+
+ @Override
+ public void setInitialCapacity(int numRecords) {
+ validityAllocationSizeInBytes = getValidityBufferSizeFromCount(numRecords);
+ super.setInitialCapacity(numRecords);
+ }
+
+ /**
+ * Specialized version of setInitialCapacity() for LargeListViewVector. This
is used by some
+ * callers when they want to explicitly control and be conservative about
memory allocated for
+ * inner data vector. This is very useful when we are working with memory
constraints for a query
+ * and have a fixed amount of memory reserved for the record batch. In such
cases, we are likely
+ * to face OOM or related problems when we reserve memory for a record batch
with value count x
+ * and do setInitialCapacity(x) such that each vector allocates only what is
necessary and not the
+ * default amount, but the multiplier forces the memory requirement to go
beyond what was needed.
+ *
+ * @param numRecords value count
+ * @param density density of LargeListViewVector. Density is the average
size of a list per
+ * position in the LargeListViewVector. For example, a density value of
10 implies each
+ * position in the list vector has a list of 10 values. A density value
of 0.1 implies out of
+ * 10 positions in the list vector, 1 position has a list of size 1, and
the remaining
+ * positions are null (no lists) or empty lists. This helps in tightly
controlling the memory
+ * we provision for inner data vector.
+ */
+ @Override
+ public void setInitialCapacity(int numRecords, double density) {
+ validityAllocationSizeInBytes = getValidityBufferSizeFromCount(numRecords);
+ super.setInitialCapacity(numRecords, density);
+ }
+
+ /**
+ * Specialized version of setInitialTotalCapacity() for LargeListViewVector.
This is used by some
+ * callers when they want to explicitly control and be conservative about
memory allocated for
+ * inner data vector. This is very useful when we are working with memory
constraints for a query
+ * and have a fixed amount of memory reserved for the record batch. In such
cases, we are likely
+ * to face OOM or related problems when we reserve memory for a record batch
with value count x
+ * and do setInitialCapacity(x) such that each vector allocates only what is
necessary and not the
+ * default amount, but the multiplier forces the memory requirement to go
beyond what was needed.
+ *
+ * @param numRecords value count
+ * @param totalNumberOfElements the total number of elements to allow for in
this vector across
+ * all records.
+ */
+ @Override
+ public void setInitialTotalCapacity(int numRecords, int
totalNumberOfElements) {
+ validityAllocationSizeInBytes = getValidityBufferSizeFromCount(numRecords);
+ super.setInitialTotalCapacity(numRecords, totalNumberOfElements);
+ }
+
+ @Override
+ public List<FieldVector> getChildrenFromFields() {
+ return singletonList(getDataVector());
+ }
+
+ /**
+ * Load the buffers associated with this Field.
+ *
+ * @param fieldNode the fieldNode
+ * @param ownBuffers the buffers for this Field (own buffers only, children
not included)
+ */
+ @Override
+ public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf>
ownBuffers) {
+ if (ownBuffers.size() != 3) {
+ throw new IllegalArgumentException(
+ "Illegal buffer count, expected " + 3 + ", got: " +
ownBuffers.size());
+ }
+
+ ArrowBuf bitBuffer = ownBuffers.get(0);
+ ArrowBuf offBuffer = ownBuffers.get(1);
+ ArrowBuf szBuffer = ownBuffers.get(2);
+
+ validityBuffer.getReferenceManager().release();
+ validityBuffer = BitVectorHelper.loadValidityBuffer(fieldNode, bitBuffer,
allocator);
+ offsetBuffer.getReferenceManager().release();
+ offsetBuffer = offBuffer.getReferenceManager().retain(offBuffer,
allocator);
+ sizeBuffer.getReferenceManager().release();
+ sizeBuffer = szBuffer.getReferenceManager().retain(szBuffer, allocator);
+
+ validityAllocationSizeInBytes =
checkedCastToInt(validityBuffer.capacity());
+ offsetAllocationSizeInBytes = offsetBuffer.capacity();
+ sizeAllocationSizeInBytes = sizeBuffer.capacity();
+
+ valueCount = fieldNode.getLength();
+ }
+
+ /** Set the reader and writer indexes for the inner buffers. */
+ private void setReaderAndWriterIndex() {
+ validityBuffer.readerIndex(0);
+ offsetBuffer.readerIndex(0);
+ sizeBuffer.readerIndex(0);
+ if (valueCount == 0) {
+ validityBuffer.writerIndex(0);
+ offsetBuffer.writerIndex(0);
+ sizeBuffer.writerIndex(0);
+ } else {
+ validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount));
+ offsetBuffer.writerIndex((long) valueCount * OFFSET_WIDTH);
+ sizeBuffer.writerIndex((long) valueCount * SIZE_WIDTH);
+ }
+ }
+
+ @Override
+ public List<ArrowBuf> getFieldBuffers() {
+ List<ArrowBuf> result = new ArrayList<>(2);
+ setReaderAndWriterIndex();
+ result.add(validityBuffer);
+ result.add(offsetBuffer);
+ result.add(sizeBuffer);
+
+ return result;
+ }
+
+ /**
+ * Export the buffers of the fields for C Data Interface. This method
traverses the buffers and
+ * export buffer and buffer's memory address into a list of buffers and a
pointer to the list of
+ * buffers.
+ */
+ @Override
+ public void exportCDataBuffers(List<ArrowBuf> buffers, ArrowBuf buffersPtr,
long nullValue) {
+ throw new UnsupportedOperationException("exportCDataBuffers Not
implemented yet");
+ }
+
+ @Override
+ public void allocateNew() throws OutOfMemoryException {
+ if (!allocateNewSafe()) {
+ throw new OutOfMemoryException("Failure while allocating memory");
+ }
+ }
+
+ @Override
+ public boolean allocateNewSafe() {
+ boolean success = false;
+ try {
+ /* release the current buffers, hence this is a new allocation
+ * Note that, the `clear` method call below is releasing validityBuffer
+ * calling the superclass clear method which is releasing the associated
buffers
+ * (sizeBuffer and offsetBuffer).
+ */
+ clear();
+ /* allocate validity buffer */
+ allocateValidityBuffer(validityAllocationSizeInBytes);
+ /* allocate offset, data and sizes buffer */
+ success = super.allocateNewSafe();
+ } finally {
+ if (!success) {
+ clear();
+ }
+ }
+ return success;
+ }
+
+ protected void allocateValidityBuffer(final long size) {
+ final int curSize = (int) size;
+ validityBuffer = allocator.buffer(curSize);
+ validityBuffer.readerIndex(0);
+ validityAllocationSizeInBytes = curSize;
+ validityBuffer.setZero(0, validityBuffer.capacity());
+ }
+
+ @Override
+ public void reAlloc() {
+ /* reallocate the validity buffer */
+ reallocValidityBuffer();
+ /* reallocate the offset, size, and data */
+ super.reAlloc();
+ }
+
+ protected void reallocValidityAndSizeAndOffsetBuffers() {
+ reallocateBuffers();
+ reallocValidityBuffer();
+ }
+
+ private void reallocValidityBuffer() {
+ final int currentBufferCapacity =
checkedCastToInt(validityBuffer.capacity());
+ long newAllocationSize = getNewAllocationSize(currentBufferCapacity);
+
+ final ArrowBuf newBuf = allocator.buffer(newAllocationSize);
+ newBuf.setBytes(0, validityBuffer, 0, currentBufferCapacity);
+ newBuf.setZero(currentBufferCapacity, newBuf.capacity() -
currentBufferCapacity);
+ validityBuffer.getReferenceManager().release(1);
+ validityBuffer = newBuf;
+ validityAllocationSizeInBytes = (int) newAllocationSize;
+ }
+
+ private long getNewAllocationSize(int currentBufferCapacity) {
+ long newAllocationSize = currentBufferCapacity * 2L;
+ if (newAllocationSize == 0) {
+ if (validityAllocationSizeInBytes > 0) {
+ newAllocationSize = validityAllocationSizeInBytes;
+ } else {
+ newAllocationSize =
getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION) * 2L;
+ }
+ }
+ newAllocationSize = CommonUtil.nextPowerOfTwo(newAllocationSize);
+ assert newAllocationSize >= 1;
+
+ if (newAllocationSize > MAX_ALLOCATION_SIZE) {
+ throw new OversizedAllocationException("Unable to expand the buffer");
+ }
+ return newAllocationSize;
+ }
+
+ @Override
+ public void copyFromSafe(int inIndex, int outIndex, ValueVector from) {
+ throw new UnsupportedOperationException(
+ "LargeListViewVector does not support copyFromSafe operation yet.");
+ }
+
+ @Override
+ public void copyFrom(int inIndex, int outIndex, ValueVector from) {
+ throw new UnsupportedOperationException(
+ "LargeListViewVector does not support copyFrom operation yet.");
+ }
+
+ @Override
+ public FieldVector getDataVector() {
+ return vector;
+ }
+
+ @Override
+ public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+ return getTransferPair(ref, allocator, null);
+ }
+
+ @Override
+ public TransferPair getTransferPair(Field field, BufferAllocator allocator) {
+ return getTransferPair(field, allocator, null);
+ }
+
+ @Override
+ public TransferPair getTransferPair(String ref, BufferAllocator allocator,
CallBack callBack) {
+ throw new UnsupportedOperationException(
+ "LargeListViewVector does not support getTransferPair(String,
BufferAllocator, CallBack) yet");
+ }
+
+ @Override
+ public TransferPair getTransferPair(Field field, BufferAllocator allocator,
CallBack callBack) {
+ throw new UnsupportedOperationException(
+ "LargeListViewVector does not support getTransferPair(Field,
BufferAllocator, CallBack) yet");
+ }
+
+ @Override
+ public TransferPair makeTransferPair(ValueVector target) {
+ throw new UnsupportedOperationException(
+ "LargeListViewVector does not support makeTransferPair(ValueVector)
yet");
+ }
+
+ @Override
+ public long getValidityBufferAddress() {
+ return validityBuffer.memoryAddress();
+ }
+
+ @Override
+ public long getDataBufferAddress() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long getOffsetBufferAddress() {
+ return offsetBuffer.memoryAddress();
+ }
+
+ @Override
+ public ArrowBuf getValidityBuffer() {
+ return validityBuffer;
+ }
+
+ @Override
+ public ArrowBuf getDataBuffer() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public ArrowBuf getOffsetBuffer() {
+ return offsetBuffer;
+ }
+
+ public ArrowBuf getSizeBuffer() {
+ return sizeBuffer;
+ }
+
+ public long getSizeBufferAddress() {
+ return sizeBuffer.memoryAddress();
+ }
+
+ /**
+ * Get the hash code for the element at the given index.
+ *
+ * @param index position of the element
+ * @return hash code for the element at the given index
+ */
+ @Override
+ public int hashCode(int index) {
+ return hashCode(index, null);
+ }
+
+ /**
+ * Get the hash code for the element at the given index.
+ *
+ * @param index position of the element
+ * @param hasher hasher to use
+ * @return hash code for the element at the given index
+ */
+ @Override
+ public int hashCode(int index, ArrowBufHasher hasher) {
+ if (isSet(index) == 0) {
+ return ArrowBufPointer.NULL_HASH_CODE;
+ }
+ int hash = 0;
+ final int start = offsetBuffer.getInt((long) index * OFFSET_WIDTH);
+ final int end = sizeBuffer.getInt((long) index * OFFSET_WIDTH);
+ for (int i = start; i < end; i++) {
+ hash = ByteFunctionHelpers.combineHash(hash,
vector.hashCode(checkedCastToInt(i), hasher));
+ }
+ return hash;
+ }
+
+ @Override
+ public <OUT, IN> OUT accept(VectorVisitor<OUT, IN> visitor, IN value) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ protected FieldReader getReaderImpl() {
+ throw new UnsupportedOperationException(
+ "LargeListViewVector does not support getReaderImpl operation yet.");
+ }
+
+ @Override
+ public UnionListReader getReader() {
+ throw new UnsupportedOperationException(
+ "LargeListViewVector does not support getReader operation yet.");
+ }
+
+ /**
+ * Get the size (number of bytes) of underlying buffers used by this vector.
+ *
+ * @return size of underlying buffers.
+ */
+ @Override
+ public int getBufferSize() {
+ if (valueCount == 0) {
+ return 0;
+ }
+ final int offsetBufferSize = valueCount * OFFSET_WIDTH;
+ final int sizeBufferSize = valueCount * SIZE_WIDTH;
+ final int validityBufferSize = getValidityBufferSizeFromCount(valueCount);
+ return offsetBufferSize + sizeBufferSize + validityBufferSize +
vector.getBufferSize();
+ }
+
+ /**
+ * Get the size (number of bytes) of underlying buffers used by this.
+ *
+ * @param valueCount the number of values to assume this vector contains
+ * @return size of underlying buffers.
+ */
+ @Override
+ public int getBufferSizeFor(int valueCount) {
+ if (valueCount == 0) {
+ return 0;
+ }
+ final int validityBufferSize = getValidityBufferSizeFromCount(valueCount);
+
+ return super.getBufferSizeFor(valueCount) + validityBufferSize;
+ }
+
+ /**
+ * Get the field associated with the list view vector.
+ *
+ * @return the field
+ */
+ @Override
+ public Field getField() {
+ if (field.getChildren().contains(getDataVector().getField())) {
+ return field;
+ }
+ field =
+ new Field(
+ field.getName(),
+ field.getFieldType(),
+ Collections.singletonList(getDataVector().getField()));
+ return field;
+ }
+
+ /**
+ * Get the minor type for the vector.
+ *
+ * @return the minor type
+ */
+ @Override
+ public MinorType getMinorType() {
+ return MinorType.LARGELISTVIEW;
+ }
+
+ /** Clear the vector data. */
+ @Override
+ public void clear() {
+ // calling superclass clear method which is releasing the sizeBufer and
offsetBuffer
+ super.clear();
+ validityBuffer = releaseBuffer(validityBuffer);
+ }
+
+ /** Release the buffers associated with this vector. */
+ @Override
+ public void reset() {
+ super.reset();
+ validityBuffer.setZero(0, validityBuffer.capacity());
+ }
+
+ /**
+ * Return the underlying buffers associated with this vector. Note that this
doesn't impact the
+ * reference counts for this buffer, so it only should be used for
in-context access. Also note
+ * that this buffer changes regularly, thus external classes shouldn't hold
a reference to it
+ * (unless they change it).
+ *
+ * @param clear Whether to clear vector before returning, the buffers will
still be refcounted but
+ * the returned array will be the only reference to them
+ * @return The underlying {@link ArrowBuf buffers} that is used by this
vector instance.
+ */
+ @Override
+ public ArrowBuf[] getBuffers(boolean clear) {
+ setReaderAndWriterIndex();
+ final ArrowBuf[] buffers;
+ if (getBufferSize() == 0) {
+ buffers = new ArrowBuf[0];
+ } else {
+ List<ArrowBuf> list = new ArrayList<>();
+ // the order must be validity, offset and size buffers
+ list.add(validityBuffer);
+ list.add(offsetBuffer);
+ list.add(sizeBuffer);
+ list.addAll(Arrays.asList(vector.getBuffers(clear)));
+ buffers = list.toArray(new ArrowBuf[list.size()]);
+ }
+ if (clear) {
+ for (ArrowBuf buffer : buffers) {
+ buffer.getReferenceManager().retain();
+ }
+ clear();
+ }
+ return buffers;
+ }
+
+ /**
+ * Get the element in the list view vector at a particular index.
+ *
+ * @param index position of the element
+ * @return Object at given position
+ */
+ @Override
+ public List<?> getObject(int index) {
+ if (isSet(index) == 0) {
+ return null;
+ }
+ final List<Object> vals = new JsonStringArrayList<>();
+ final int start = offsetBuffer.getInt(index * OFFSET_WIDTH);
+ final int end = start + sizeBuffer.getInt((index) * SIZE_WIDTH);
+ final ValueVector vv = getDataVector();
+ for (int i = start; i < end; i++) {
+ vals.add(vv.getObject(checkedCastToInt(i)));
+ }
+
+ return vals;
+ }
+
+ /**
+ * Check if an element at given index is null.
+ *
+ * @param index position of an element
+ * @return true if an element at given index is null, false otherwise
+ */
+ @Override
+ public boolean isNull(int index) {
+ return (isSet(index) == 0);
+ }
+
+ /**
+ * Check if an element at given index is an empty list.
+ *
+ * @param index position of an element
+ * @return true if an element at given index is an empty list or NULL, false
otherwise
+ */
+ @Override
+ public boolean isEmpty(int index) {
+ if (isNull(index)) {
+ return true;
+ } else {
+ return sizeBuffer.getInt(index * SIZE_WIDTH) == 0;
+ }
+ }
+
+ /**
+ * Same as {@link #isNull(int)}.
+ *
+ * @param index position of the element
+ * @return 1 if element at given index is not null, 0 otherwise
+ */
+ public int isSet(int index) {
+ final int byteIndex = index >> 3;
+ final byte b = validityBuffer.getByte(byteIndex);
+ final int bitIndex = index & 7;
+ return (b >> bitIndex) & 0x01;
+ }
+
+ /**
+ * Get the number of elements that are null in the vector.
+ *
+ * @return the number of null elements.
+ */
+ @Override
+ public int getNullCount() {
+ return BitVectorHelper.getNullCount(validityBuffer, valueCount);
+ }
+
+ /**
+ * Get the value capacity by considering validity and offset capacity. Note
that the size buffer
+ * capacity is not considered here since it has the same capacity as the
offset buffer.
+ *
+ * @return the value capacity
+ */
+ @Override
+ public int getValueCapacity() {
+ return getValidityAndOffsetValueCapacity();
+ }
+
+ private int getValidityAndSizeValueCapacity() {
+ final int offsetValueCapacity = Math.max(getOffsetBufferValueCapacity(),
0);
+ final int sizeValueCapacity = Math.max(getSizeBufferValueCapacity(), 0);
+ return Math.min(offsetValueCapacity, sizeValueCapacity);
+ }
+
+ private int getValidityAndOffsetValueCapacity() {
+ final int offsetValueCapacity = Math.max(getOffsetBufferValueCapacity(),
0);
+ return Math.min(offsetValueCapacity, getValidityBufferValueCapacity());
+ }
+
+ private int getValidityBufferValueCapacity() {
+ return capAtMaxInt(validityBuffer.capacity() * 8);
+ }
+
+ /**
+ * Set the element at the given index to null.
+ *
+ * @param index the value to change
+ */
+ @Override
+ public void setNull(int index) {
+ while (index >= getValidityAndSizeValueCapacity()) {
+ reallocValidityAndSizeAndOffsetBuffers();
+ }
+
+ offsetBuffer.setInt(index * OFFSET_WIDTH, 0);
+ sizeBuffer.setInt(index * SIZE_WIDTH, 0);
+ BitVectorHelper.unsetBit(validityBuffer, index);
+ }
+
+ /**
+ * Start new value in the ListView vector.
+ *
+ * @param index index of the value to start
+ * @return offset of the new value
+ */
+ @Override
+ public int startNewValue(int index) {
+ while (index >= getValidityAndSizeValueCapacity()) {
+ reallocValidityAndSizeAndOffsetBuffers();
+ }
+
+ if (index > 0) {
+ final int prevOffset = getLengthOfChildVectorByIndex(index);
+ offsetBuffer.setInt(index * OFFSET_WIDTH, prevOffset);
+ }
+
+ BitVectorHelper.setBit(validityBuffer, index);
+ return offsetBuffer.getInt(index * OFFSET_WIDTH);
+ }
+
+ /**
+ * Validate the invariants of the offset and size buffers. 0 <= offsets[i]
<= length of the child
+ * array 0 <= offsets[i] + size[i] <= length of the child array
+ *
+ * @param offset the offset at a given index
+ * @param size the size at a given index
+ */
+ private void validateInvariants(int offset, int size) {
+ if (offset < 0) {
+ throw new IllegalArgumentException("Offset cannot be negative");
+ }
+
+ if (size < 0) {
+ throw new IllegalArgumentException("Size cannot be negative");
+ }
+
+ // 0 <= offsets[i] <= length of the child array
+ if (offset > this.vector.getValueCount()) {
+ throw new IllegalArgumentException("Offset is out of bounds.");
+ }
+
+ // 0 <= offsets[i] + size[i] <= length of the child array
+ if (offset + size > this.vector.getValueCount()) {
+ throw new IllegalArgumentException("Offset + size <= length of the child
array.");
+ }
+ }
+
+ /**
+ * Set the offset at the given index. Make sure to use this function after
updating `field` vector
+ * and using `setValidity`
+ *
+ * @param index index of the value to set
+ * @param value value to set
+ */
+ public void setOffset(int index, int value) {
+ validateInvariants(value, sizeBuffer.getInt(index * SIZE_WIDTH));
+
+ offsetBuffer.setInt(index * OFFSET_WIDTH, value);
+ }
+
+ /**
+ * Set the size at the given index. Make sure to use this function after
using `setOffset`.
+ *
+ * @param index index of the value to set
+ * @param value value to set
+ */
+ public void setSize(int index, int value) {
+ validateInvariants(offsetBuffer.getInt(index * SIZE_WIDTH), value);
+
+ sizeBuffer.setInt(index * SIZE_WIDTH, value);
+ }
+
+ /**
+ * Set the validity at the given index.
+ *
+ * @param index index of the value to set
+ * @param value value to set (0 for unset and 1 for a set)
+ */
+ public void setValidity(int index, int value) {
+ if (value == 0) {
+ BitVectorHelper.unsetBit(validityBuffer, index);
+ } else {
+ BitVectorHelper.setBit(validityBuffer, index);
+ }
+ }
+
+ /**
+ * Sets the value count for the vector.
+ *
+ * <p>Important note: The underlying vector does not support 64-bit
allocations yet. This may
+ * throw if attempting to hold larger than what a 32-bit vector can store.
+ *
+ * @param valueCount value count
+ */
+ @Override
+ public void setValueCount(int valueCount) {
+ this.valueCount = valueCount;
+ if (valueCount > 0) {
+ while (valueCount > getValidityAndSizeValueCapacity()) {
+ /* check if validity and offset buffers need to be re-allocated */
+ reallocValidityAndSizeAndOffsetBuffers();
+ }
+ }
+ /* valueCount for the data vector is the current end offset */
+ final long childValueCount = (valueCount == 0) ? 0 :
getLengthOfChildVector();
+ /* set the value count of data vector and this will take care of
+ * checking whether data buffer needs to be reallocated.
+ * TODO: revisit when 64-bit vectors are supported
+ */
+ Preconditions.checkArgument(
+ childValueCount <= Integer.MAX_VALUE && childValueCount >= 0,
+ "LargeListViewVector doesn't yet support 64-bit allocations: %s",
+ childValueCount);
+ vector.setValueCount((int) childValueCount);
+ }
+
+ @Override
+ public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(FieldType
fieldType) {
+ AddOrGetResult<T> result = super.addOrGetVector(fieldType);
+ invalidateReader();
+ return result;
+ }
+
+ @Override
+ public UnionVector promoteToUnion() {
+ UnionVector vector = new UnionVector("$data$", allocator, /* field type*/
null, callBack);
+ replaceDataVector(vector);
+ invalidateReader();
+ if (callBack != null) {
+ callBack.doWork();
+ }
+ return vector;
+ }
+
+ private void invalidateReader() {
+ reader = null;
+ }
+
+ @Deprecated
+ @Override
+ public List<BufferBacked> getFieldInnerVectors() {
+ throw new UnsupportedOperationException("There are no inner vectors. Use
getFieldBuffers");
+ }
+
+ public UnionLargeListViewWriter getWriter() {
+ return new UnionLargeListViewWriter(this);
+ }
+
+ @Override
+ public int getValueCount() {
+ return valueCount;
+ }
+
+ /**
+ * Get the density of this LargeListViewVector.
+ *
+ * @return density
+ */
+ public double getDensity() {
+ if (valueCount == 0) {
+ return 0.0D;
+ }
+ final double totalListSize = getLengthOfChildVector();
+ return totalListSize / valueCount;
+ }
+
+ /** Validating LargeListViewVector creation based on the specification
guideline. */
+ @Override
+ public void validate() {
+ for (int i = 0; i < valueCount; i++) {
+ final int offset = offsetBuffer.getInt((long) i * OFFSET_WIDTH);
+ final int size = sizeBuffer.getInt((long) i * SIZE_WIDTH);
+ validateInvariants(offset, size);
+ }
+ }
+
+ /**
+ * End the current value.
+ *
+ * @param index index of the value to end
+ * @param size number of elements in the list that was written
+ */
+ public void endValue(int index, int size) {
+ sizeBuffer.setInt((long) index * SIZE_WIDTH, size);
+ }
+}
diff --git
a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionLargeListViewReader.java
b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionLargeListViewReader.java
new file mode 100644
index 0000000000..4bcd028de3
--- /dev/null
+++
b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionLargeListViewReader.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.complex.impl;
+
+import static org.apache.arrow.memory.util.LargeMemoryUtil.checkedCastToInt;
+
+import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.complex.BaseLargeRepeatedValueViewVector;
+import org.apache.arrow.vector.complex.LargeListViewVector;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.holders.UnionHolder;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.Field;
+
+/** {@link FieldReader} for largeListView of union types. */
+public class UnionLargeListViewReader extends AbstractFieldReader {
+
+ private final LargeListViewVector vector;
+ private final ValueVector data;
+ private int currentOffset;
+ private int size;
+
+ /**
+ * Constructor for UnionLargeListViewReader.
+ *
+ * @param vector the vector to read from
+ */
+ public UnionLargeListViewReader(LargeListViewVector vector) {
+ this.vector = vector;
+ this.data = vector.getDataVector();
+ }
+
+ @Override
+ public Field getField() {
+ return vector.getField();
+ }
+
+ @Override
+ public boolean isSet() {
+ return !vector.isNull(idx());
+ }
+
+ @Override
+ public void setPosition(int index) {
+ super.setPosition(index);
+ if (vector.getOffsetBuffer().capacity() == 0) {
+ currentOffset = 0;
+ size = 0;
+ } else {
+ currentOffset =
+ vector
+ .getOffsetBuffer()
+ .getInt(index * (long)
BaseLargeRepeatedValueViewVector.OFFSET_WIDTH);
+ size =
+ vector.getSizeBuffer().getInt(index * (long)
BaseLargeRepeatedValueViewVector.SIZE_WIDTH);
+ }
+ }
+
+ @Override
+ public FieldReader reader() {
+ return data.getReader();
+ }
+
+ @Override
+ public Object readObject() {
+ return vector.getObject(idx());
+ }
+
+ @Override
+ public MinorType getMinorType() {
+ return MinorType.LISTVIEW;
+ }
+
+ @Override
+ public void read(int index, UnionHolder holder) {
+ setPosition(idx());
+ for (int i = -1; i < index; i++) {
+ next();
+ }
+ holder.reader = data.getReader();
+ holder.isSet = data.getReader().isSet() ? 1 : 0;
+ }
+
+ @Override
+ public int size() {
+ return Math.max(size, 0);
+ }
+
+ @Override
+ public boolean next() {
+ // Here, the currentOffSet keeps track of the current position in the
vector inside the list at
+ // set position.
+ // And, size keeps track of the elements count in the list, so to make
sure we traverse
+ // the full list, we need to check if the currentOffset is less than the
currentOffset + size
+ if (currentOffset < currentOffset + size) {
+ data.getReader().setPosition(checkedCastToInt(currentOffset++));
+ return true;
+ } else {
+ return false;
+ }
+ }
+}
diff --git
a/java/vector/src/main/java/org/apache/arrow/vector/extension/OpaqueType.java
b/java/vector/src/main/java/org/apache/arrow/vector/extension/OpaqueType.java
index a0e898a543..f4f06dad2a 100644
---
a/java/vector/src/main/java/org/apache/arrow/vector/extension/OpaqueType.java
+++
b/java/vector/src/main/java/org/apache/arrow/vector/extension/OpaqueType.java
@@ -389,5 +389,10 @@ public class OpaqueType extends ArrowType.ExtensionType {
public FieldVector visit(ListView type) {
throw unsupported(type);
}
+
+ @Override
+ public FieldVector visit(LargeListView type) {
+ throw unsupported(type);
+ }
}
}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
index ed099890e1..6b2c56de01 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
@@ -69,6 +69,7 @@ import org.apache.arrow.vector.ViewVarCharVector;
import org.apache.arrow.vector.complex.DenseUnionVector;
import org.apache.arrow.vector.complex.FixedSizeListVector;
import org.apache.arrow.vector.complex.LargeListVector;
+import org.apache.arrow.vector.complex.LargeListViewVector;
import org.apache.arrow.vector.complex.ListVector;
import org.apache.arrow.vector.complex.ListViewVector;
import org.apache.arrow.vector.complex.MapVector;
@@ -111,6 +112,7 @@ import org.apache.arrow.vector.complex.impl.UInt1WriterImpl;
import org.apache.arrow.vector.complex.impl.UInt2WriterImpl;
import org.apache.arrow.vector.complex.impl.UInt4WriterImpl;
import org.apache.arrow.vector.complex.impl.UInt8WriterImpl;
+import org.apache.arrow.vector.complex.impl.UnionLargeListViewWriter;
import org.apache.arrow.vector.complex.impl.UnionLargeListWriter;
import org.apache.arrow.vector.complex.impl.UnionListWriter;
import org.apache.arrow.vector.complex.impl.UnionWriter;
@@ -134,6 +136,7 @@ import
org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint;
import org.apache.arrow.vector.types.pojo.ArrowType.Int;
import org.apache.arrow.vector.types.pojo.ArrowType.Interval;
import org.apache.arrow.vector.types.pojo.ArrowType.LargeBinary;
+import org.apache.arrow.vector.types.pojo.ArrowType.LargeListView;
import org.apache.arrow.vector.types.pojo.ArrowType.LargeUtf8;
import org.apache.arrow.vector.types.pojo.ArrowType.List;
import org.apache.arrow.vector.types.pojo.ArrowType.ListView;
@@ -645,6 +648,19 @@ public class Types {
return new UnionLargeListWriter((LargeListVector) vector);
}
},
+ LARGELISTVIEW(ArrowType.LargeListView.INSTANCE) {
+ @Override
+ public FieldVector getNewVector(
+ Field field, BufferAllocator allocator, CallBack
schemaChangeCallback) {
+ return new LargeListViewVector(
+ field.getName(), allocator, field.getFieldType(),
schemaChangeCallback);
+ }
+
+ @Override
+ public FieldWriter getNewFieldWriter(ValueVector vector) {
+ return new UnionLargeListViewWriter((LargeListViewVector) vector);
+ }
+ },
FIXED_SIZE_LIST(null) {
@Override
public FieldVector getNewVector(
@@ -996,6 +1012,11 @@ public class Types {
return MinorType.LISTVIEW;
}
+ @Override
+ public MinorType visit(LargeListView type) {
+ return MinorType.LARGELISTVIEW;
+ }
+
@Override
public MinorType visit(ExtensionType type) {
return MinorType.EXTENSIONTYPE;
diff --git
a/java/vector/src/test/java/org/apache/arrow/vector/TestLargeListViewVector.java
b/java/vector/src/test/java/org/apache/arrow/vector/TestLargeListViewVector.java
new file mode 100644
index 0000000000..563ac811c4
--- /dev/null
+++
b/java/vector/src/test/java/org/apache/arrow/vector/TestLargeListViewVector.java
@@ -0,0 +1,1649 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import org.apache.arrow.memory.ArrowBuf;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.complex.BaseLargeRepeatedValueViewVector;
+import org.apache.arrow.vector.complex.LargeListViewVector;
+import org.apache.arrow.vector.complex.impl.UnionLargeListViewWriter;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+public class TestLargeListViewVector {
+
+ private BufferAllocator allocator;
+
+ @BeforeEach
+ public void init() {
+ allocator = new DirtyRootAllocator(Long.MAX_VALUE, (byte) 100);
+ }
+
+ @AfterEach
+ public void terminate() throws Exception {
+ allocator.close();
+ }
+
+ @Test
+ public void testBasicLargeListViewVector() {
+ try (LargeListViewVector largeListViewVector =
+ LargeListViewVector.empty("sourceVector", allocator)) {
+ UnionLargeListViewWriter largeListViewWriter =
largeListViewVector.getWriter();
+
+ /* allocate memory */
+ largeListViewWriter.allocate();
+
+ /* write the first list at index 0 */
+ largeListViewWriter.setPosition(0);
+ largeListViewWriter.startListView();
+
+ largeListViewWriter.bigInt().writeBigInt(12);
+ largeListViewWriter.bigInt().writeBigInt(-7);
+ largeListViewWriter.bigInt().writeBigInt(25);
+ largeListViewWriter.endListView();
+
+ /* the second list at index 1 is null (we are not setting any)*/
+
+ /* write the third list at index 2 */
+ largeListViewWriter.setPosition(2);
+ largeListViewWriter.startListView();
+
+ largeListViewWriter.bigInt().writeBigInt(0);
+ largeListViewWriter.bigInt().writeBigInt(-127);
+ largeListViewWriter.bigInt().writeBigInt(127);
+ largeListViewWriter.bigInt().writeBigInt(50);
+ largeListViewWriter.endListView();
+
+ /* write the fourth list at index 3 (empty list) */
+ largeListViewWriter.setPosition(3);
+ largeListViewWriter.startListView();
+ largeListViewWriter.endListView();
+
+ /* write the fifth list at index 4 */
+ largeListViewWriter.setPosition(4);
+ largeListViewWriter.startListView();
+ largeListViewWriter.bigInt().writeBigInt(1);
+ largeListViewWriter.bigInt().writeBigInt(2);
+ largeListViewWriter.bigInt().writeBigInt(3);
+ largeListViewWriter.bigInt().writeBigInt(4);
+ largeListViewWriter.endListView();
+
+ largeListViewWriter.setValueCount(5);
+ // check value count
+ assertEquals(5, largeListViewVector.getValueCount());
+
+ /* get vector at index 0 -- the value is a BigIntVector*/
+ final ArrowBuf offSetBuffer = largeListViewVector.getOffsetBuffer();
+ final ArrowBuf sizeBuffer = largeListViewVector.getSizeBuffer();
+ final FieldVector dataVec = largeListViewVector.getDataVector();
+
+ // check offset buffer
+ assertEquals(0, offSetBuffer.getInt(0 *
BaseLargeRepeatedValueViewVector.OFFSET_WIDTH));
+ assertEquals(0, offSetBuffer.getInt(1 *
BaseLargeRepeatedValueViewVector.OFFSET_WIDTH));
+ assertEquals(3, offSetBuffer.getInt(2 *
BaseLargeRepeatedValueViewVector.OFFSET_WIDTH));
+ assertEquals(7, offSetBuffer.getInt(3 *
BaseLargeRepeatedValueViewVector.OFFSET_WIDTH));
+ assertEquals(7, offSetBuffer.getInt(4 *
BaseLargeRepeatedValueViewVector.OFFSET_WIDTH));
+
+ // check size buffer
+ assertEquals(3, sizeBuffer.getInt(0 *
BaseLargeRepeatedValueViewVector.SIZE_WIDTH));
+ assertEquals(0, sizeBuffer.getInt(1 *
BaseLargeRepeatedValueViewVector.SIZE_WIDTH));
+ assertEquals(4, sizeBuffer.getInt(2 *
BaseLargeRepeatedValueViewVector.SIZE_WIDTH));
+ assertEquals(0, sizeBuffer.getInt(3 *
BaseLargeRepeatedValueViewVector.SIZE_WIDTH));
+ assertEquals(4, sizeBuffer.getInt(4 *
BaseLargeRepeatedValueViewVector.SIZE_WIDTH));
+
+ // check data vector
+ assertEquals(12, ((BigIntVector) dataVec).get(0));
+ assertEquals(-7, ((BigIntVector) dataVec).get(1));
+ assertEquals(25, ((BigIntVector) dataVec).get(2));
+ assertEquals(0, ((BigIntVector) dataVec).get(3));
+ assertEquals(-127, ((BigIntVector) dataVec).get(4));
+ assertEquals(127, ((BigIntVector) dataVec).get(5));
+ assertEquals(50, ((BigIntVector) dataVec).get(6));
+ assertEquals(1, ((BigIntVector) dataVec).get(7));
+ assertEquals(2, ((BigIntVector) dataVec).get(8));
+ assertEquals(3, ((BigIntVector) dataVec).get(9));
+ assertEquals(4, ((BigIntVector) dataVec).get(10));
+
+ largeListViewVector.validate();
+ }
+ }
+
+ @Test
+ public void testImplicitNullVectors() {
+ try (LargeListViewVector largeListViewVector =
+ LargeListViewVector.empty("sourceVector", allocator)) {
+ UnionLargeListViewWriter largeListViewWriter =
largeListViewVector.getWriter();
+ /* allocate memory */
+ largeListViewWriter.allocate();
+
+ final ArrowBuf offSetBuffer = largeListViewVector.getOffsetBuffer();
+ final ArrowBuf sizeBuffer = largeListViewVector.getSizeBuffer();
+
+ /* write the first list at index 0 */
+ largeListViewWriter.setPosition(0);
+ largeListViewWriter.startListView();
+
+ largeListViewWriter.bigInt().writeBigInt(12);
+ largeListViewWriter.bigInt().writeBigInt(-7);
+ largeListViewWriter.bigInt().writeBigInt(25);
+ largeListViewWriter.endListView();
+
+ int offSet0 = offSetBuffer.getInt(0 *
BaseLargeRepeatedValueViewVector.OFFSET_WIDTH);
+ int size0 = sizeBuffer.getInt(0 *
BaseLargeRepeatedValueViewVector.SIZE_WIDTH);
+
+ // after the first list is written,
+ // the initial offset must be 0,
+ // the size must be 3 (as there are 3 elements in the array),
+ // the lastSet must be 0 since, the first list is written at index 0.
+
+ assertEquals(0, offSet0);
+ assertEquals(3, size0);
+
+ largeListViewWriter.setPosition(5);
+ largeListViewWriter.startListView();
+
+ // writing the 6th list at index 5,
+ // and the list items from index 1 through 4 are not populated.
+ // but since there is a gap between the 0th and 5th list, in terms
+ // of buffer allocation, the offset and size buffers must be updated
+ // to reflect the implicit null vectors.
+
+ for (int i = 1; i < 5; i++) {
+ int offSet = offSetBuffer.getInt(i *
BaseLargeRepeatedValueViewVector.OFFSET_WIDTH);
+ int size = sizeBuffer.getInt(i *
BaseLargeRepeatedValueViewVector.SIZE_WIDTH);
+ // Since the list is not written, the offset and size must equal to
child vector's size
+ // i.e., 3, and size should be 0 as the list is not written.
+ // And the last set value is the value currently being written, which
is 5.
+ assertEquals(0, offSet);
+ assertEquals(0, size);
+ }
+
+ largeListViewWriter.bigInt().writeBigInt(12);
+ largeListViewWriter.bigInt().writeBigInt(25);
+ largeListViewWriter.endListView();
+
+ int offSet5 = offSetBuffer.getInt(5 *
BaseLargeRepeatedValueViewVector.OFFSET_WIDTH);
+ int size5 = sizeBuffer.getInt(5 *
BaseLargeRepeatedValueViewVector.SIZE_WIDTH);
+
+ assertEquals(3, offSet5);
+ assertEquals(2, size5);
+
+ largeListViewWriter.setPosition(10);
+ largeListViewWriter.startListView();
+
+ // writing the 11th list at index 10,
+ // and the list items from index 6 through 10 are not populated.
+ // but since there is a gap between the 5th and 11th list, in terms
+ // of buffer allocation, the offset and size buffers must be updated
+ // to reflect the implicit null vectors.
+ for (int i = 6; i < 10; i++) {
+ int offSet = offSetBuffer.getInt(i *
BaseLargeRepeatedValueViewVector.OFFSET_WIDTH);
+ int size = sizeBuffer.getInt(i *
BaseLargeRepeatedValueViewVector.SIZE_WIDTH);
+ // Since the list is not written, the offset and size must equal to 0
+ // and size should be 0 as the list is not written.
+ // And the last set value is the value currently being written, which
is 10.
+ assertEquals(0, offSet);
+ assertEquals(0, size);
+ }
+
+ largeListViewWriter.bigInt().writeBigInt(12);
+ largeListViewWriter.endListView();
+
+ int offSet11 = offSetBuffer.getInt(10 *
BaseLargeRepeatedValueViewVector.OFFSET_WIDTH);
+ int size11 = sizeBuffer.getInt(10 *
BaseLargeRepeatedValueViewVector.SIZE_WIDTH);
+
+ assertEquals(5, offSet11);
+ assertEquals(1, size11);
+
+ largeListViewVector.setValueCount(11);
+
+ largeListViewVector.validate();
+ }
+ }
+
+ @Test
+ public void testNestedLargeListViewVector() throws Exception {
+ try (LargeListViewVector largeListViewVector =
+ LargeListViewVector.empty("sourceVector", allocator)) {
+
+ UnionLargeListViewWriter largeListViewWriter =
largeListViewVector.getWriter();
+
+ /* allocate memory */
+ largeListViewWriter.allocate();
+
+ /* the dataVector that backs a largeListViewVector will also be a
+ * largeListViewVector for this test.
+ */
+
+ /* write one or more inner lists at index 0 */
+ largeListViewWriter.setPosition(0);
+ largeListViewWriter.startListView();
+
+ largeListViewWriter.listView().startListView();
+ largeListViewWriter.listView().bigInt().writeBigInt(50);
+ largeListViewWriter.listView().bigInt().writeBigInt(100);
+ largeListViewWriter.listView().bigInt().writeBigInt(200);
+ largeListViewWriter.listView().endListView();
+
+ largeListViewWriter.listView().startListView();
+ largeListViewWriter.listView().bigInt().writeBigInt(75);
+ largeListViewWriter.listView().bigInt().writeBigInt(125);
+ largeListViewWriter.listView().bigInt().writeBigInt(150);
+ largeListViewWriter.listView().bigInt().writeBigInt(175);
+ largeListViewWriter.listView().endListView();
+
+ largeListViewWriter.endListView();
+
+ /* write one or more inner lists at index 1 */
+ largeListViewWriter.setPosition(1);
+ largeListViewWriter.startListView();
+
+ largeListViewWriter.listView().startListView();
+ largeListViewWriter.listView().bigInt().writeBigInt(10);
+ largeListViewWriter.listView().endListView();
+
+ largeListViewWriter.listView().startListView();
+ largeListViewWriter.listView().bigInt().writeBigInt(15);
+ largeListViewWriter.listView().bigInt().writeBigInt(20);
+ largeListViewWriter.listView().endListView();
+
+ largeListViewWriter.listView().startListView();
+ largeListViewWriter.listView().bigInt().writeBigInt(25);
+ largeListViewWriter.listView().bigInt().writeBigInt(30);
+ largeListViewWriter.listView().bigInt().writeBigInt(35);
+ largeListViewWriter.listView().endListView();
+
+ largeListViewWriter.endListView();
+
+ largeListViewVector.setValueCount(2);
+
+ assertEquals(2, largeListViewVector.getValueCount());
+
+ /* get largeListViewVector value at index 0 -- the value itself is a
largeListViewVector */
+ Object result = largeListViewVector.getObject(0);
+ ArrayList<ArrayList<Long>> resultSet = (ArrayList<ArrayList<Long>>)
result;
+ ArrayList<Long> list;
+
+ assertEquals(2, resultSet.size()); /* 2 inner lists at index 0 */
+ assertEquals(3, resultSet.get(0).size()); /* size of first inner list */
+ assertEquals(4, resultSet.get(1).size()); /* size of second inner list */
+
+ list = resultSet.get(0);
+ assertEquals(Long.valueOf(50), list.get(0));
+ assertEquals(Long.valueOf(100), list.get(1));
+ assertEquals(Long.valueOf(200), list.get(2));
+
+ list = resultSet.get(1);
+ assertEquals(Long.valueOf(75), list.get(0));
+ assertEquals(Long.valueOf(125), list.get(1));
+ assertEquals(Long.valueOf(150), list.get(2));
+ assertEquals(Long.valueOf(175), list.get(3));
+
+ /* get largeListViewVector value at index 1 -- the value itself is a
largeListViewVector */
+ result = largeListViewVector.getObject(1);
+ resultSet = (ArrayList<ArrayList<Long>>) result;
+
+ assertEquals(3, resultSet.size()); /* 3 inner lists at index 1 */
+ assertEquals(1, resultSet.get(0).size()); /* size of first inner list */
+ assertEquals(2, resultSet.get(1).size()); /* size of second inner list */
+ assertEquals(3, resultSet.get(2).size()); /* size of third inner list */
+
+ list = resultSet.get(0);
+ assertEquals(Long.valueOf(10), list.get(0));
+
+ list = resultSet.get(1);
+ assertEquals(Long.valueOf(15), list.get(0));
+ assertEquals(Long.valueOf(20), list.get(1));
+
+ list = resultSet.get(2);
+ assertEquals(Long.valueOf(25), list.get(0));
+ assertEquals(Long.valueOf(30), list.get(1));
+ assertEquals(Long.valueOf(35), list.get(2));
+
+ /* check underlying bitVector */
+ assertFalse(largeListViewVector.isNull(0));
+ assertFalse(largeListViewVector.isNull(1));
+
+ /* check underlying offsets */
+ final ArrowBuf offsetBuffer = largeListViewVector.getOffsetBuffer();
+
+ /* largeListViewVector has 2 lists at index 0 and 3 lists at index 1 */
+ assertEquals(0, offsetBuffer.getLong(0 *
LargeListViewVector.OFFSET_WIDTH));
+ assertEquals(2, offsetBuffer.getLong(1 *
LargeListViewVector.OFFSET_WIDTH));
+ }
+ }
+
+ @Test
+ public void testNestedLargeListViewVector1() {
+ try (LargeListViewVector largeListViewVector =
+ LargeListViewVector.empty("sourceVector", allocator)) {
+
+ MinorType listViewType = MinorType.LARGELISTVIEW;
+ MinorType scalarType = MinorType.BIGINT;
+
+
largeListViewVector.addOrGetVector(FieldType.nullable(listViewType.getType()));
+
+ LargeListViewVector innerList1 = (LargeListViewVector)
largeListViewVector.getDataVector();
+ innerList1.addOrGetVector(FieldType.nullable(listViewType.getType()));
+
+ LargeListViewVector innerList2 = (LargeListViewVector)
innerList1.getDataVector();
+ innerList2.addOrGetVector(FieldType.nullable(listViewType.getType()));
+
+ LargeListViewVector innerList3 = (LargeListViewVector)
innerList2.getDataVector();
+ innerList3.addOrGetVector(FieldType.nullable(listViewType.getType()));
+
+ LargeListViewVector innerList4 = (LargeListViewVector)
innerList3.getDataVector();
+ innerList4.addOrGetVector(FieldType.nullable(listViewType.getType()));
+
+ LargeListViewVector innerList5 = (LargeListViewVector)
innerList4.getDataVector();
+ innerList5.addOrGetVector(FieldType.nullable(listViewType.getType()));
+
+ LargeListViewVector innerList6 = (LargeListViewVector)
innerList5.getDataVector();
+ innerList6.addOrGetVector(FieldType.nullable(scalarType.getType()));
+
+ largeListViewVector.setInitialCapacity(128);
+ }
+ }
+
+ @Test
+ public void testNestedLargeListViewVector2() throws Exception {
+ try (LargeListViewVector largeListViewVector =
+ LargeListViewVector.empty("sourceVector", allocator)) {
+ largeListViewVector.setInitialCapacity(1);
+ UnionLargeListViewWriter largeListViewWriter =
largeListViewVector.getWriter();
+ /* allocate memory */
+ largeListViewWriter.allocate();
+
+ /* write one or more inner lists at index 0 */
+ largeListViewWriter.setPosition(0);
+ largeListViewWriter.startListView();
+
+ largeListViewWriter.listView().startListView();
+ largeListViewWriter.listView().bigInt().writeBigInt(50);
+ largeListViewWriter.listView().bigInt().writeBigInt(100);
+ largeListViewWriter.listView().bigInt().writeBigInt(200);
+ largeListViewWriter.listView().endListView();
+
+ largeListViewWriter.listView().startListView();
+ largeListViewWriter.listView().bigInt().writeBigInt(75);
+ largeListViewWriter.listView().bigInt().writeBigInt(125);
+ largeListViewWriter.listView().endListView();
+
+ largeListViewWriter.endListView();
+
+ /* write one or more inner lists at index 1 */
+ largeListViewWriter.setPosition(1);
+ largeListViewWriter.startListView();
+
+ largeListViewWriter.listView().startListView();
+ largeListViewWriter.listView().bigInt().writeBigInt(15);
+ largeListViewWriter.listView().bigInt().writeBigInt(20);
+ largeListViewWriter.listView().endListView();
+
+ largeListViewWriter.listView().startListView();
+ largeListViewWriter.listView().bigInt().writeBigInt(25);
+ largeListViewWriter.listView().bigInt().writeBigInt(30);
+ largeListViewWriter.listView().bigInt().writeBigInt(35);
+ largeListViewWriter.listView().endListView();
+
+ largeListViewWriter.endListView();
+
+ largeListViewVector.setValueCount(2);
+
+ assertEquals(2, largeListViewVector.getValueCount());
+
+ /* get largeListViewVector value at index 0 -- the value itself is a
largeListViewVector */
+ Object result = largeListViewVector.getObject(0);
+ ArrayList<ArrayList<Long>> resultSet = (ArrayList<ArrayList<Long>>)
result;
+ ArrayList<Long> list;
+
+ assertEquals(2, resultSet.size()); /* 2 inner lists at index 0 */
+ assertEquals(3, resultSet.get(0).size()); /* size of first inner list */
+ assertEquals(2, resultSet.get(1).size()); /* size of second inner list */
+
+ list = resultSet.get(0);
+ assertEquals(Long.valueOf(50), list.get(0));
+ assertEquals(Long.valueOf(100), list.get(1));
+ assertEquals(Long.valueOf(200), list.get(2));
+
+ list = resultSet.get(1);
+ assertEquals(Long.valueOf(75), list.get(0));
+ assertEquals(Long.valueOf(125), list.get(1));
+
+ /* get largeListViewVector value at index 1 -- the value itself is a
largeListViewVector */
+ result = largeListViewVector.getObject(1);
+ resultSet = (ArrayList<ArrayList<Long>>) result;
+
+ assertEquals(2, resultSet.size()); /* 3 inner lists at index 1 */
+ assertEquals(2, resultSet.get(0).size()); /* size of first inner list */
+ assertEquals(3, resultSet.get(1).size()); /* size of second inner list */
+
+ list = resultSet.get(0);
+ assertEquals(Long.valueOf(15), list.get(0));
+ assertEquals(Long.valueOf(20), list.get(1));
+
+ list = resultSet.get(1);
+ assertEquals(Long.valueOf(25), list.get(0));
+ assertEquals(Long.valueOf(30), list.get(1));
+ assertEquals(Long.valueOf(35), list.get(2));
+
+ /* check underlying bitVector */
+ assertFalse(largeListViewVector.isNull(0));
+ assertFalse(largeListViewVector.isNull(1));
+
+ /* check underlying offsets */
+ final ArrowBuf offsetBuffer = largeListViewVector.getOffsetBuffer();
+
+ /* largeListViewVector has 2 lists at index 0 and 3 lists at index 1 */
+ assertEquals(0, offsetBuffer.getLong(0 *
LargeListViewVector.OFFSET_WIDTH));
+ assertEquals(2, offsetBuffer.getLong(1 *
LargeListViewVector.OFFSET_WIDTH));
+ }
+ }
+
+ @Test
+ public void testGetBufferAddress() throws Exception {
+ try (LargeListViewVector largeListViewVector =
LargeListViewVector.empty("vector", allocator)) {
+
+ UnionLargeListViewWriter largeListViewWriter =
largeListViewVector.getWriter();
+ boolean error = false;
+
+ largeListViewWriter.allocate();
+
+ largeListViewWriter.setPosition(0);
+ largeListViewWriter.startListView();
+ largeListViewWriter.bigInt().writeBigInt(50);
+ largeListViewWriter.bigInt().writeBigInt(100);
+ largeListViewWriter.bigInt().writeBigInt(200);
+ largeListViewWriter.endListView();
+
+ largeListViewWriter.setPosition(1);
+ largeListViewWriter.startListView();
+ largeListViewWriter.bigInt().writeBigInt(250);
+ largeListViewWriter.bigInt().writeBigInt(300);
+ largeListViewWriter.endListView();
+
+ largeListViewVector.setValueCount(2);
+
+ /* check largeListViewVector contents */
+ Object result = largeListViewVector.getObject(0);
+ ArrayList<Long> resultSet = (ArrayList<Long>) result;
+ assertEquals(3, resultSet.size());
+ assertEquals(Long.valueOf(50), resultSet.get(0));
+ assertEquals(Long.valueOf(100), resultSet.get(1));
+ assertEquals(Long.valueOf(200), resultSet.get(2));
+
+ result = largeListViewVector.getObject(1);
+ resultSet = (ArrayList<Long>) result;
+ assertEquals(2, resultSet.size());
+ assertEquals(Long.valueOf(250), resultSet.get(0));
+ assertEquals(Long.valueOf(300), resultSet.get(1));
+
+ List<ArrowBuf> buffers = largeListViewVector.getFieldBuffers();
+
+ long bitAddress = largeListViewVector.getValidityBufferAddress();
+ long offsetAddress = largeListViewVector.getOffsetBufferAddress();
+ long sizeAddress = largeListViewVector.getSizeBufferAddress();
+
+ try {
+ largeListViewVector.getDataBufferAddress();
+ } catch (UnsupportedOperationException ue) {
+ error = true;
+ } finally {
+ assertTrue(error);
+ }
+
+ assertEquals(3, buffers.size());
+ assertEquals(bitAddress, buffers.get(0).memoryAddress());
+ assertEquals(offsetAddress, buffers.get(1).memoryAddress());
+ assertEquals(sizeAddress, buffers.get(2).memoryAddress());
+
+ /* (3+2)/2 */
+ assertEquals(2.5, largeListViewVector.getDensity(), 0);
+ }
+ }
+
+ /*
+ * Setting up the buffers directly needs to be validated with the base
method used in
+ * the LargeListViewVector class where we use the approach of
startListView(),
+ * write to the child vector and endListView().
+ * <p>
+ * To support this, we have to consider the following scenarios;
+ * <p>
+ * 1. Only using directly buffer-based inserts.
+ * 2. Default list insertion followed by buffer-based inserts.
+ * 3. Buffer-based inserts followed by default list insertion.
+ */
+
+ /* Setting up buffers directly would require the following steps to be taken
+ * 0. Allocate buffers in largeListViewVector by calling `allocateNew`
method.
+ * 1. Initialize the child vector using `initializeChildrenFromFields`
method.
+ * 2. Set values in the child vector.
+ * 3. Set validity, offset and size buffers using `setValidity`,
+ * `setOffset` and `setSize` methods.
+ * 4. Set value count using `setValueCount` method.
+ */
+ @Test
+ public void testBasicLargeListViewSet() {
+
+ try (LargeListViewVector largeListViewVector =
+ LargeListViewVector.empty("sourceVector", allocator)) {
+ // Allocate buffers in largeListViewVector by calling `allocateNew`
method.
+ largeListViewVector.allocateNew();
+
+ // Initialize the child vector using `initializeChildrenFromFields`
method.
+ FieldType fieldType = new FieldType(true, new ArrowType.Int(64, true),
null, null);
+ Field field = new Field("child-vector", fieldType, null);
+
largeListViewVector.initializeChildrenFromFields(Collections.singletonList(field));
+
+ // Set values in the child vector.
+ FieldVector fieldVector = largeListViewVector.getDataVector();
+ fieldVector.clear();
+
+ BigIntVector childVector = (BigIntVector) fieldVector;
+ childVector.allocateNew(7);
+
+ childVector.set(0, 12);
+ childVector.set(1, -7);
+ childVector.set(2, 25);
+ childVector.set(3, 0);
+ childVector.set(4, -127);
+ childVector.set(5, 127);
+ childVector.set(6, 50);
+
+ childVector.setValueCount(7);
+
+ // Set validity, offset and size buffers using `setValidity`,
+ // `setOffset` and `setSize` methods.
+ largeListViewVector.setOffset(0, 0);
+ largeListViewVector.setOffset(1, 3);
+ largeListViewVector.setOffset(2, 3);
+ largeListViewVector.setOffset(3, 7);
+
+ largeListViewVector.setSize(0, 3);
+ largeListViewVector.setSize(1, 0);
+ largeListViewVector.setSize(2, 4);
+ largeListViewVector.setSize(3, 0);
+
+ largeListViewVector.setValidity(0, 1);
+ largeListViewVector.setValidity(1, 0);
+ largeListViewVector.setValidity(2, 1);
+ largeListViewVector.setValidity(3, 1);
+
+ // Set value count using `setValueCount` method.
+ largeListViewVector.setValueCount(4);
+
+ final ArrowBuf offSetBuffer = largeListViewVector.getOffsetBuffer();
+ final ArrowBuf sizeBuffer = largeListViewVector.getSizeBuffer();
+
+ // check offset buffer
+ assertEquals(0, offSetBuffer.getInt(0 *
BaseLargeRepeatedValueViewVector.OFFSET_WIDTH));
+ assertEquals(3, offSetBuffer.getInt(1 *
BaseLargeRepeatedValueViewVector.OFFSET_WIDTH));
+ assertEquals(3, offSetBuffer.getInt(2 *
BaseLargeRepeatedValueViewVector.OFFSET_WIDTH));
+ assertEquals(7, offSetBuffer.getInt(3 *
BaseLargeRepeatedValueViewVector.OFFSET_WIDTH));
+
+ // check size buffer
+ assertEquals(3, sizeBuffer.getInt(0 *
BaseLargeRepeatedValueViewVector.SIZE_WIDTH));
+ assertEquals(0, sizeBuffer.getInt(1 *
BaseLargeRepeatedValueViewVector.SIZE_WIDTH));
+ assertEquals(4, sizeBuffer.getInt(2 *
BaseLargeRepeatedValueViewVector.SIZE_WIDTH));
+ assertEquals(0, sizeBuffer.getInt(3 *
BaseLargeRepeatedValueViewVector.SIZE_WIDTH));
+
+ // check values
+ assertEquals(12, ((BigIntVector)
largeListViewVector.getDataVector()).get(0));
+ assertEquals(-7, ((BigIntVector)
largeListViewVector.getDataVector()).get(1));
+ assertEquals(25, ((BigIntVector)
largeListViewVector.getDataVector()).get(2));
+ assertEquals(0, ((BigIntVector)
largeListViewVector.getDataVector()).get(3));
+ assertEquals(-127, ((BigIntVector)
largeListViewVector.getDataVector()).get(4));
+ assertEquals(127, ((BigIntVector)
largeListViewVector.getDataVector()).get(5));
+ assertEquals(50, ((BigIntVector)
largeListViewVector.getDataVector()).get(6));
+
+ largeListViewVector.validate();
+ }
+ }
+
+ @Test
+ public void testBasicLargeListViewSetNested() {
+ // Expected largeListViewVector
+ // [[[50,100,200],[75,125,150,175]],[[10],[15,20],[25,30,35]]]
+
+ // Setting child vector
+ // [[50,100,200],[75,125,150,175],[10],[15,20],[25,30,35]]
+ try (LargeListViewVector largeListViewVector =
+ LargeListViewVector.empty("sourceVector", allocator)) {
+ // Allocate buffers in largeListViewVector by calling `allocateNew`
method.
+ largeListViewVector.allocateNew();
+
+ // Initialize the child vector using `initializeChildrenFromFields`
method.
+ FieldType fieldType = new FieldType(true, new ArrowType.LargeListView(),
null, null);
+ FieldType childFieldType = new FieldType(true, new ArrowType.Int(64,
true), null, null);
+ Field childField = new Field("child-vector", childFieldType, null);
+ List<Field> children = new ArrayList<>();
+ children.add(childField);
+ Field field = new Field("child-vector", fieldType, children);
+
largeListViewVector.initializeChildrenFromFields(Collections.singletonList(field));
+
+ // Set values in the child vector.
+ FieldVector fieldVector = largeListViewVector.getDataVector();
+ fieldVector.clear();
+
+ LargeListViewVector childVector = (LargeListViewVector) fieldVector;
+ UnionLargeListViewWriter largeListViewWriter = childVector.getWriter();
+ largeListViewWriter.allocate();
+
+ largeListViewWriter.setPosition(0);
+ largeListViewWriter.startListView();
+
+ largeListViewWriter.bigInt().writeBigInt(50);
+ largeListViewWriter.bigInt().writeBigInt(100);
+ largeListViewWriter.bigInt().writeBigInt(200);
+
+ largeListViewWriter.endListView();
+
+ largeListViewWriter.setPosition(1);
+ largeListViewWriter.startListView();
+
+ largeListViewWriter.bigInt().writeBigInt(75);
+ largeListViewWriter.bigInt().writeBigInt(125);
+ largeListViewWriter.bigInt().writeBigInt(150);
+ largeListViewWriter.bigInt().writeBigInt(175);
+
+ largeListViewWriter.endListView();
+
+ largeListViewWriter.setPosition(2);
+ largeListViewWriter.startListView();
+
+ largeListViewWriter.bigInt().writeBigInt(10);
+
+ largeListViewWriter.endListView();
+
+ largeListViewWriter.startListView();
+ largeListViewWriter.setPosition(3);
+
+ largeListViewWriter.bigInt().writeBigInt(15);
+ largeListViewWriter.bigInt().writeBigInt(20);
+
+ largeListViewWriter.endListView();
+
+ largeListViewWriter.startListView();
+ largeListViewWriter.setPosition(4);
+
+ largeListViewWriter.bigInt().writeBigInt(25);
+ largeListViewWriter.bigInt().writeBigInt(30);
+ largeListViewWriter.bigInt().writeBigInt(35);
+
+ largeListViewWriter.endListView();
+
+ childVector.setValueCount(5);
+
+ // Set validity, offset and size buffers using `setValidity`,
+ // `setOffset` and `setSize` methods.
+
+ largeListViewVector.setValidity(0, 1);
+ largeListViewVector.setValidity(1, 1);
+
+ largeListViewVector.setOffset(0, 0);
+ largeListViewVector.setOffset(1, 2);
+
+ largeListViewVector.setSize(0, 2);
+ largeListViewVector.setSize(1, 3);
+
+ // Set value count using `setValueCount` method.
+ largeListViewVector.setValueCount(2);
+
+ assertEquals(2, largeListViewVector.getValueCount());
+
+ /* get largeListViewVector value at index 0 -- the value itself is a
largeListViewVector */
+ Object result = largeListViewVector.getObject(0);
+ ArrayList<ArrayList<Long>> resultSet = (ArrayList<ArrayList<Long>>)
result;
+ ArrayList<Long> list;
+
+ assertEquals(2, resultSet.size()); /* 2 inner lists at index 0 */
+ assertEquals(3, resultSet.get(0).size()); /* size of the first inner
list */
+ assertEquals(4, resultSet.get(1).size()); /* size of the second inner
list */
+
+ list = resultSet.get(0);
+ assertEquals(Long.valueOf(50), list.get(0));
+ assertEquals(Long.valueOf(100), list.get(1));
+ assertEquals(Long.valueOf(200), list.get(2));
+
+ list = resultSet.get(1);
+ assertEquals(Long.valueOf(75), list.get(0));
+ assertEquals(Long.valueOf(125), list.get(1));
+ assertEquals(Long.valueOf(150), list.get(2));
+ assertEquals(Long.valueOf(175), list.get(3));
+
+ /* get largeListViewVector value at index 1 -- the value itself is a
largeListViewVector */
+ result = largeListViewVector.getObject(1);
+ resultSet = (ArrayList<ArrayList<Long>>) result;
+
+ assertEquals(3, resultSet.size()); /* 3 inner lists at index 1 */
+ assertEquals(1, resultSet.get(0).size()); /* size of the first inner
list */
+ assertEquals(2, resultSet.get(1).size()); /* size of the second inner
list */
+ assertEquals(3, resultSet.get(2).size()); /* size of the third inner
list */
+
+ list = resultSet.get(0);
+ assertEquals(Long.valueOf(10), list.get(0));
+
+ list = resultSet.get(1);
+ assertEquals(Long.valueOf(15), list.get(0));
+ assertEquals(Long.valueOf(20), list.get(1));
+
+ list = resultSet.get(2);
+ assertEquals(Long.valueOf(25), list.get(0));
+ assertEquals(Long.valueOf(30), list.get(1));
+ assertEquals(Long.valueOf(35), list.get(2));
+
+ /* check underlying bitVector */
+ assertFalse(largeListViewVector.isNull(0));
+ assertFalse(largeListViewVector.isNull(1));
+
+ final ArrowBuf offSetBuffer = largeListViewVector.getOffsetBuffer();
+ final ArrowBuf sizeBuffer = largeListViewVector.getSizeBuffer();
+
+ // check offset buffer
+ assertEquals(0, offSetBuffer.getInt(0 *
BaseLargeRepeatedValueViewVector.OFFSET_WIDTH));
+ assertEquals(2, offSetBuffer.getInt(1 *
BaseLargeRepeatedValueViewVector.OFFSET_WIDTH));
+
+ // check size buffer
+ assertEquals(2, sizeBuffer.getInt(0 *
BaseLargeRepeatedValueViewVector.SIZE_WIDTH));
+ assertEquals(3, sizeBuffer.getInt(1 *
BaseLargeRepeatedValueViewVector.SIZE_WIDTH));
+
+ largeListViewVector.validate();
+ }
+ }
+
+ @Test
+ public void testBasicLargeListViewSetWithListViewWriter() {
+ try (LargeListViewVector largeListViewVector =
+ LargeListViewVector.empty("sourceVector", allocator)) {
+ // Allocate buffers in largeListViewVector by calling `allocateNew`
method.
+ largeListViewVector.allocateNew();
+
+ // Initialize the child vector using `initializeChildrenFromFields`
method.
+ FieldType fieldType = new FieldType(true, new ArrowType.Int(64, true),
null, null);
+ Field field = new Field("child-vector", fieldType, null);
+
largeListViewVector.initializeChildrenFromFields(Collections.singletonList(field));
+
+ // Set values in the child vector.
+ FieldVector fieldVector = largeListViewVector.getDataVector();
+ fieldVector.clear();
+
+ BigIntVector childVector = (BigIntVector) fieldVector;
+ childVector.allocateNew(7);
+
+ childVector.set(0, 12);
+ childVector.set(1, -7);
+ childVector.set(2, 25);
+ childVector.set(3, 0);
+ childVector.set(4, -127);
+ childVector.set(5, 127);
+ childVector.set(6, 50);
+
+ childVector.setValueCount(7);
+
+ // Set validity, offset and size buffers using `setValidity`,
+ // `setOffset` and `setSize` methods.
+
+ largeListViewVector.setValidity(0, 1);
+ largeListViewVector.setValidity(1, 0);
+ largeListViewVector.setValidity(2, 1);
+ largeListViewVector.setValidity(3, 1);
+
+ largeListViewVector.setOffset(0, 0);
+ largeListViewVector.setOffset(1, 3);
+ largeListViewVector.setOffset(2, 3);
+ largeListViewVector.setOffset(3, 7);
+
+ largeListViewVector.setSize(0, 3);
+ largeListViewVector.setSize(1, 0);
+ largeListViewVector.setSize(2, 4);
+ largeListViewVector.setSize(3, 0);
+
+ // Set value count using `setValueCount` method.
+ largeListViewVector.setValueCount(4);
+
+ final ArrowBuf offSetBuffer = largeListViewVector.getOffsetBuffer();
+ final ArrowBuf sizeBuffer = largeListViewVector.getSizeBuffer();
+
+ // check offset buffer
+ assertEquals(0, offSetBuffer.getInt(0 *
BaseLargeRepeatedValueViewVector.OFFSET_WIDTH));
+ assertEquals(3, offSetBuffer.getInt(1 *
BaseLargeRepeatedValueViewVector.OFFSET_WIDTH));
+ assertEquals(3, offSetBuffer.getInt(2 *
BaseLargeRepeatedValueViewVector.OFFSET_WIDTH));
+ assertEquals(7, offSetBuffer.getInt(3 *
BaseLargeRepeatedValueViewVector.OFFSET_WIDTH));
+
+ // check size buffer
+ assertEquals(3, sizeBuffer.getInt(0 *
BaseLargeRepeatedValueViewVector.SIZE_WIDTH));
+ assertEquals(0, sizeBuffer.getInt(1 *
BaseLargeRepeatedValueViewVector.SIZE_WIDTH));
+ assertEquals(4, sizeBuffer.getInt(2 *
BaseLargeRepeatedValueViewVector.SIZE_WIDTH));
+ assertEquals(0, sizeBuffer.getInt(3 *
BaseLargeRepeatedValueViewVector.SIZE_WIDTH));
+
+ // check values
+ assertEquals(12, ((BigIntVector)
largeListViewVector.getDataVector()).get(0));
+ assertEquals(-7, ((BigIntVector)
largeListViewVector.getDataVector()).get(1));
+ assertEquals(25, ((BigIntVector)
largeListViewVector.getDataVector()).get(2));
+ assertEquals(0, ((BigIntVector)
largeListViewVector.getDataVector()).get(3));
+ assertEquals(-127, ((BigIntVector)
largeListViewVector.getDataVector()).get(4));
+ assertEquals(127, ((BigIntVector)
largeListViewVector.getDataVector()).get(5));
+ assertEquals(50, ((BigIntVector)
largeListViewVector.getDataVector()).get(6));
+
+ UnionLargeListViewWriter listViewWriter =
largeListViewVector.getWriter();
+
+ listViewWriter.setPosition(4);
+ listViewWriter.startListView();
+
+ listViewWriter.bigInt().writeBigInt(121);
+ listViewWriter.bigInt().writeBigInt(-71);
+ listViewWriter.bigInt().writeBigInt(251);
+ listViewWriter.endListView();
+
+ largeListViewVector.setValueCount(5);
+
+ // check offset buffer
+ assertEquals(0, offSetBuffer.getInt(0 *
BaseLargeRepeatedValueViewVector.OFFSET_WIDTH));
+ assertEquals(3, offSetBuffer.getInt(1 *
BaseLargeRepeatedValueViewVector.OFFSET_WIDTH));
+ assertEquals(3, offSetBuffer.getInt(2 *
BaseLargeRepeatedValueViewVector.OFFSET_WIDTH));
+ assertEquals(7, offSetBuffer.getInt(3 *
BaseLargeRepeatedValueViewVector.OFFSET_WIDTH));
+ assertEquals(7, offSetBuffer.getInt(4 *
BaseLargeRepeatedValueViewVector.OFFSET_WIDTH));
+
+ // check size buffer
+ assertEquals(3, sizeBuffer.getInt(0 *
BaseLargeRepeatedValueViewVector.SIZE_WIDTH));
+ assertEquals(0, sizeBuffer.getInt(1 *
BaseLargeRepeatedValueViewVector.SIZE_WIDTH));
+ assertEquals(4, sizeBuffer.getInt(2 *
BaseLargeRepeatedValueViewVector.SIZE_WIDTH));
+ assertEquals(0, sizeBuffer.getInt(3 *
BaseLargeRepeatedValueViewVector.SIZE_WIDTH));
+ assertEquals(3, sizeBuffer.getInt(4 *
BaseLargeRepeatedValueViewVector.SIZE_WIDTH));
+
+ // check values
+ assertEquals(12, ((BigIntVector)
largeListViewVector.getDataVector()).get(0));
+ assertEquals(-7, ((BigIntVector)
largeListViewVector.getDataVector()).get(1));
+ assertEquals(25, ((BigIntVector)
largeListViewVector.getDataVector()).get(2));
+ assertEquals(0, ((BigIntVector)
largeListViewVector.getDataVector()).get(3));
+ assertEquals(-127, ((BigIntVector)
largeListViewVector.getDataVector()).get(4));
+ assertEquals(127, ((BigIntVector)
largeListViewVector.getDataVector()).get(5));
+ assertEquals(50, ((BigIntVector)
largeListViewVector.getDataVector()).get(6));
+ assertEquals(121, ((BigIntVector)
largeListViewVector.getDataVector()).get(7));
+ assertEquals(-71, ((BigIntVector)
largeListViewVector.getDataVector()).get(8));
+ assertEquals(251, ((BigIntVector)
largeListViewVector.getDataVector()).get(9));
+
+ largeListViewVector.validate();
+ }
+ }
+
+ @Test
+ public void testConsistentChildName() throws Exception {
+ try (LargeListViewVector largeListViewVector =
+ LargeListViewVector.empty("sourceVector", allocator)) {
+ String emptyListStr = largeListViewVector.getField().toString();
+ assertTrue(emptyListStr.contains(LargeListViewVector.DATA_VECTOR_NAME));
+
+
largeListViewVector.addOrGetVector(FieldType.nullable(MinorType.INT.getType()));
+ String emptyVectorStr = largeListViewVector.getField().toString();
+
assertTrue(emptyVectorStr.contains(LargeListViewVector.DATA_VECTOR_NAME));
+ }
+ }
+
+ @Test
+ public void testSetInitialCapacity() {
+ try (final LargeListViewVector vector = LargeListViewVector.empty("",
allocator)) {
+ vector.addOrGetVector(FieldType.nullable(MinorType.INT.getType()));
+
+ vector.setInitialCapacity(512);
+ vector.allocateNew();
+ assertEquals(512, vector.getValueCapacity());
+ assertTrue(vector.getDataVector().getValueCapacity() >= 512);
+
+ vector.setInitialCapacity(512, 4);
+ vector.allocateNew();
+ assertEquals(512, vector.getValueCapacity());
+ assertTrue(vector.getDataVector().getValueCapacity() >= 512 * 4);
+
+ vector.setInitialCapacity(512, 0.1);
+ vector.allocateNew();
+ assertEquals(512, vector.getValueCapacity());
+ assertTrue(vector.getDataVector().getValueCapacity() >= 51);
+
+ vector.setInitialCapacity(512, 0.01);
+ vector.allocateNew();
+ assertEquals(512, vector.getValueCapacity());
+ assertTrue(vector.getDataVector().getValueCapacity() >= 5);
+
+ vector.setInitialCapacity(5, 0.1);
+ vector.allocateNew();
+ assertEquals(8, vector.getValueCapacity());
+ assertTrue(vector.getDataVector().getValueCapacity() >= 1);
+
+ vector.validate();
+ }
+ }
+
+ @Test
+ public void testClearAndReuse() {
+ try (final LargeListViewVector vector = LargeListViewVector.empty("list",
allocator)) {
+ BigIntVector bigIntVector =
+ (BigIntVector)
+
vector.addOrGetVector(FieldType.nullable(MinorType.BIGINT.getType())).getVector();
+ vector.setInitialCapacity(10);
+ vector.allocateNew();
+
+ vector.startNewValue(0);
+ bigIntVector.setSafe(0, 7);
+ vector.endValue(0, 1);
+ vector.startNewValue(1);
+ bigIntVector.setSafe(1, 8);
+ vector.endValue(1, 1);
+ vector.setValueCount(2);
+
+ Object result = vector.getObject(0);
+ ArrayList<Long> resultSet = (ArrayList<Long>) result;
+ assertEquals(Long.valueOf(7), resultSet.get(0));
+
+ result = vector.getObject(1);
+ resultSet = (ArrayList<Long>) result;
+ assertEquals(Long.valueOf(8), resultSet.get(0));
+
+ // Clear and release the buffers to trigger a realloc when adding next
value
+ vector.clear();
+
+ // The list vector should reuse a buffer when reallocating the offset
buffer
+ vector.startNewValue(0);
+ bigIntVector.setSafe(0, 7);
+ vector.endValue(0, 1);
+ vector.startNewValue(1);
+ bigIntVector.setSafe(1, 8);
+ vector.endValue(1, 1);
+ vector.setValueCount(2);
+
+ result = vector.getObject(0);
+ resultSet = (ArrayList<Long>) result;
+ assertEquals(Long.valueOf(7), resultSet.get(0));
+
+ result = vector.getObject(1);
+ resultSet = (ArrayList<Long>) result;
+ assertEquals(Long.valueOf(8), resultSet.get(0));
+ }
+ }
+
+ @Test
+ public void testWriterGetField() {
+ try (final LargeListViewVector vector = LargeListViewVector.empty("list",
allocator)) {
+
+ UnionLargeListViewWriter writer = vector.getWriter();
+ writer.allocate();
+
+ // set some values
+ writer.startListView();
+ writer.integer().writeInt(1);
+ writer.integer().writeInt(2);
+ writer.endListView();
+ vector.setValueCount(2);
+
+ Field expectedDataField =
+ new Field(
+ BaseLargeRepeatedValueViewVector.DATA_VECTOR_NAME,
+ FieldType.nullable(new ArrowType.Int(32, true)),
+ null);
+ Field expectedField =
+ new Field(
+ vector.getName(),
+ FieldType.nullable(ArrowType.LargeListView.INSTANCE),
+ Collections.singletonList(expectedDataField));
+
+ assertEquals(expectedField, writer.getField());
+ }
+ }
+
+ @Test
+ public void testClose() throws Exception {
+ try (final LargeListViewVector vector =
LargeListViewVector.empty("largelistview", allocator)) {
+
+ UnionLargeListViewWriter writer = vector.getWriter();
+ writer.allocate();
+
+ // set some values
+ writer.startListView();
+ writer.integer().writeInt(1);
+ writer.integer().writeInt(2);
+ writer.endListView();
+ vector.setValueCount(2);
+
+ assertTrue(vector.getBufferSize() > 0);
+ assertTrue(vector.getDataVector().getBufferSize() > 0);
+
+ writer.close();
+ assertEquals(0, vector.getBufferSize());
+ assertEquals(0, vector.getDataVector().getBufferSize());
+
+ vector.validate();
+ }
+ }
+
+ @Test
+ public void testGetBufferSizeFor() {
+ try (final LargeListViewVector vector =
LargeListViewVector.empty("largelistview", allocator)) {
+
+ UnionLargeListViewWriter writer = vector.getWriter();
+ writer.allocate();
+
+ // set some values
+ writeIntValues(writer, new int[] {1, 2});
+ writeIntValues(writer, new int[] {3, 4});
+ writeIntValues(writer, new int[] {5, 6});
+ writeIntValues(writer, new int[] {7, 8, 9, 10});
+ writeIntValues(writer, new int[] {11, 12, 13, 14});
+ writer.setValueCount(5);
+
+ IntVector dataVector = (IntVector) vector.getDataVector();
+ int[] indices = new int[] {0, 2, 4, 6, 10, 14};
+
+ for (int valueCount = 1; valueCount <= 5; valueCount++) {
+ int validityBufferSize =
BitVectorHelper.getValidityBufferSize(valueCount);
+ int offsetBufferSize = valueCount *
BaseLargeRepeatedValueViewVector.OFFSET_WIDTH;
+ int sizeBufferSize = valueCount *
BaseLargeRepeatedValueViewVector.SIZE_WIDTH;
+
+ int expectedSize =
+ validityBufferSize
+ + offsetBufferSize
+ + sizeBufferSize
+ + dataVector.getBufferSizeFor(indices[valueCount]);
+ assertEquals(expectedSize, vector.getBufferSizeFor(valueCount));
+ }
+ vector.validate();
+ }
+ }
+
+ @Test
+ public void testIsEmpty() {
+ try (final LargeListViewVector vector =
LargeListViewVector.empty("largelistview", allocator)) {
+ UnionLargeListViewWriter writer = vector.getWriter();
+ writer.allocate();
+
+ // set values [1,2], null, [], [5,6]
+ writeIntValues(writer, new int[] {1, 2});
+ writer.setPosition(2);
+ writeIntValues(writer, new int[] {});
+ writeIntValues(writer, new int[] {5, 6});
+ writer.setValueCount(4);
+
+ assertFalse(vector.isEmpty(0));
+ assertTrue(vector.isNull(1));
+ assertTrue(vector.isEmpty(1));
+ assertFalse(vector.isNull(2));
+ assertTrue(vector.isEmpty(2));
+ assertFalse(vector.isEmpty(3));
+
+ vector.validate();
+ }
+ }
+
+ @Test
+ public void testTotalCapacity() {
+ final FieldType type = FieldType.nullable(MinorType.INT.getType());
+ try (final LargeListViewVector vector =
+ new LargeListViewVector("largelistview", allocator, type, null)) {
+ // Force the child vector to be allocated based on the type
+ // (this is a bad API: we have to track and repeat the type twice)
+ vector.addOrGetVector(type);
+
+ // Specify the allocation size but do not actually allocate
+ vector.setInitialTotalCapacity(10, 100);
+
+ // Finally, actually do the allocation
+ vector.allocateNewSafe();
+
+ // Note: allocator rounds up and can be greater than the requested
allocation.
+ assertTrue(vector.getValueCapacity() >= 10);
+ assertTrue(vector.getDataVector().getValueCapacity() >= 100);
+ }
+ }
+
+ @Test
+ public void testSetNull1() {
+ try (LargeListViewVector vector =
LargeListViewVector.empty("largelistview", allocator)) {
+ UnionLargeListViewWriter writer = vector.getWriter();
+ writer.allocate();
+
+ writer.setPosition(0);
+ writer.startListView();
+ writer.bigInt().writeBigInt(10);
+ writer.bigInt().writeBigInt(20);
+ writer.endListView();
+
+ vector.setNull(1);
+
+ writer.setPosition(2);
+ writer.startListView();
+ writer.bigInt().writeBigInt(30);
+ writer.bigInt().writeBigInt(40);
+ writer.endListView();
+
+ vector.setNull(3);
+ vector.setNull(4);
+
+ writer.setPosition(5);
+ writer.startListView();
+ writer.bigInt().writeBigInt(50);
+ writer.bigInt().writeBigInt(60);
+ writer.endListView();
+
+ vector.setValueCount(6);
+
+ assertFalse(vector.isNull(0));
+ assertTrue(vector.isNull(1));
+ assertFalse(vector.isNull(2));
+ assertTrue(vector.isNull(3));
+ assertTrue(vector.isNull(4));
+ assertFalse(vector.isNull(5));
+
+ // validate buffers
+
+ final ArrowBuf validityBuffer = vector.getValidityBuffer();
+ final ArrowBuf offsetBuffer = vector.getOffsetBuffer();
+ final ArrowBuf sizeBuffer = vector.getSizeBuffer();
+
+ assertEquals(1, BitVectorHelper.get(validityBuffer, 0));
+ assertEquals(0, BitVectorHelper.get(validityBuffer, 1));
+ assertEquals(1, BitVectorHelper.get(validityBuffer, 2));
+ assertEquals(0, BitVectorHelper.get(validityBuffer, 3));
+ assertEquals(0, BitVectorHelper.get(validityBuffer, 4));
+ assertEquals(1, BitVectorHelper.get(validityBuffer, 5));
+
+ assertEquals(0, offsetBuffer.getInt(0 *
BaseLargeRepeatedValueViewVector.OFFSET_WIDTH));
+ assertEquals(0, offsetBuffer.getInt(1 *
BaseLargeRepeatedValueViewVector.OFFSET_WIDTH));
+ assertEquals(2, offsetBuffer.getInt(2 *
BaseLargeRepeatedValueViewVector.OFFSET_WIDTH));
+ assertEquals(0, offsetBuffer.getInt(3 *
BaseLargeRepeatedValueViewVector.OFFSET_WIDTH));
+ assertEquals(0, offsetBuffer.getInt(4 *
BaseLargeRepeatedValueViewVector.OFFSET_WIDTH));
+ assertEquals(4, offsetBuffer.getInt(5 *
BaseLargeRepeatedValueViewVector.OFFSET_WIDTH));
+
+ assertEquals(2, sizeBuffer.getInt(0 *
BaseLargeRepeatedValueViewVector.SIZE_WIDTH));
+ assertEquals(0, sizeBuffer.getInt(1 *
BaseLargeRepeatedValueViewVector.SIZE_WIDTH));
+ assertEquals(2, sizeBuffer.getInt(2 *
BaseLargeRepeatedValueViewVector.SIZE_WIDTH));
+ assertEquals(0, sizeBuffer.getInt(3 *
BaseLargeRepeatedValueViewVector.SIZE_WIDTH));
+ assertEquals(0, sizeBuffer.getInt(4 *
BaseLargeRepeatedValueViewVector.SIZE_WIDTH));
+ assertEquals(2, sizeBuffer.getInt(5 *
BaseLargeRepeatedValueViewVector.SIZE_WIDTH));
+
+ // validate values
+
+ Object result = vector.getObject(0);
+ ArrayList<Long> resultSet = (ArrayList<Long>) result;
+ assertEquals(2, resultSet.size());
+ assertEquals(Long.valueOf(10), resultSet.get(0));
+ assertEquals(Long.valueOf(20), resultSet.get(1));
+
+ result = vector.getObject(2);
+ resultSet = (ArrayList<Long>) result;
+ assertEquals(2, resultSet.size());
+ assertEquals(Long.valueOf(30), resultSet.get(0));
+ assertEquals(Long.valueOf(40), resultSet.get(1));
+
+ result = vector.getObject(5);
+ resultSet = (ArrayList<Long>) result;
+ assertEquals(2, resultSet.size());
+ assertEquals(Long.valueOf(50), resultSet.get(0));
+ assertEquals(Long.valueOf(60), resultSet.get(1));
+
+ vector.validate();
+ }
+ }
+
+ @Test
+ public void testSetNull2() {
+ try (LargeListViewVector vector =
LargeListViewVector.empty("largelistview", allocator)) {
+ // validate setting nulls first and then writing values
+ UnionLargeListViewWriter writer = vector.getWriter();
+ writer.allocate();
+
+ vector.setNull(0);
+ vector.setNull(2);
+ vector.setNull(4);
+
+ writer.setPosition(1);
+ writer.startListView();
+ writer.bigInt().writeBigInt(10);
+ writer.bigInt().writeBigInt(20);
+ writer.bigInt().writeBigInt(30);
+ writer.endListView();
+
+ writer.setPosition(3);
+ writer.startListView();
+ writer.bigInt().writeBigInt(40);
+ writer.bigInt().writeBigInt(50);
+ writer.endListView();
+
+ writer.setPosition(5);
+ writer.startListView();
+ writer.bigInt().writeBigInt(60);
+ writer.bigInt().writeBigInt(70);
+ writer.bigInt().writeBigInt(80);
+ writer.endListView();
+
+ vector.setValueCount(6);
+
+ assertTrue(vector.isNull(0));
+ assertFalse(vector.isNull(1));
+ assertTrue(vector.isNull(2));
+ assertFalse(vector.isNull(3));
+ assertTrue(vector.isNull(4));
+ assertFalse(vector.isNull(5));
+
+ // validate buffers
+
+ final ArrowBuf validityBuffer = vector.getValidityBuffer();
+ final ArrowBuf offsetBuffer = vector.getOffsetBuffer();
+ final ArrowBuf sizeBuffer = vector.getSizeBuffer();
+
+ assertEquals(0, BitVectorHelper.get(validityBuffer, 0));
+ assertEquals(1, BitVectorHelper.get(validityBuffer, 1));
+ assertEquals(0, BitVectorHelper.get(validityBuffer, 2));
+ assertEquals(1, BitVectorHelper.get(validityBuffer, 3));
+ assertEquals(0, BitVectorHelper.get(validityBuffer, 4));
+ assertEquals(1, BitVectorHelper.get(validityBuffer, 5));
+
+ assertEquals(0, offsetBuffer.getInt(0 *
BaseLargeRepeatedValueViewVector.OFFSET_WIDTH));
+ assertEquals(0, offsetBuffer.getInt(1 *
BaseLargeRepeatedValueViewVector.OFFSET_WIDTH));
+ assertEquals(0, offsetBuffer.getInt(2 *
BaseLargeRepeatedValueViewVector.OFFSET_WIDTH));
+ assertEquals(3, offsetBuffer.getInt(3 *
BaseLargeRepeatedValueViewVector.OFFSET_WIDTH));
+ assertEquals(0, offsetBuffer.getInt(4 *
BaseLargeRepeatedValueViewVector.OFFSET_WIDTH));
+ assertEquals(5, offsetBuffer.getInt(5 *
BaseLargeRepeatedValueViewVector.OFFSET_WIDTH));
+
+ assertEquals(0, sizeBuffer.getInt(0 *
BaseLargeRepeatedValueViewVector.SIZE_WIDTH));
+ assertEquals(3, sizeBuffer.getInt(1 *
BaseLargeRepeatedValueViewVector.SIZE_WIDTH));
+ assertEquals(0, sizeBuffer.getInt(2 *
BaseLargeRepeatedValueViewVector.SIZE_WIDTH));
+ assertEquals(2, sizeBuffer.getInt(3 *
BaseLargeRepeatedValueViewVector.SIZE_WIDTH));
+ assertEquals(0, sizeBuffer.getInt(4 *
BaseLargeRepeatedValueViewVector.SIZE_WIDTH));
+ assertEquals(3, sizeBuffer.getInt(5 *
BaseLargeRepeatedValueViewVector.SIZE_WIDTH));
+
+ // validate values
+
+ Object result = vector.getObject(1);
+ ArrayList<Long> resultSet = (ArrayList<Long>) result;
+ assertEquals(3, resultSet.size());
+ assertEquals(Long.valueOf(10), resultSet.get(0));
+ assertEquals(Long.valueOf(20), resultSet.get(1));
+ assertEquals(Long.valueOf(30), resultSet.get(2));
+
+ result = vector.getObject(3);
+ resultSet = (ArrayList<Long>) result;
+ assertEquals(2, resultSet.size());
+ assertEquals(Long.valueOf(40), resultSet.get(0));
+ assertEquals(Long.valueOf(50), resultSet.get(1));
+
+ result = vector.getObject(5);
+ resultSet = (ArrayList<Long>) result;
+ assertEquals(3, resultSet.size());
+ assertEquals(Long.valueOf(60), resultSet.get(0));
+ assertEquals(Long.valueOf(70), resultSet.get(1));
+ assertEquals(Long.valueOf(80), resultSet.get(2));
+
+ vector.validate();
+ }
+ }
+
+ @Test
+ public void testSetNull3() {
+ try (LargeListViewVector vector =
LargeListViewVector.empty("largelistview", allocator)) {
+ // validate setting values first and then writing nulls
+ UnionLargeListViewWriter writer = vector.getWriter();
+ writer.allocate();
+
+ writer.setPosition(1);
+ writer.startListView();
+ writer.bigInt().writeBigInt(10);
+ writer.bigInt().writeBigInt(20);
+ writer.bigInt().writeBigInt(30);
+ writer.endListView();
+
+ writer.setPosition(3);
+ writer.startListView();
+ writer.bigInt().writeBigInt(40);
+ writer.bigInt().writeBigInt(50);
+ writer.endListView();
+
+ writer.setPosition(5);
+ writer.startListView();
+ writer.bigInt().writeBigInt(60);
+ writer.bigInt().writeBigInt(70);
+ writer.bigInt().writeBigInt(80);
+ writer.endListView();
+
+ vector.setNull(0);
+ vector.setNull(2);
+ vector.setNull(4);
+
+ vector.setValueCount(6);
+
+ assertTrue(vector.isNull(0));
+ assertFalse(vector.isNull(1));
+ assertTrue(vector.isNull(2));
+ assertFalse(vector.isNull(3));
+ assertTrue(vector.isNull(4));
+ assertFalse(vector.isNull(5));
+
+ // validate buffers
+
+ final ArrowBuf validityBuffer = vector.getValidityBuffer();
+ final ArrowBuf offsetBuffer = vector.getOffsetBuffer();
+ final ArrowBuf sizeBuffer = vector.getSizeBuffer();
+
+ assertEquals(0, BitVectorHelper.get(validityBuffer, 0));
+ assertEquals(1, BitVectorHelper.get(validityBuffer, 1));
+ assertEquals(0, BitVectorHelper.get(validityBuffer, 2));
+ assertEquals(1, BitVectorHelper.get(validityBuffer, 3));
+ assertEquals(0, BitVectorHelper.get(validityBuffer, 4));
+ assertEquals(1, BitVectorHelper.get(validityBuffer, 5));
+
+ assertEquals(0, offsetBuffer.getInt(0 *
BaseLargeRepeatedValueViewVector.OFFSET_WIDTH));
+ assertEquals(0, offsetBuffer.getInt(1 *
BaseLargeRepeatedValueViewVector.OFFSET_WIDTH));
+ assertEquals(0, offsetBuffer.getInt(2 *
BaseLargeRepeatedValueViewVector.OFFSET_WIDTH));
+ assertEquals(3, offsetBuffer.getInt(3 *
BaseLargeRepeatedValueViewVector.OFFSET_WIDTH));
+ assertEquals(0, offsetBuffer.getInt(4 *
BaseLargeRepeatedValueViewVector.OFFSET_WIDTH));
+ assertEquals(5, offsetBuffer.getInt(5 *
BaseLargeRepeatedValueViewVector.OFFSET_WIDTH));
+
+ assertEquals(0, sizeBuffer.getInt(0 *
BaseLargeRepeatedValueViewVector.SIZE_WIDTH));
+ assertEquals(3, sizeBuffer.getInt(1 *
BaseLargeRepeatedValueViewVector.SIZE_WIDTH));
+ assertEquals(0, sizeBuffer.getInt(2 *
BaseLargeRepeatedValueViewVector.SIZE_WIDTH));
+ assertEquals(2, sizeBuffer.getInt(3 *
BaseLargeRepeatedValueViewVector.SIZE_WIDTH));
+ assertEquals(0, sizeBuffer.getInt(4 *
BaseLargeRepeatedValueViewVector.SIZE_WIDTH));
+ assertEquals(3, sizeBuffer.getInt(5 *
BaseLargeRepeatedValueViewVector.SIZE_WIDTH));
+
+ // validate values
+
+ Object result = vector.getObject(1);
+ ArrayList<Long> resultSet = (ArrayList<Long>) result;
+ assertEquals(3, resultSet.size());
+ assertEquals(Long.valueOf(10), resultSet.get(0));
+ assertEquals(Long.valueOf(20), resultSet.get(1));
+ assertEquals(Long.valueOf(30), resultSet.get(2));
+
+ result = vector.getObject(3);
+ resultSet = (ArrayList<Long>) result;
+ assertEquals(2, resultSet.size());
+ assertEquals(Long.valueOf(40), resultSet.get(0));
+ assertEquals(Long.valueOf(50), resultSet.get(1));
+
+ result = vector.getObject(5);
+ resultSet = (ArrayList<Long>) result;
+ assertEquals(3, resultSet.size());
+ assertEquals(Long.valueOf(60), resultSet.get(0));
+ assertEquals(Long.valueOf(70), resultSet.get(1));
+ assertEquals(Long.valueOf(80), resultSet.get(2));
+
+ vector.validate();
+ }
+ }
+
+ @Test
+ public void testOverWrite1() {
+ try (LargeListViewVector vector =
LargeListViewVector.empty("largelistview", allocator)) {
+ UnionLargeListViewWriter writer = vector.getWriter();
+ writer.allocate();
+
+ writer.setPosition(0);
+ writer.startListView();
+ writer.bigInt().writeBigInt(10);
+ writer.bigInt().writeBigInt(20);
+ writer.bigInt().writeBigInt(30);
+ writer.endListView();
+
+ writer.setPosition(1);
+ writer.startListView();
+ writer.bigInt().writeBigInt(40);
+ writer.bigInt().writeBigInt(50);
+ writer.endListView();
+
+ vector.setValueCount(2);
+
+ writer.setPosition(0);
+ writer.startListView();
+ writer.bigInt().writeBigInt(60);
+ writer.bigInt().writeBigInt(70);
+ writer.endListView();
+
+ writer.setPosition(1);
+ writer.startListView();
+ writer.bigInt().writeBigInt(80);
+ writer.bigInt().writeBigInt(90);
+ writer.endListView();
+
+ vector.setValueCount(2);
+
+ Object result = vector.getObject(0);
+ ArrayList<Long> resultSet = (ArrayList<Long>) result;
+ assertEquals(2, resultSet.size());
+ assertEquals(Long.valueOf(60), resultSet.get(0));
+ assertEquals(Long.valueOf(70), resultSet.get(1));
+
+ result = vector.getObject(1);
+ resultSet = (ArrayList<Long>) result;
+ assertEquals(2, resultSet.size());
+ assertEquals(Long.valueOf(80), resultSet.get(0));
+ assertEquals(Long.valueOf(90), resultSet.get(1));
+
+ vector.validate();
+ }
+ }
+
+ @Test
+ public void testOverwriteWithNull() {
+ try (LargeListViewVector vector =
LargeListViewVector.empty("largelistview", allocator)) {
+ UnionLargeListViewWriter writer = vector.getWriter();
+ writer.allocate();
+
+ ArrowBuf offsetBuffer = vector.getOffsetBuffer();
+ ArrowBuf sizeBuffer = vector.getSizeBuffer();
+
+ writer.setPosition(0);
+ writer.startListView();
+ writer.bigInt().writeBigInt(10);
+ writer.bigInt().writeBigInt(20);
+ writer.bigInt().writeBigInt(30);
+ writer.endListView();
+
+ writer.setPosition(1);
+ writer.startListView();
+ writer.bigInt().writeBigInt(40);
+ writer.bigInt().writeBigInt(50);
+ writer.endListView();
+
+ vector.setValueCount(2);
+
+ assertEquals(0, offsetBuffer.getInt(0 *
BaseLargeRepeatedValueViewVector.OFFSET_WIDTH));
+ assertEquals(3, offsetBuffer.getInt(1 *
BaseLargeRepeatedValueViewVector.OFFSET_WIDTH));
+
+ assertEquals(3, sizeBuffer.getInt(0 *
BaseLargeRepeatedValueViewVector.SIZE_WIDTH));
+ assertEquals(2, sizeBuffer.getInt(1 *
BaseLargeRepeatedValueViewVector.SIZE_WIDTH));
+
+ vector.setNull(0);
+
+ assertEquals(0, offsetBuffer.getInt(0 *
BaseLargeRepeatedValueViewVector.OFFSET_WIDTH));
+ assertEquals(0, sizeBuffer.getInt(0 *
BaseLargeRepeatedValueViewVector.SIZE_WIDTH));
+
+ vector.setNull(1);
+
+ assertEquals(0, offsetBuffer.getInt(0 *
BaseLargeRepeatedValueViewVector.OFFSET_WIDTH));
+ assertEquals(0, sizeBuffer.getInt(0 *
BaseLargeRepeatedValueViewVector.SIZE_WIDTH));
+
+ assertTrue(vector.isNull(0));
+ assertTrue(vector.isNull(1));
+
+ writer.setPosition(0);
+ writer.startListView();
+ writer.bigInt().writeBigInt(60);
+ writer.bigInt().writeBigInt(70);
+ writer.endListView();
+
+ assertEquals(0, offsetBuffer.getInt(0 *
BaseLargeRepeatedValueViewVector.OFFSET_WIDTH));
+ assertEquals(2, sizeBuffer.getInt(0 *
BaseLargeRepeatedValueViewVector.SIZE_WIDTH));
+
+ writer.setPosition(1);
+ writer.startListView();
+ writer.bigInt().writeBigInt(80);
+ writer.bigInt().writeBigInt(90);
+ writer.endListView();
+
+ assertEquals(2, offsetBuffer.getInt(1 *
BaseLargeRepeatedValueViewVector.OFFSET_WIDTH));
+ assertEquals(2, sizeBuffer.getInt(1 *
BaseLargeRepeatedValueViewVector.SIZE_WIDTH));
+
+ vector.setValueCount(2);
+
+ assertFalse(vector.isNull(0));
+ assertFalse(vector.isNull(1));
+
+ Object result = vector.getObject(0);
+ ArrayList<Long> resultSet = (ArrayList<Long>) result;
+ assertEquals(2, resultSet.size());
+ assertEquals(Long.valueOf(60), resultSet.get(0));
+ assertEquals(Long.valueOf(70), resultSet.get(1));
+
+ result = vector.getObject(1);
+ resultSet = (ArrayList<Long>) result;
+ assertEquals(2, resultSet.size());
+ assertEquals(Long.valueOf(80), resultSet.get(0));
+ assertEquals(Long.valueOf(90), resultSet.get(1));
+
+ vector.validate();
+ }
+ }
+
+ @Test
+ public void testOutOfOrderOffset1() {
+ // [[12, -7, 25], null, [0, -127, 127, 50], [], [50, 12]]
+ try (LargeListViewVector largeListViewVector =
+ LargeListViewVector.empty("largelistview", allocator)) {
+ // Allocate buffers in largeListViewVector by calling `allocateNew`
method.
+ largeListViewVector.allocateNew();
+
+ // Initialize the child vector using `initializeChildrenFromFields`
method.
+
+ FieldType fieldType = new FieldType(true, new ArrowType.Int(16, true),
null, null);
+ Field field = new Field("child-vector", fieldType, null);
+
largeListViewVector.initializeChildrenFromFields(Collections.singletonList(field));
+
+ // Set values in the child vector.
+ FieldVector fieldVector = largeListViewVector.getDataVector();
+ fieldVector.clear();
+
+ SmallIntVector childVector = (SmallIntVector) fieldVector;
+
+ childVector.allocateNew(7);
+
+ childVector.set(0, 0);
+ childVector.set(1, -127);
+ childVector.set(2, 127);
+ childVector.set(3, 50);
+ childVector.set(4, 12);
+ childVector.set(5, -7);
+ childVector.set(6, 25);
+
+ childVector.setValueCount(7);
+
+ // Set validity, offset and size buffers using `setValidity`,
+ // `setOffset` and `setSize` methods.
+ largeListViewVector.setValidity(0, 1);
+ largeListViewVector.setValidity(1, 0);
+ largeListViewVector.setValidity(2, 1);
+ largeListViewVector.setValidity(3, 1);
+ largeListViewVector.setValidity(4, 1);
+
+ largeListViewVector.setOffset(0, 4);
+ largeListViewVector.setOffset(1, 7);
+ largeListViewVector.setOffset(2, 0);
+ largeListViewVector.setOffset(3, 0);
+ largeListViewVector.setOffset(4, 3);
+
+ largeListViewVector.setSize(0, 3);
+ largeListViewVector.setSize(1, 0);
+ largeListViewVector.setSize(2, 4);
+ largeListViewVector.setSize(3, 0);
+ largeListViewVector.setSize(4, 2);
+
+ // Set value count using `setValueCount` method.
+ largeListViewVector.setValueCount(5);
+
+ final ArrowBuf offSetBuffer = largeListViewVector.getOffsetBuffer();
+ final ArrowBuf sizeBuffer = largeListViewVector.getSizeBuffer();
+
+ // check offset buffer
+ assertEquals(4, offSetBuffer.getInt(0 *
BaseLargeRepeatedValueViewVector.OFFSET_WIDTH));
+ assertEquals(7, offSetBuffer.getInt(1 *
BaseLargeRepeatedValueViewVector.OFFSET_WIDTH));
+ assertEquals(0, offSetBuffer.getInt(2 *
BaseLargeRepeatedValueViewVector.OFFSET_WIDTH));
+ assertEquals(0, offSetBuffer.getInt(3 *
BaseLargeRepeatedValueViewVector.OFFSET_WIDTH));
+ assertEquals(3, offSetBuffer.getInt(4 *
BaseLargeRepeatedValueViewVector.OFFSET_WIDTH));
+
+ // check size buffer
+ assertEquals(3, sizeBuffer.getInt(0 *
BaseLargeRepeatedValueViewVector.SIZE_WIDTH));
+ assertEquals(0, sizeBuffer.getInt(1 *
BaseLargeRepeatedValueViewVector.SIZE_WIDTH));
+ assertEquals(4, sizeBuffer.getInt(2 *
BaseLargeRepeatedValueViewVector.SIZE_WIDTH));
+ assertEquals(0, sizeBuffer.getInt(3 *
BaseLargeRepeatedValueViewVector.SIZE_WIDTH));
+ assertEquals(2, sizeBuffer.getInt(4 *
BaseLargeRepeatedValueViewVector.SIZE_WIDTH));
+
+ // check child vector
+ assertEquals(0, ((SmallIntVector)
largeListViewVector.getDataVector()).get(0));
+ assertEquals(-127, ((SmallIntVector)
largeListViewVector.getDataVector()).get(1));
+ assertEquals(127, ((SmallIntVector)
largeListViewVector.getDataVector()).get(2));
+ assertEquals(50, ((SmallIntVector)
largeListViewVector.getDataVector()).get(3));
+ assertEquals(12, ((SmallIntVector)
largeListViewVector.getDataVector()).get(4));
+ assertEquals(-7, ((SmallIntVector)
largeListViewVector.getDataVector()).get(5));
+ assertEquals(25, ((SmallIntVector)
largeListViewVector.getDataVector()).get(6));
+
+ // check values
+ Object result = largeListViewVector.getObject(0);
+ ArrayList<Integer> resultSet = (ArrayList<Integer>) result;
+ assertEquals(3, resultSet.size());
+ assertEquals(Short.valueOf("12"), resultSet.get(0));
+ assertEquals(Short.valueOf("-7"), resultSet.get(1));
+ assertEquals(Short.valueOf("25"), resultSet.get(2));
+
+ assertTrue(largeListViewVector.isNull(1));
+
+ result = largeListViewVector.getObject(2);
+ resultSet = (ArrayList<Integer>) result;
+ assertEquals(4, resultSet.size());
+ assertEquals(Short.valueOf("0"), resultSet.get(0));
+ assertEquals(Short.valueOf("-127"), resultSet.get(1));
+ assertEquals(Short.valueOf("127"), resultSet.get(2));
+ assertEquals(Short.valueOf("50"), resultSet.get(3));
+
+ assertTrue(largeListViewVector.isEmpty(3));
+
+ result = largeListViewVector.getObject(4);
+ resultSet = (ArrayList<Integer>) result;
+ assertEquals(2, resultSet.size());
+ assertEquals(Short.valueOf("50"), resultSet.get(0));
+ assertEquals(Short.valueOf("12"), resultSet.get(1));
+
+ largeListViewVector.validate();
+ }
+ }
+
+ private void writeIntValues(UnionLargeListViewWriter writer, int[] values) {
+ writer.startListView();
+ for (int v : values) {
+ writer.integer().writeInt(v);
+ }
+ writer.endListView();
+ }
+}
