This is an automated email from the ASF dual-hosted git repository.
cutlerb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 262d1dd ARROW-5911: [Java] Make ListVector and MapVector create
reader lazily
262d1dd is described below
commit 262d1dded495703efc2d1656b0d94f54dbd3d349
Author: liyafan82 <[email protected]>
AuthorDate: Tue Jul 16 11:31:32 2019 -0700
ARROW-5911: [Java] Make ListVector and MapVector create reader lazily
Current implementation creates reader eagerly, which may cause unnecessary
resource and time. This issue changes the behavior to lazily create the reader.
This is a follow-up issue for ARROW-5897.
Author: liyafan82 <[email protected]>
Closes #4854 from liyafan82/fly_0711_lazy and squashes the following
commits:
9d32376 <liyafan82> Support FixedSizeListVector
3d99651 <liyafan82> Make ListVector and MapVector create reader lazily
---
.../org/apache/arrow/vector/complex/FixedSizeListVector.java | 12 +++++++++---
.../java/org/apache/arrow/vector/complex/ListVector.java | 12 +++++++-----
.../main/java/org/apache/arrow/vector/complex/MapVector.java | 8 +++-----
3 files changed, 19 insertions(+), 13 deletions(-)
diff --git
a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
index f62ec59..1d2c8db 100644
---
a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
+++
b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
@@ -103,7 +103,6 @@ public class FixedSizeListVector extends BaseValueVector
implements FieldVector,
this.fieldType = fieldType;
this.listSize = ((ArrowType.FixedSizeList)
fieldType.getType()).getListSize();
Preconditions.checkArgument(listSize > 0, "list size must be positive");
- this.reader = new UnionFixedSizeListReader(this);
this.valueCount = 0;
this.validityAllocationSizeInBytes =
getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION);
}
@@ -184,9 +183,16 @@ public class FixedSizeListVector extends BaseValueVector
implements FieldVector,
@Override
public UnionFixedSizeListReader getReader() {
+ if (reader == null) {
+ reader = new UnionFixedSizeListReader(this);
+ }
return reader;
}
+ private void invalidateReader() {
+ reader = null;
+ }
+
@Override
public void allocateNew() throws OutOfMemoryException {
if (!allocateNewSafe()) {
@@ -346,7 +352,7 @@ public class FixedSizeListVector extends BaseValueVector
implements FieldVector,
boolean created = false;
if (vector == ZeroVector.INSTANCE) {
vector = type.createNewSingleVector(DATA_VECTOR_NAME, allocator, null);
- this.reader = new UnionFixedSizeListReader(this);
+ invalidateReader();
created = true;
}
// returned vector must have the same field
@@ -373,7 +379,7 @@ public class FixedSizeListVector extends BaseValueVector
implements FieldVector,
UnionVector vector = new UnionVector(name, allocator, null);
this.vector.clear();
this.vector = vector;
- this.reader = new UnionFixedSizeListReader(this);
+ invalidateReader();
return vector;
}
diff --git
a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
index af5333c..945f50f 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
@@ -105,7 +105,6 @@ public class ListVector extends BaseRepeatedValueVector
implements FieldVector,
public ListVector(String name, BufferAllocator allocator, FieldType
fieldType, CallBack callBack) {
super(name, allocator, callBack);
this.validityBuffer = allocator.getEmpty();
- createReader();
this.fieldType = checkNotNull(fieldType);
this.callBack = callBack;
this.validityAllocationSizeInBytes =
getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION);
@@ -545,13 +544,16 @@ public class ListVector extends BaseRepeatedValueVector
implements FieldVector,
@Override
public UnionListReader getReader() {
+ if (reader == null) {
+ reader = new UnionListReader(this);
+ }
return reader;
}
/** Initialize the child data vector to field type. */
public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(FieldType
fieldType) {
AddOrGetResult<T> result = super.addOrGetVector(fieldType);
- createReader();
+ invalidateReader();
return result;
}
@@ -631,15 +633,15 @@ public class ListVector extends BaseRepeatedValueVector
implements FieldVector,
public UnionVector promoteToUnion() {
UnionVector vector = new UnionVector("$data$", allocator, callBack);
replaceDataVector(vector);
- createReader();
+ invalidateReader();
if (callBack != null) {
callBack.doWork();
}
return vector;
}
- protected void createReader() {
- reader = new UnionListReader(this);
+ protected void invalidateReader() {
+ reader = null;
}
/**
diff --git
a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
index 0fcb19c..994a7d0 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
@@ -109,11 +109,9 @@ public class MapVector extends ListVector {
*/
@Override
public UnionMapReader getReader() {
+ if (reader == null) {
+ reader = new UnionMapReader(this);
+ }
return (UnionMapReader)reader;
}
-
- @Override
- protected void createReader() {
- reader = new UnionMapReader(this);
- }
}
