[
https://issues.apache.org/jira/browse/ARROW-1885?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16277686#comment-16277686
]
ASF GitHub Bot commented on ARROW-1885:
---------------------------------------
wesm closed pull request #1389: ARROW-1885: [Java] Restore MapVector class
names prior to ARROW-1710
URL: https://github.com/apache/arrow/pull/1389
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/java/vector/src/main/codegen/templates/CaseSensitiveMapWriters.java
b/java/vector/src/main/codegen/templates/CaseSensitiveMapWriters.java
index 7c652902e..5357f9b8a 100644
--- a/java/vector/src/main/codegen/templates/CaseSensitiveMapWriters.java
+++ b/java/vector/src/main/codegen/templates/CaseSensitiveMapWriters.java
@@ -23,7 +23,7 @@
<#if mode == "Single">
<#assign containerClass = "MapVector" />
<#else>
-<#assign containerClass = "MapVector" />
+<#assign containerClass = "NullableMapVector" />
</#if>
<#include "/@includes/license.ftl" />
diff --git a/java/vector/src/main/codegen/templates/MapWriters.java
b/java/vector/src/main/codegen/templates/MapWriters.java
index a5ac1b717..ac59e59b3 100644
--- a/java/vector/src/main/codegen/templates/MapWriters.java
+++ b/java/vector/src/main/codegen/templates/MapWriters.java
@@ -21,9 +21,9 @@
<@pp.changeOutputFile
name="/org/apache/arrow/vector/complex/impl/${mode}MapWriter.java" />
<#assign index = "idx()">
<#if mode == "Single">
-<#assign containerClass = "NonNullableMapVector" />
-<#else>
<#assign containerClass = "MapVector" />
+<#else>
+<#assign containerClass = "NullableMapVector" />
</#if>
<#include "/@includes/license.ftl" />
@@ -51,7 +51,7 @@
private final Map<String, FieldWriter> fields = Maps.newHashMap();
public ${mode}MapWriter(${containerClass} container) {
<#if mode == "Single">
- if (container instanceof MapVector) {
+ if (container instanceof NullableMapVector) {
throw new IllegalArgumentException("Invalid container: " + container);
}
</#if>
@@ -124,7 +124,7 @@ public MapWriter map(String name) {
FieldWriter writer = fields.get(finalName);
if(writer == null){
int vectorCount=container.size();
- MapVector vector = container.addOrGet(name,
FieldType.nullable(MinorType.MAP.getType()), MapVector.class);
+ NullableMapVector vector = container.addOrGet(name,
FieldType.nullable(MinorType.MAP.getType()), NullableMapVector.class);
writer = new PromotableWriter(vector, container,
getNullableMapWriterFactory());
if(vectorCount != container.size()) {
writer.allocate();
diff --git a/java/vector/src/main/codegen/templates/UnionVector.java
b/java/vector/src/main/codegen/templates/UnionVector.java
index 60cd24dcc..9377bd0da 100644
--- a/java/vector/src/main/codegen/templates/UnionVector.java
+++ b/java/vector/src/main/codegen/templates/UnionVector.java
@@ -48,8 +48,8 @@
/**
- * A vector which can hold values of different types. It does so by using a
MapVector which contains a vector for each
- * primitive type that is stored. MapVector is used in order to take advantage
of its serialization/deserialization methods,
+ * A vector which can hold values of different types. It does so by using a
NullableMapVector which contains a vector for each
+ * primitive type that is stored. NullableMapVector is used in order to take
advantage of its serialization/deserialization methods,
* as well as the addOrGet method.
*
* For performance reasons, UnionVector stores a cached reference to each
subtype vector, to avoid having to do the map lookup
@@ -65,7 +65,7 @@
MapVector internalMap;
protected ArrowBuf typeBuffer;
- private MapVector mapVector;
+ private NullableMapVector mapVector;
private ListVector listVector;
private FieldReader reader;
@@ -174,10 +174,10 @@ public long getOffsetBufferAddress() {
@Override
public ArrowBuf getOffsetBuffer() { throw new
UnsupportedOperationException(); }
- public MapVector getMap() {
+ public NullableMapVector getMap() {
if (mapVector == null) {
int vectorCount = internalMap.size();
- mapVector = addOrGet(MinorType.MAP, MapVector.class);
+ mapVector = addOrGet(MinorType.MAP, NullableMapVector.class);
if (internalMap.size() > vectorCount) {
mapVector.allocateNew();
if (callBack != null) {
diff --git
a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java
b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java
index 98820a236..fbadb35d7 100644
---
a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java
+++
b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java
@@ -24,7 +24,7 @@
import org.apache.arrow.memory.OutOfMemoryException;
import org.apache.arrow.memory.BaseAllocator;
import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.vector.complex.MapVector;
+import org.apache.arrow.vector.complex.NullableMapVector;
import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.FieldType;
diff --git
a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java
b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java
index 5ded1cc6f..db0ff86df 100644
---
a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java
+++
b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java
@@ -100,8 +100,8 @@ protected boolean supportsDirectRead() {
// return the child vector's ordinal in the composite container
public abstract VectorWithOrdinal getChildVectorWithOrdinal(String name);
- public MapVector addOrGetMap(String name) {
- return addOrGet(name, FieldType.nullable(new Struct()), MapVector.class);
+ public NullableMapVector addOrGetMap(String name) {
+ return addOrGet(name, FieldType.nullable(new Struct()),
NullableMapVector.class);
}
public ListVector addOrGetList(String name) {
diff --git
a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
index 72cc29f30..e13084588 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- * <p>
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- * <p>
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -21,480 +21,322 @@
import static com.google.common.base.Preconditions.checkNotNull;
import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
+import java.util.Collection;
+import java.util.Iterator;
import java.util.List;
+import java.util.Map;
-import com.google.common.collect.ObjectArrays;
+import javax.annotation.Nullable;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Ordering;
+import com.google.common.primitives.Ints;
import io.netty.buffer.ArrowBuf;
-import org.apache.arrow.memory.BaseAllocator;
+
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.vector.*;
-import org.apache.arrow.vector.complex.impl.NullableMapReaderImpl;
-import org.apache.arrow.vector.complex.impl.NullableMapWriter;
+import org.apache.arrow.vector.complex.impl.SingleMapReaderImpl;
+import org.apache.arrow.vector.complex.reader.FieldReader;
import org.apache.arrow.vector.holders.ComplexHolder;
-import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
+import org.apache.arrow.vector.types.Types.MinorType;
import org.apache.arrow.vector.types.pojo.ArrowType;
-import org.apache.arrow.vector.types.pojo.ArrowType.Struct;
-import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
-import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.util.CallBack;
-import org.apache.arrow.vector.util.OversizedAllocationException;
+import org.apache.arrow.vector.util.JsonStringHashMap;
import org.apache.arrow.vector.util.TransferPair;
-public class MapVector extends NonNullableMapVector implements FieldVector {
+public class MapVector extends AbstractMapVector {
+ //private static final org.slf4j.Logger logger =
org.slf4j.LoggerFactory.getLogger(MapVector.class);
public static MapVector empty(String name, BufferAllocator allocator) {
- FieldType fieldType = FieldType.nullable(Struct.INSTANCE);
+ FieldType fieldType = new FieldType(false, ArrowType.Struct.INSTANCE,
null, null);
return new MapVector(name, allocator, fieldType, null);
}
- private final NullableMapReaderImpl reader = new NullableMapReaderImpl(this);
- private final NullableMapWriter writer = new NullableMapWriter(this);
-
- protected ArrowBuf validityBuffer;
- private int validityAllocationSizeInBytes;
+ private final SingleMapReaderImpl reader = new SingleMapReaderImpl(this);
+ protected final FieldType fieldType;
+ public int valueCount;
// deprecated, use FieldType or static constructor instead
@Deprecated
public MapVector(String name, BufferAllocator allocator, CallBack callBack) {
- this(name, allocator, FieldType.nullable(ArrowType.Struct.INSTANCE),
callBack);
+ this(name, allocator, new FieldType(false, ArrowType.Struct.INSTANCE,
null, null), callBack);
}
- // deprecated, use FieldType or static constructor instead
- @Deprecated
- public MapVector(String name, BufferAllocator allocator, DictionaryEncoding
dictionary, CallBack callBack) {
- this(name, allocator, new FieldType(true, ArrowType.Struct.INSTANCE,
dictionary, null), callBack);
+ public MapVector(String name, BufferAllocator allocator, FieldType
fieldType, CallBack callBack) {
+ super(name, allocator, callBack);
+ this.fieldType = checkNotNull(fieldType);
+ this.valueCount = 0;
}
- public MapVector(String name, BufferAllocator allocator, FieldType
fieldType, CallBack callBack) {
- super(name, checkNotNull(allocator), fieldType, callBack);
- this.validityBuffer = allocator.getEmpty();
- this.validityAllocationSizeInBytes =
BitVectorHelper.getValidityBufferSize(BaseValueVector.INITIAL_VALUE_ALLOCATION);
+ @Override
+ public FieldReader getReader() {
+ return reader;
+ }
+
+ transient private MapTransferPair ephPair;
+
+ public void copyFromSafe(int fromIndex, int thisIndex, MapVector from) {
+ if (ephPair == null || ephPair.from != from) {
+ ephPair = (MapTransferPair) from.makeTransferPair(this);
+ }
+ ephPair.copyValueSafe(fromIndex, thisIndex);
}
@Override
- public Field getField() {
- Field f = super.getField();
- FieldType type = new FieldType(true, f.getType(),
f.getFieldType().getDictionary(), f.getFieldType().getMetadata());
- return new Field(f.getName(), type, f.getChildren());
+ protected boolean supportsDirectRead() {
+ return true;
+ }
+
+ public Iterator<String> fieldNameIterator() {
+ return getChildFieldNames().iterator();
}
@Override
- public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf>
ownBuffers) {
- if (ownBuffers.size() != 1) {
- throw new IllegalArgumentException("Illegal buffer count, expected " + 1
+ ", got: " + ownBuffers.size());
+ public void setInitialCapacity(int numRecords) {
+ for (final ValueVector v : (Iterable<ValueVector>) this) {
+ v.setInitialCapacity(numRecords);
}
+ }
- ArrowBuf bitBuffer = ownBuffers.get(0);
+ @Override
+ public int getBufferSize() {
+ if (valueCount == 0 || size() == 0) {
+ return 0;
+ }
+ long buffer = 0;
+ for (final ValueVector v : (Iterable<ValueVector>) this) {
+ buffer += v.getBufferSize();
+ }
- validityBuffer.release();
- validityBuffer = BitVectorHelper.loadValidityBuffer(fieldNode, bitBuffer,
allocator);
- valueCount = fieldNode.getLength();
- validityAllocationSizeInBytes = validityBuffer.capacity();
+ return (int) buffer;
}
@Override
- public List<ArrowBuf> getFieldBuffers() {
- List<ArrowBuf> result = new ArrayList<>(1);
- setReaderAndWriterIndex();
- result.add(validityBuffer);
+ public int getBufferSizeFor(final int valueCount) {
+ if (valueCount == 0) {
+ return 0;
+ }
- return result;
- }
+ long bufferSize = 0;
+ for (final ValueVector v : (Iterable<ValueVector>) this) {
+ bufferSize += v.getBufferSizeFor(valueCount);
+ }
- private void setReaderAndWriterIndex() {
- validityBuffer.readerIndex(0);
-
validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSize(valueCount));
+ return (int) bufferSize;
}
@Override
- @Deprecated
- public List<BufferBacked> getFieldInnerVectors() {
- throw new UnsupportedOperationException("There are no inner vectors. Use
getFieldBuffers");
+ public ArrowBuf getValidityBuffer() {
+ throw new UnsupportedOperationException();
}
@Override
- public NullableMapReaderImpl getReader() {
- return reader;
+ public ArrowBuf getDataBuffer() {
+ throw new UnsupportedOperationException();
}
- public NullableMapWriter getWriter() {
- return writer;
+ @Override
+ public ArrowBuf getOffsetBuffer() {
+ throw new UnsupportedOperationException();
}
@Override
public TransferPair getTransferPair(BufferAllocator allocator) {
- return new NullableMapTransferPair(this, new MapVector(name, allocator,
fieldType, null), false);
+ return getTransferPair(name, allocator, null);
}
@Override
- public TransferPair makeTransferPair(ValueVector to) {
- return new NullableMapTransferPair(this, (MapVector) to, true);
+ public TransferPair getTransferPair(String ref, BufferAllocator allocator,
CallBack callBack) {
+ return new MapTransferPair(this, new MapVector(name, allocator, fieldType,
callBack), false);
}
@Override
- public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
- return new NullableMapTransferPair(this, new MapVector(ref, allocator,
fieldType, null), false);
+ public TransferPair makeTransferPair(ValueVector to) {
+ return new MapTransferPair(this, (MapVector) to);
}
@Override
- public TransferPair getTransferPair(String ref, BufferAllocator allocator,
CallBack callBack) {
- return new NullableMapTransferPair(this, new MapVector(ref, allocator,
fieldType, callBack), false);
+ public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+ return new MapTransferPair(this, new MapVector(ref, allocator, fieldType,
callBack), false);
}
- protected class NullableMapTransferPair extends MapTransferPair {
-
- private MapVector target;
+ protected static class MapTransferPair implements TransferPair {
+ private final TransferPair[] pairs;
+ private final MapVector from;
+ private final MapVector to;
- protected NullableMapTransferPair(MapVector from, MapVector to, boolean
allocate) {
- super(from, to, allocate);
- this.target = to;
+ public MapTransferPair(MapVector from, MapVector to) {
+ this(from, to, true);
}
- @Override
- public void transfer() {
- target.clear();
- target.validityBuffer =
validityBuffer.transferOwnership(target.allocator).buffer;
- super.transfer();
- clear();
+ protected MapTransferPair(MapVector from, MapVector to, boolean allocate) {
+ this.from = from;
+ this.to = to;
+ this.pairs = new TransferPair[from.size()];
+ this.to.ephPair = null;
+
+ int i = 0;
+ FieldVector vector;
+ for (String child : from.getChildFieldNames()) {
+ int preSize = to.size();
+ vector = from.getChild(child);
+ if (vector == null) {
+ continue;
+ }
+ //DRILL-1872: we add the child fields for the vector, looking up the
field by name. For a map vector,
+ // the child fields may be nested fields of the top level child. For
example if the structure
+ // of a child field is oa.oab.oabc then we add oa, then add oab to oa
then oabc to oab.
+ // But the children member of a Materialized field is a HashSet. If
the fields are added in the
+ // children HashSet, and the hashCode of the Materialized field
includes the hash code of the
+ // children, the hashCode value of oa changes *after* the field has
been added to the HashSet.
+ // (This is similar to what happens in ScanBatch where the children
cannot be added till they are
+ // read). To take care of this, we ensure that the hashCode of the
MaterializedField does not
+ // include the hashCode of the children but is based only on
MaterializedField$key.
+ final FieldVector newVector = to.addOrGet(child,
vector.getField().getFieldType(), vector.getClass());
+ if (allocate && to.size() != preSize) {
+ newVector.allocateNew();
+ }
+ pairs[i++] = vector.makeTransferPair(newVector);
+ }
}
@Override
- public void copyValueSafe(int fromIndex, int toIndex) {
- while (toIndex >= target.getValidityBufferValueCapacity()) {
- target.reallocValidityBuffer();
+ public void transfer() {
+ for (final TransferPair p : pairs) {
+ p.transfer();
}
- BitVectorHelper.setValidityBit(target.validityBuffer, toIndex,
isSet(fromIndex));
- super.copyValueSafe(fromIndex, toIndex);
+ to.valueCount = from.valueCount;
+ from.clear();
}
@Override
- public void splitAndTransfer(int startIndex, int length) {
- target.clear();
- splitAndTransferValidityBuffer(startIndex, length, target);
- super.splitAndTransfer(startIndex, length);
+ public ValueVector getTo() {
+ return to;
}
- }
- /*
- * transfer the validity.
- */
- private void splitAndTransferValidityBuffer(int startIndex, int length,
MapVector target) {
- assert startIndex + length <= valueCount;
- int firstByteSource = BitVectorHelper.byteIndex(startIndex);
- int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1);
- int byteSizeTarget = BitVectorHelper.getValidityBufferSize(length);
- int offset = startIndex % 8;
-
- if (length > 0) {
- if (offset == 0) {
- // slice
- if (target.validityBuffer != null) {
- target.validityBuffer.release();
- }
- target.validityBuffer = validityBuffer.slice(firstByteSource,
byteSizeTarget);
- target.validityBuffer.retain(1);
- } else {
- /* Copy data
- * When the first bit starts from the middle of a byte (offset != 0),
- * copy data from src BitVector.
- * Each byte in the target is composed by a part in i-th byte,
- * another part in (i+1)-th byte.
- */
- target.allocateValidityBuffer(byteSizeTarget);
-
- for (int i = 0; i < byteSizeTarget - 1; i++) {
- byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer,
firstByteSource + i, offset);
- byte b2 = BitVectorHelper.getBitsFromNextByte(validityBuffer,
firstByteSource + i + 1, offset);
-
- target.validityBuffer.setByte(i, (b1 + b2));
- }
-
- /* Copying the last piece is done in the following manner:
- * if the source vector has 1 or more bytes remaining, we copy
- * the last piece as a byte formed by shifting data
- * from the current byte and the next byte.
- *
- * if the source vector has no more bytes remaining
- * (we are at the last byte), we copy the last piece as a byte
- * by shifting data from the current byte.
- */
- if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) {
- byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer,
- firstByteSource + byteSizeTarget - 1, offset);
- byte b2 = BitVectorHelper.getBitsFromNextByte(validityBuffer,
- firstByteSource + byteSizeTarget, offset);
-
- target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2);
- } else {
- byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer,
- firstByteSource + byteSizeTarget - 1, offset);
- target.validityBuffer.setByte(byteSizeTarget - 1, b1);
- }
+ @Override
+ public void copyValueSafe(int from, int to) {
+ for (TransferPair p : pairs) {
+ p.copyValueSafe(from, to);
}
}
- }
-
- /**
- * Get the value capacity of the internal validity buffer.
- * @return number of elements that validity buffer can hold
- */
- private int getValidityBufferValueCapacity() {
- return (int) (validityBuffer.capacity() * 8L);
- }
-
- /**
- * Get the current value capacity for the vector
- * @return number of elements that vector can hold.
- */
- @Override
- public int getValueCapacity() {
- return Math.min(getValidityBufferValueCapacity(),
- super.getValueCapacity());
- }
- /**
- * Return the underlying buffers associated with this vector. Note that this
doesn't
- * impact the reference counts for this buffer so it only should be used for
in-context
- * access. Also note that this buffer changes regularly thus
- * external classes shouldn't hold a reference to it (unless they change it).
- *
- * @param clear Whether to clear vector before returning; the buffers will
still be refcounted
- * but the returned array will be the only reference to them
- * @return The underlying {@link io.netty.buffer.ArrowBuf buffers} that is
used by this
- * vector instance.
- */
- @Override
- public ArrowBuf[] getBuffers(boolean clear) {
- setReaderAndWriterIndex();
- final ArrowBuf[] buffers;
- if (getBufferSize() == 0) {
- buffers = new ArrowBuf[0];
- } else {
- buffers = ObjectArrays.concat(new ArrowBuf[]{validityBuffer},
super.getBuffers(false),
- ArrowBuf.class);
- }
- if (clear) {
- for (ArrowBuf buffer : buffers) {
- buffer.retain();
+ @Override
+ public void splitAndTransfer(int startIndex, int length) {
+ for (TransferPair p : pairs) {
+ p.splitAndTransfer(startIndex, length);
}
- clear();
+ to.setValueCount(length);
}
-
- return buffers;
}
- /**
- * Close the vector and release the associated buffers.
- */
@Override
- public void close() {
- clearValidityBuffer();
- super.close();
- }
-
- /**
- * Same as {@link #close()}
- */
- @Override
- public void clear() {
- clearValidityBuffer();
- super.clear();
- }
-
- /**
- * Release the validity buffer
- */
- private void clearValidityBuffer() {
- validityBuffer.release();
- validityBuffer = allocator.getEmpty();
- }
-
- /**
- * Get the size (number of bytes) of underlying buffers used by this
- * vector
- * @return size of underlying buffers.
- */
- @Override
- public int getBufferSize() {
- if (valueCount == 0) {
+ public int getValueCapacity() {
+ if (size() == 0) {
return 0;
}
- return super.getBufferSize() +
- BitVectorHelper.getValidityBufferSize(valueCount);
- }
- /**
- * Get the potential buffer size for a particular number of records.
- * @param valueCount desired number of elements in the vector
- * @return estimated size of underlying buffers if the vector holds
- * a given number of elements
- */
- @Override
- public int getBufferSizeFor(final int valueCount) {
- if (valueCount == 0) {
- return 0;
- }
- return super.getBufferSizeFor(valueCount)
- + BitVectorHelper.getValidityBufferSize(valueCount);
- }
+ final Ordering<ValueVector> natural = new Ordering<ValueVector>() {
+ @Override
+ public int compare(@Nullable ValueVector left, @Nullable ValueVector
right) {
+ return Ints.compare(
+ checkNotNull(left).getValueCapacity(),
+ checkNotNull(right).getValueCapacity()
+ );
+ }
+ };
- @Override
- public void setInitialCapacity(int numRecords) {
- validityAllocationSizeInBytes =
BitVectorHelper.getValidityBufferSize(numRecords);
- super.setInitialCapacity(numRecords);
+ return natural.min(getChildren()).getValueCapacity();
}
@Override
- public boolean allocateNewSafe() {
- /* Boolean to keep track if all the memory allocations were successful
- * Used in the case of composite vectors when we need to allocate multiple
- * buffers for multiple vectors. If one of the allocations failed we need
to
- * clear all the memory that we allocated
- */
- boolean success = false;
- try {
- clear();
- allocateValidityBuffer(validityAllocationSizeInBytes);
- success = super.allocateNewSafe();
- } finally {
- if (!success) {
- clear();
- return false;
+ public Object getObject(int index) {
+ Map<String, Object> vv = new JsonStringHashMap<>();
+ for (String child : getChildFieldNames()) {
+ ValueVector v = getChild(child);
+ if (v != null && index < v.getValueCount()) {
+ Object value = v.getObject(index);
+ if (value != null) {
+ vv.put(child, value);
+ }
}
}
- return true;
- }
-
- private void allocateValidityBuffer(final long size) {
- final int curSize = (int) size;
- validityBuffer = allocator.buffer(curSize);
- validityBuffer.readerIndex(0);
- validityAllocationSizeInBytes = curSize;
- validityBuffer.setZero(0, validityBuffer.capacity());
+ return vv;
}
@Override
- public void reAlloc() {
- /* reallocate the validity buffer */
- reallocValidityBuffer();
- super.reAlloc();
- }
-
- private void reallocValidityBuffer() {
- final int currentBufferCapacity = validityBuffer.capacity();
- long baseSize = validityAllocationSizeInBytes;
-
- if (baseSize < (long) currentBufferCapacity) {
- baseSize = (long) currentBufferCapacity;
- }
-
- long newAllocationSize = baseSize * 2L;
- newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize);
-
- if (newAllocationSize > BaseValueVector.MAX_ALLOCATION_SIZE) {
- throw new OversizedAllocationException("Unable to expand the buffer");
- }
-
- final ArrowBuf newBuf = allocator.buffer((int) newAllocationSize);
- newBuf.setZero(0, newBuf.capacity());
- newBuf.setBytes(0, validityBuffer, 0, currentBufferCapacity);
- validityBuffer.release(1);
- validityBuffer = newBuf;
- validityAllocationSizeInBytes = (int) newAllocationSize;
- }
-
+ public boolean isNull(int index) { return false; }
@Override
- public long getValidityBufferAddress() {
- return validityBuffer.memoryAddress();
- }
+ public int getNullCount() { return 0; }
- @Override
- public long getDataBufferAddress() {
- throw new UnsupportedOperationException();
+ public void get(int index, ComplexHolder holder) {
+ reader.setPosition(index);
+ holder.reader = reader;
}
@Override
- public long getOffsetBufferAddress() {
- throw new UnsupportedOperationException();
+ public int getValueCount() {
+ return valueCount;
}
- @Override
- public ArrowBuf getValidityBuffer() {
- return validityBuffer;
- }
+ public ValueVector getVectorById(int id) {
+ return getChildByOrdinal(id);
+}
@Override
- public ArrowBuf getDataBuffer() {
- throw new UnsupportedOperationException();
+ public void setValueCount(int valueCount) {
+ for (final ValueVector v : getChildren()) {
+ v.setValueCount(valueCount);
+ }
+ MapVector.this.valueCount = valueCount;
}
@Override
- public ArrowBuf getOffsetBuffer() {
- throw new UnsupportedOperationException();
+ public void clear() {
+ for (final ValueVector v : getChildren()) {
+ v.clear();
+ }
+ valueCount = 0;
}
@Override
- public Object getObject(int index) {
- if (isSet(index) == 0) {
- return null;
- } else {
- return super.getObject(index);
+ public Field getField() {
+ List<Field> children = new ArrayList<>();
+ for (ValueVector child : getChildren()) {
+ children.add(child.getField());
}
+ return new Field(name, fieldType, children);
}
@Override
- public void get(int index, ComplexHolder holder) {
- holder.isSet = isSet(index);
- super.get(index, holder);
+ public MinorType getMinorType() {
+ return MinorType.MAP;
}
- public int getNullCount() {
- return BitVectorHelper.getNullCount(validityBuffer, valueCount);
- }
-
- public boolean isNull(int index) {
- return isSet(index) == 0;
- }
-
- public int isSet(int index) {
- final int byteIndex = index >> 3;
- final byte b = validityBuffer.getByte(byteIndex);
- final int bitIndex = index & 7;
- return Long.bitCount(b & (1L << bitIndex));
- }
-
- public void setIndexDefined(int index) {
- while (index >= getValidityBufferValueCapacity()) {
- /* realloc the inner buffers if needed */
- reallocValidityBuffer();
+ @Override
+ public void close() {
+ final Collection<FieldVector> vectors = getChildren();
+ for (final FieldVector v : vectors) {
+ v.close();
}
- BitVectorHelper.setValidityBitToOne(validityBuffer, index);
- }
+ vectors.clear();
- public void setNull(int index) {
- while (index >= getValidityBufferValueCapacity()) {
- /* realloc the inner buffers if needed */
- reallocValidityBuffer();
- }
- BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+ valueCount = 0;
+
+ super.close();
}
- @Override
- public void setValueCount(int valueCount) {
- assert valueCount >= 0;
- while (valueCount > getValidityBufferValueCapacity()) {
- /* realloc the inner buffers if needed */
- reallocValidityBuffer();
+ public void initializeChildrenFromFields(List<Field> children) {
+ for (Field field : children) {
+ FieldVector vector = (FieldVector) this.add(field.getName(),
field.getFieldType());
+ vector.initializeChildrenFromFields(field.getChildren());
}
- super.setValueCount(valueCount);
- this.valueCount = valueCount;
}
- public void reset() {
- valueCount = 0;
+ public List<FieldVector> getChildrenFromFields() {
+ return getChildren();
}
}
diff --git
a/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableMapVector.java
b/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableMapVector.java
deleted file mode 100644
index 1cd7b70d7..000000000
---
a/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableMapVector.java
+++ /dev/null
@@ -1,342 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.arrow.vector.complex;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-import javax.annotation.Nullable;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Ordering;
-import com.google.common.primitives.Ints;
-
-import io.netty.buffer.ArrowBuf;
-
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.vector.*;
-import org.apache.arrow.vector.complex.impl.SingleMapReaderImpl;
-import org.apache.arrow.vector.complex.reader.FieldReader;
-import org.apache.arrow.vector.holders.ComplexHolder;
-import org.apache.arrow.vector.types.Types.MinorType;
-import org.apache.arrow.vector.types.pojo.ArrowType;
-import org.apache.arrow.vector.types.pojo.Field;
-import org.apache.arrow.vector.types.pojo.FieldType;
-import org.apache.arrow.vector.util.CallBack;
-import org.apache.arrow.vector.util.JsonStringHashMap;
-import org.apache.arrow.vector.util.TransferPair;
-
-public class NonNullableMapVector extends AbstractMapVector {
- //private static final org.slf4j.Logger logger =
org.slf4j.LoggerFactory.getLogger(NonNullableMapVector.class);
-
- public static NonNullableMapVector empty(String name, BufferAllocator
allocator) {
- FieldType fieldType = new FieldType(false, ArrowType.Struct.INSTANCE,
null, null);
- return new NonNullableMapVector(name, allocator, fieldType, null);
- }
-
- private final SingleMapReaderImpl reader = new SingleMapReaderImpl(this);
- protected final FieldType fieldType;
- public int valueCount;
-
- // deprecated, use FieldType or static constructor instead
- @Deprecated
- public NonNullableMapVector(String name, BufferAllocator allocator, CallBack
callBack) {
- this(name, allocator, new FieldType(false, ArrowType.Struct.INSTANCE,
null, null), callBack);
- }
-
- public NonNullableMapVector(String name, BufferAllocator allocator,
FieldType fieldType, CallBack callBack) {
- super(name, allocator, callBack);
- this.fieldType = checkNotNull(fieldType);
- this.valueCount = 0;
- }
-
- @Override
- public FieldReader getReader() {
- return reader;
- }
-
- transient private MapTransferPair ephPair;
-
- public void copyFromSafe(int fromIndex, int thisIndex, NonNullableMapVector
from) {
- if (ephPair == null || ephPair.from != from) {
- ephPair = (MapTransferPair) from.makeTransferPair(this);
- }
- ephPair.copyValueSafe(fromIndex, thisIndex);
- }
-
- @Override
- protected boolean supportsDirectRead() {
- return true;
- }
-
- public Iterator<String> fieldNameIterator() {
- return getChildFieldNames().iterator();
- }
-
- @Override
- public void setInitialCapacity(int numRecords) {
- for (final ValueVector v : (Iterable<ValueVector>) this) {
- v.setInitialCapacity(numRecords);
- }
- }
-
- @Override
- public int getBufferSize() {
- if (valueCount == 0 || size() == 0) {
- return 0;
- }
- long buffer = 0;
- for (final ValueVector v : (Iterable<ValueVector>) this) {
- buffer += v.getBufferSize();
- }
-
- return (int) buffer;
- }
-
- @Override
- public int getBufferSizeFor(final int valueCount) {
- if (valueCount == 0) {
- return 0;
- }
-
- long bufferSize = 0;
- for (final ValueVector v : (Iterable<ValueVector>) this) {
- bufferSize += v.getBufferSizeFor(valueCount);
- }
-
- return (int) bufferSize;
- }
-
- @Override
- public ArrowBuf getValidityBuffer() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public ArrowBuf getDataBuffer() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public ArrowBuf getOffsetBuffer() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public TransferPair getTransferPair(BufferAllocator allocator) {
- return getTransferPair(name, allocator, null);
- }
-
- @Override
- public TransferPair getTransferPair(String ref, BufferAllocator allocator,
CallBack callBack) {
- return new MapTransferPair(this, new NonNullableMapVector(name, allocator,
fieldType, callBack), false);
- }
-
- @Override
- public TransferPair makeTransferPair(ValueVector to) {
- return new MapTransferPair(this, (NonNullableMapVector) to);
- }
-
- @Override
- public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
- return new MapTransferPair(this, new NonNullableMapVector(ref, allocator,
fieldType, callBack), false);
- }
-
- protected static class MapTransferPair implements TransferPair {
- private final TransferPair[] pairs;
- private final NonNullableMapVector from;
- private final NonNullableMapVector to;
-
- public MapTransferPair(NonNullableMapVector from, NonNullableMapVector to)
{
- this(from, to, true);
- }
-
- protected MapTransferPair(NonNullableMapVector from, NonNullableMapVector
to, boolean allocate) {
- this.from = from;
- this.to = to;
- this.pairs = new TransferPair[from.size()];
- this.to.ephPair = null;
-
- int i = 0;
- FieldVector vector;
- for (String child : from.getChildFieldNames()) {
- int preSize = to.size();
- vector = from.getChild(child);
- if (vector == null) {
- continue;
- }
- //DRILL-1872: we add the child fields for the vector, looking up the
field by name. For a map vector,
- // the child fields may be nested fields of the top level child. For
example if the structure
- // of a child field is oa.oab.oabc then we add oa, then add oab to oa
then oabc to oab.
- // But the children member of a Materialized field is a HashSet. If
the fields are added in the
- // children HashSet, and the hashCode of the Materialized field
includes the hash code of the
- // children, the hashCode value of oa changes *after* the field has
been added to the HashSet.
- // (This is similar to what happens in ScanBatch where the children
cannot be added till they are
- // read). To take care of this, we ensure that the hashCode of the
MaterializedField does not
- // include the hashCode of the children but is based only on
MaterializedField$key.
- final FieldVector newVector = to.addOrGet(child,
vector.getField().getFieldType(), vector.getClass());
- if (allocate && to.size() != preSize) {
- newVector.allocateNew();
- }
- pairs[i++] = vector.makeTransferPair(newVector);
- }
- }
-
- @Override
- public void transfer() {
- for (final TransferPair p : pairs) {
- p.transfer();
- }
- to.valueCount = from.valueCount;
- from.clear();
- }
-
- @Override
- public ValueVector getTo() {
- return to;
- }
-
- @Override
- public void copyValueSafe(int from, int to) {
- for (TransferPair p : pairs) {
- p.copyValueSafe(from, to);
- }
- }
-
- @Override
- public void splitAndTransfer(int startIndex, int length) {
- for (TransferPair p : pairs) {
- p.splitAndTransfer(startIndex, length);
- }
- to.setValueCount(length);
- }
- }
-
- @Override
- public int getValueCapacity() {
- if (size() == 0) {
- return 0;
- }
-
- final Ordering<ValueVector> natural = new Ordering<ValueVector>() {
- @Override
- public int compare(@Nullable ValueVector left, @Nullable ValueVector
right) {
- return Ints.compare(
- checkNotNull(left).getValueCapacity(),
- checkNotNull(right).getValueCapacity()
- );
- }
- };
-
- return natural.min(getChildren()).getValueCapacity();
- }
-
- @Override
- public Object getObject(int index) {
- Map<String, Object> vv = new JsonStringHashMap<>();
- for (String child : getChildFieldNames()) {
- ValueVector v = getChild(child);
- if (v != null && index < v.getValueCount()) {
- Object value = v.getObject(index);
- if (value != null) {
- vv.put(child, value);
- }
- }
- }
- return vv;
- }
-
- @Override
- public boolean isNull(int index) { return false; }
- @Override
- public int getNullCount() { return 0; }
-
- public void get(int index, ComplexHolder holder) {
- reader.setPosition(index);
- holder.reader = reader;
- }
-
- @Override
- public int getValueCount() {
- return valueCount;
- }
-
- public ValueVector getVectorById(int id) {
- return getChildByOrdinal(id);
-}
-
- @Override
- public void setValueCount(int valueCount) {
- for (final ValueVector v : getChildren()) {
- v.setValueCount(valueCount);
- }
- NonNullableMapVector.this.valueCount = valueCount;
- }
-
- @Override
- public void clear() {
- for (final ValueVector v : getChildren()) {
- v.clear();
- }
- valueCount = 0;
- }
-
- @Override
- public Field getField() {
- List<Field> children = new ArrayList<>();
- for (ValueVector child : getChildren()) {
- children.add(child.getField());
- }
- return new Field(name, fieldType, children);
- }
-
- @Override
- public MinorType getMinorType() {
- return MinorType.MAP;
- }
-
- @Override
- public void close() {
- final Collection<FieldVector> vectors = getChildren();
- for (final FieldVector v : vectors) {
- v.close();
- }
- vectors.clear();
-
- valueCount = 0;
-
- super.close();
- }
-
- public void initializeChildrenFromFields(List<Field> children) {
- for (Field field : children) {
- FieldVector vector = (FieldVector) this.add(field.getName(),
field.getFieldType());
- vector.initializeChildrenFromFields(field.getChildren());
- }
- }
-
- public List<FieldVector> getChildrenFromFields() {
- return getChildren();
- }
-}
diff --git
a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
new file mode 100644
index 000000000..d887b7306
--- /dev/null
+++
b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
@@ -0,0 +1,500 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector.complex;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import com.google.common.collect.ObjectArrays;
+
+import io.netty.buffer.ArrowBuf;
+import org.apache.arrow.memory.BaseAllocator;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.*;
+import org.apache.arrow.vector.complex.impl.NullableMapReaderImpl;
+import org.apache.arrow.vector.complex.impl.NullableMapWriter;
+import org.apache.arrow.vector.holders.ComplexHolder;
+import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.ArrowType.Struct;
+import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.util.CallBack;
+import org.apache.arrow.vector.util.OversizedAllocationException;
+import org.apache.arrow.vector.util.TransferPair;
+
+public class NullableMapVector extends MapVector implements FieldVector {
+
+ public static NullableMapVector empty(String name, BufferAllocator
allocator) {
+ FieldType fieldType = FieldType.nullable(Struct.INSTANCE);
+ return new NullableMapVector(name, allocator, fieldType, null);
+ }
+
+ private final NullableMapReaderImpl reader = new NullableMapReaderImpl(this);
+ private final NullableMapWriter writer = new NullableMapWriter(this);
+
+ protected ArrowBuf validityBuffer;
+ private int validityAllocationSizeInBytes;
+
+ // deprecated, use FieldType or static constructor instead
+ @Deprecated
+ public NullableMapVector(String name, BufferAllocator allocator, CallBack
callBack) {
+ this(name, allocator, FieldType.nullable(ArrowType.Struct.INSTANCE),
callBack);
+ }
+
+ // deprecated, use FieldType or static constructor instead
+ @Deprecated
+ public NullableMapVector(String name, BufferAllocator allocator,
DictionaryEncoding dictionary, CallBack callBack) {
+ this(name, allocator, new FieldType(true, ArrowType.Struct.INSTANCE,
dictionary, null), callBack);
+ }
+
+ public NullableMapVector(String name, BufferAllocator allocator, FieldType
fieldType, CallBack callBack) {
+ super(name, checkNotNull(allocator), fieldType, callBack);
+ this.validityBuffer = allocator.getEmpty();
+ this.validityAllocationSizeInBytes =
BitVectorHelper.getValidityBufferSize(BaseValueVector.INITIAL_VALUE_ALLOCATION);
+ }
+
+ @Override
+ public Field getField() {
+ Field f = super.getField();
+ FieldType type = new FieldType(true, f.getType(),
f.getFieldType().getDictionary(), f.getFieldType().getMetadata());
+ return new Field(f.getName(), type, f.getChildren());
+ }
+
+ @Override
+ public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf>
ownBuffers) {
+ if (ownBuffers.size() != 1) {
+ throw new IllegalArgumentException("Illegal buffer count, expected " + 1
+ ", got: " + ownBuffers.size());
+ }
+
+ ArrowBuf bitBuffer = ownBuffers.get(0);
+
+ validityBuffer.release();
+ validityBuffer = BitVectorHelper.loadValidityBuffer(fieldNode, bitBuffer,
allocator);
+ valueCount = fieldNode.getLength();
+ validityAllocationSizeInBytes = validityBuffer.capacity();
+ }
+
+ @Override
+ public List<ArrowBuf> getFieldBuffers() {
+ List<ArrowBuf> result = new ArrayList<>(1);
+ setReaderAndWriterIndex();
+ result.add(validityBuffer);
+
+ return result;
+ }
+
+ private void setReaderAndWriterIndex() {
+ validityBuffer.readerIndex(0);
+
validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSize(valueCount));
+ }
+
+ @Override
+ @Deprecated
+ public List<BufferBacked> getFieldInnerVectors() {
+ throw new UnsupportedOperationException("There are no inner vectors. Use
getFieldBuffers");
+ }
+
+ @Override
+ public NullableMapReaderImpl getReader() {
+ return reader;
+ }
+
+ public NullableMapWriter getWriter() {
+ return writer;
+ }
+
+ @Override
+ public TransferPair getTransferPair(BufferAllocator allocator) {
+ return new NullableMapTransferPair(this, new NullableMapVector(name,
allocator, fieldType, null), false);
+ }
+
+ @Override
+ public TransferPair makeTransferPair(ValueVector to) {
+ return new NullableMapTransferPair(this, (NullableMapVector) to, true);
+ }
+
+ @Override
+ public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+ return new NullableMapTransferPair(this, new NullableMapVector(ref,
allocator, fieldType, null), false);
+ }
+
+ @Override
+ public TransferPair getTransferPair(String ref, BufferAllocator allocator,
CallBack callBack) {
+ return new NullableMapTransferPair(this, new NullableMapVector(ref,
allocator, fieldType, callBack), false);
+ }
+
+ protected class NullableMapTransferPair extends MapTransferPair {
+
+ private NullableMapVector target;
+
+ protected NullableMapTransferPair(NullableMapVector from,
NullableMapVector to, boolean allocate) {
+ super(from, to, allocate);
+ this.target = to;
+ }
+
+ @Override
+ public void transfer() {
+ target.clear();
+ target.validityBuffer =
validityBuffer.transferOwnership(target.allocator).buffer;
+ super.transfer();
+ clear();
+ }
+
+ @Override
+ public void copyValueSafe(int fromIndex, int toIndex) {
+ while (toIndex >= target.getValidityBufferValueCapacity()) {
+ target.reallocValidityBuffer();
+ }
+ BitVectorHelper.setValidityBit(target.validityBuffer, toIndex,
isSet(fromIndex));
+ super.copyValueSafe(fromIndex, toIndex);
+ }
+
+ @Override
+ public void splitAndTransfer(int startIndex, int length) {
+ target.clear();
+ splitAndTransferValidityBuffer(startIndex, length, target);
+ super.splitAndTransfer(startIndex, length);
+ }
+ }
+
+ /*
+ * transfer the validity.
+ */
+ private void splitAndTransferValidityBuffer(int startIndex, int length,
NullableMapVector target) {
+ assert startIndex + length <= valueCount;
+ int firstByteSource = BitVectorHelper.byteIndex(startIndex);
+ int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1);
+ int byteSizeTarget = BitVectorHelper.getValidityBufferSize(length);
+ int offset = startIndex % 8;
+
+ if (length > 0) {
+ if (offset == 0) {
+ // slice
+ if (target.validityBuffer != null) {
+ target.validityBuffer.release();
+ }
+ target.validityBuffer = validityBuffer.slice(firstByteSource,
byteSizeTarget);
+ target.validityBuffer.retain(1);
+ } else {
+ /* Copy data
+ * When the first bit starts from the middle of a byte (offset != 0),
+ * copy data from src BitVector.
+ * Each byte in the target is composed by a part in i-th byte,
+ * another part in (i+1)-th byte.
+ */
+ target.allocateValidityBuffer(byteSizeTarget);
+
+ for (int i = 0; i < byteSizeTarget - 1; i++) {
+ byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer,
firstByteSource + i, offset);
+ byte b2 = BitVectorHelper.getBitsFromNextByte(validityBuffer,
firstByteSource + i + 1, offset);
+
+ target.validityBuffer.setByte(i, (b1 + b2));
+ }
+
+ /* Copying the last piece is done in the following manner:
+ * if the source vector has 1 or more bytes remaining, we copy
+ * the last piece as a byte formed by shifting data
+ * from the current byte and the next byte.
+ *
+ * if the source vector has no more bytes remaining
+ * (we are at the last byte), we copy the last piece as a byte
+ * by shifting data from the current byte.
+ */
+ if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) {
+ byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer,
+ firstByteSource + byteSizeTarget - 1, offset);
+ byte b2 = BitVectorHelper.getBitsFromNextByte(validityBuffer,
+ firstByteSource + byteSizeTarget, offset);
+
+ target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2);
+ } else {
+ byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer,
+ firstByteSource + byteSizeTarget - 1, offset);
+ target.validityBuffer.setByte(byteSizeTarget - 1, b1);
+ }
+ }
+ }
+ }
+
+ /**
+ * Get the value capacity of the internal validity buffer.
+ * @return number of elements that validity buffer can hold
+ */
+ private int getValidityBufferValueCapacity() {
+ return (int) (validityBuffer.capacity() * 8L);
+ }
+
+ /**
+ * Get the current value capacity for the vector
+ * @return number of elements that vector can hold.
+ */
+ @Override
+ public int getValueCapacity() {
+ return Math.min(getValidityBufferValueCapacity(),
+ super.getValueCapacity());
+ }
+
+ /**
+ * Return the underlying buffers associated with this vector. Note that this
doesn't
+ * impact the reference counts for this buffer so it only should be used for
in-context
+ * access. Also note that this buffer changes regularly thus
+ * external classes shouldn't hold a reference to it (unless they change it).
+ *
+ * @param clear Whether to clear vector before returning; the buffers will
still be refcounted
+ * but the returned array will be the only reference to them
+ * @return The underlying {@link io.netty.buffer.ArrowBuf buffers} that is
used by this
+ * vector instance.
+ */
+ @Override
+ public ArrowBuf[] getBuffers(boolean clear) {
+ setReaderAndWriterIndex();
+ final ArrowBuf[] buffers;
+ if (getBufferSize() == 0) {
+ buffers = new ArrowBuf[0];
+ } else {
+ buffers = ObjectArrays.concat(new ArrowBuf[]{validityBuffer},
super.getBuffers(false),
+ ArrowBuf.class);
+ }
+ if (clear) {
+ for (ArrowBuf buffer : buffers) {
+ buffer.retain();
+ }
+ clear();
+ }
+
+ return buffers;
+ }
+
+ /**
+ * Close the vector and release the associated buffers.
+ */
+ @Override
+ public void close() {
+ clearValidityBuffer();
+ super.close();
+ }
+
+ /**
+ * Same as {@link #close()}
+ */
+ @Override
+ public void clear() {
+ clearValidityBuffer();
+ super.clear();
+ }
+
+ /**
+ * Release the validity buffer
+ */
+ private void clearValidityBuffer() {
+ validityBuffer.release();
+ validityBuffer = allocator.getEmpty();
+ }
+
+ /**
+ * Get the size (number of bytes) of underlying buffers used by this
+ * vector
+ * @return size of underlying buffers.
+ */
+ @Override
+ public int getBufferSize() {
+ if (valueCount == 0) {
+ return 0;
+ }
+ return super.getBufferSize() +
+ BitVectorHelper.getValidityBufferSize(valueCount);
+ }
+
+ /**
+ * Get the potential buffer size for a particular number of records.
+ * @param valueCount desired number of elements in the vector
+ * @return estimated size of underlying buffers if the vector holds
+ * a given number of elements
+ */
+ @Override
+ public int getBufferSizeFor(final int valueCount) {
+ if (valueCount == 0) {
+ return 0;
+ }
+ return super.getBufferSizeFor(valueCount)
+ + BitVectorHelper.getValidityBufferSize(valueCount);
+ }
+
+ @Override
+ public void setInitialCapacity(int numRecords) {
+ validityAllocationSizeInBytes =
BitVectorHelper.getValidityBufferSize(numRecords);
+ super.setInitialCapacity(numRecords);
+ }
+
+ @Override
+ public boolean allocateNewSafe() {
+ /* Boolean to keep track if all the memory allocations were successful
+ * Used in the case of composite vectors when we need to allocate multiple
+ * buffers for multiple vectors. If one of the allocations failed we need
to
+ * clear all the memory that we allocated
+ */
+ boolean success = false;
+ try {
+ clear();
+ allocateValidityBuffer(validityAllocationSizeInBytes);
+ success = super.allocateNewSafe();
+ } finally {
+ if (!success) {
+ clear();
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private void allocateValidityBuffer(final long size) {
+ final int curSize = (int) size;
+ validityBuffer = allocator.buffer(curSize);
+ validityBuffer.readerIndex(0);
+ validityAllocationSizeInBytes = curSize;
+ validityBuffer.setZero(0, validityBuffer.capacity());
+ }
+
+ @Override
+ public void reAlloc() {
+ /* reallocate the validity buffer */
+ reallocValidityBuffer();
+ super.reAlloc();
+ }
+
+ private void reallocValidityBuffer() {
+ final int currentBufferCapacity = validityBuffer.capacity();
+ long baseSize = validityAllocationSizeInBytes;
+
+ if (baseSize < (long) currentBufferCapacity) {
+ baseSize = (long) currentBufferCapacity;
+ }
+
+ long newAllocationSize = baseSize * 2L;
+ newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize);
+
+ if (newAllocationSize > BaseValueVector.MAX_ALLOCATION_SIZE) {
+ throw new OversizedAllocationException("Unable to expand the buffer");
+ }
+
+ final ArrowBuf newBuf = allocator.buffer((int) newAllocationSize);
+ newBuf.setZero(0, newBuf.capacity());
+ newBuf.setBytes(0, validityBuffer, 0, currentBufferCapacity);
+ validityBuffer.release(1);
+ validityBuffer = newBuf;
+ validityAllocationSizeInBytes = (int) newAllocationSize;
+ }
+
+ @Override
+ public long getValidityBufferAddress() {
+ return validityBuffer.memoryAddress();
+ }
+
+ @Override
+ public long getDataBufferAddress() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long getOffsetBufferAddress() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public ArrowBuf getValidityBuffer() {
+ return validityBuffer;
+ }
+
+ @Override
+ public ArrowBuf getDataBuffer() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public ArrowBuf getOffsetBuffer() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Object getObject(int index) {
+ if (isSet(index) == 0) {
+ return null;
+ } else {
+ return super.getObject(index);
+ }
+ }
+
+ @Override
+ public void get(int index, ComplexHolder holder) {
+ holder.isSet = isSet(index);
+ super.get(index, holder);
+ }
+
+ public int getNullCount() {
+ return BitVectorHelper.getNullCount(validityBuffer, valueCount);
+ }
+
+ public boolean isNull(int index) {
+ return isSet(index) == 0;
+ }
+
+ public int isSet(int index) {
+ final int byteIndex = index >> 3;
+ final byte b = validityBuffer.getByte(byteIndex);
+ final int bitIndex = index & 7;
+ return Long.bitCount(b & (1L << bitIndex));
+ }
+
+ public void setIndexDefined(int index) {
+ while (index >= getValidityBufferValueCapacity()) {
+ /* realloc the inner buffers if needed */
+ reallocValidityBuffer();
+ }
+ BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+ }
+
+ public void setNull(int index) {
+ while (index >= getValidityBufferValueCapacity()) {
+ /* realloc the inner buffers if needed */
+ reallocValidityBuffer();
+ }
+ BitVectorHelper.setValidityBit(validityBuffer, index, 0);
+ }
+
+ @Override
+ public void setValueCount(int valueCount) {
+ assert valueCount >= 0;
+ while (valueCount > getValidityBufferValueCapacity()) {
+ /* realloc the inner buffers if needed */
+ reallocValidityBuffer();
+ }
+ super.setValueCount(valueCount);
+ this.valueCount = valueCount;
+ }
+
+ public void reset() {
+ valueCount = 0;
+ }
+}
diff --git
a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java
b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java
index 970b90ef5..a6960238b 100644
---
a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java
+++
b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java
@@ -20,7 +20,7 @@
import org.apache.arrow.vector.complex.ListVector;
import org.apache.arrow.vector.complex.MapVector;
-import org.apache.arrow.vector.complex.MapVector;
+import org.apache.arrow.vector.complex.NullableMapVector;
import org.apache.arrow.vector.complex.StateTool;
import org.apache.arrow.vector.complex.writer.BaseWriter.ComplexWriter;
import org.apache.arrow.vector.types.pojo.Field;
@@ -131,7 +131,7 @@ public MapWriter directMap() {
switch (mode) {
case INIT:
- mapRoot = nullableMapWriterFactory.build((MapVector) container);
+ mapRoot = nullableMapWriterFactory.build((NullableMapVector)
container);
mapRoot.setPosition(idx());
mode = Mode.MAP;
break;
@@ -152,7 +152,7 @@ public MapWriter rootAsMap() {
case INIT:
// TODO allow dictionaries in complex types
- MapVector map = container.addOrGetMap(name);
+ NullableMapVector map = container.addOrGetMap(name);
mapRoot = nullableMapWriterFactory.build(map);
mapRoot.setPosition(idx());
mode = Mode.MAP;
diff --git
a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java
b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java
index acf155af7..06b0f4d43 100644
---
a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java
+++
b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java
@@ -20,17 +20,17 @@
package org.apache.arrow.vector.complex.impl;
import org.apache.arrow.vector.complex.MapVector;
-import org.apache.arrow.vector.complex.MapVector;
+import org.apache.arrow.vector.complex.NullableMapVector;
import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
import org.apache.arrow.vector.types.pojo.Field;
public class NullableMapReaderImpl extends SingleMapReaderImpl {
- private MapVector nullableMapVector;
+ private NullableMapVector nullableMapVector;
public NullableMapReaderImpl(MapVector vector) {
- super((MapVector) vector);
- this.nullableMapVector = (MapVector) vector;
+ super((NullableMapVector) vector);
+ this.nullableMapVector = (NullableMapVector) vector;
}
@Override
diff --git
a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapWriterFactory.java
b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapWriterFactory.java
index 06dbf784b..d2dcb2374 100644
---
a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapWriterFactory.java
+++
b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapWriterFactory.java
@@ -18,7 +18,7 @@
package org.apache.arrow.vector.complex.impl;
-import org.apache.arrow.vector.complex.MapVector;
+import org.apache.arrow.vector.complex.NullableMapVector;
public class NullableMapWriterFactory {
private final boolean caseSensitive;
@@ -29,7 +29,7 @@ public NullableMapWriterFactory(boolean caseSensitive) {
this.caseSensitive = caseSensitive;
}
- public NullableMapWriter build(MapVector container) {
+ public NullableMapWriter build(NullableMapVector container) {
return this.caseSensitive ? new NullableCaseSensitiveMapWriter(container)
: new NullableMapWriter(container);
}
diff --git
a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
index 9abd38d20..5bd439cac 100644
---
a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
+++
b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
@@ -23,7 +23,7 @@
import org.apache.arrow.vector.ZeroVector;
import org.apache.arrow.vector.complex.AbstractMapVector;
import org.apache.arrow.vector.complex.ListVector;
-import org.apache.arrow.vector.complex.MapVector;
+import org.apache.arrow.vector.complex.NullableMapVector;
import org.apache.arrow.vector.complex.UnionVector;
import org.apache.arrow.vector.complex.writer.FieldWriter;
import org.apache.arrow.vector.types.Types.MinorType;
@@ -94,7 +94,7 @@ private void setWriter(ValueVector v) {
type = v.getMinorType();
switch (type) {
case MAP:
- writer = nullableMapWriterFactory.build((MapVector) vector);
+ writer = nullableMapWriterFactory.build((NullableMapVector) vector);
break;
case LIST:
writer = new UnionListWriter((ListVector) vector,
nullableMapWriterFactory);
diff --git
a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java
b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java
index 0341b622e..c77ca4e8f 100644
---
a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java
+++
b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java
@@ -24,7 +24,6 @@
import org.apache.arrow.vector.ValueVector;
import org.apache.arrow.vector.complex.MapVector;
-import org.apache.arrow.vector.complex.NonNullableMapVector;
import org.apache.arrow.vector.complex.reader.FieldReader;
import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
import org.apache.arrow.vector.types.Types.MinorType;
@@ -35,10 +34,10 @@
@SuppressWarnings("unused")
public class SingleMapReaderImpl extends AbstractFieldReader {
- private final NonNullableMapVector vector;
+ private final MapVector vector;
private final Map<String, FieldReader> fields = Maps.newHashMap();
- public SingleMapReaderImpl(NonNullableMapVector vector) {
+ public SingleMapReaderImpl(MapVector vector) {
this.vector = vector;
}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
index 7834845aa..3c5fd81d5 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
@@ -58,7 +58,7 @@
import org.apache.arrow.vector.ZeroVector;
import org.apache.arrow.vector.complex.FixedSizeListVector;
import org.apache.arrow.vector.complex.ListVector;
-import org.apache.arrow.vector.complex.MapVector;
+import org.apache.arrow.vector.complex.NullableMapVector;
import org.apache.arrow.vector.complex.UnionVector;
import org.apache.arrow.vector.complex.impl.BigIntWriterImpl;
import org.apache.arrow.vector.complex.impl.BitWriterImpl;
@@ -131,12 +131,12 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
MAP(Struct.INSTANCE) {
@Override
public FieldVector getNewVector(String name, FieldType fieldType,
BufferAllocator allocator, CallBack schemaChangeCallback) {
- return new MapVector(name, allocator, fieldType, schemaChangeCallback);
+ return new NullableMapVector(name, allocator, fieldType,
schemaChangeCallback);
}
@Override
public FieldWriter getNewFieldWriter(ValueVector vector) {
- return new NullableMapWriter((MapVector) vector);
+ return new NullableMapWriter((NullableMapVector) vector);
}
},
TINYINT(new Int(8, true)) {
diff --git
a/java/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java
b/java/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java
index e5682955b..357df96aa 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java
@@ -22,7 +22,7 @@
import java.util.Map;
import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.vector.complex.MapVector;
+import org.apache.arrow.vector.complex.NullableMapVector;
import org.apache.arrow.vector.types.pojo.ArrowType.Struct;
import org.apache.arrow.vector.types.pojo.FieldType;
import org.junit.After;
@@ -50,7 +50,7 @@ public void testFieldMetadata() throws Exception {
Map<String, String> metadata = new HashMap<>();
metadata.put("k1", "v1");
FieldType type = new FieldType(true, Struct.INSTANCE, null, metadata);
- try (MapVector vector = new MapVector("map", allocator, type, null)) {
+ try (NullableMapVector vector = new NullableMapVector("map", allocator,
type, null)) {
Assert.assertEquals(vector.getField().getMetadata(), type.getMetadata());
}
}
diff --git
a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java
b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java
index 1b13c2ed9..f8edf8904 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java
@@ -26,7 +26,7 @@
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.complex.ListVector;
-import org.apache.arrow.vector.complex.MapVector;
+import org.apache.arrow.vector.complex.NullableMapVector;
import org.apache.arrow.vector.types.Types.MinorType;
import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.FieldType;
@@ -121,7 +121,7 @@ public void testListType() {
@Test
public void testMapType() {
- try (final MapVector vector = MapVector.empty("", allocator)) {
+ try (final NullableMapVector vector = NullableMapVector.empty("",
allocator)) {
vector.addOrGet("", FieldType.nullable(MinorType.INT.getType()),
IntVector.class);
vector.setInitialCapacity(512);
diff --git
a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
index 38b78424d..b0d6cf555 100644
---
a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
+++
b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
@@ -25,7 +25,7 @@
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.vector.DirtyRootAllocator;
import org.apache.arrow.vector.complex.MapVector;
-import org.apache.arrow.vector.complex.MapVector;
+import org.apache.arrow.vector.complex.NullableMapVector;
import org.apache.arrow.vector.complex.UnionVector;
import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
import org.apache.arrow.vector.types.pojo.ArrowType;
@@ -55,7 +55,7 @@ public void terminate() throws Exception {
public void testPromoteToUnion() throws Exception {
try (final MapVector container = MapVector.empty(EMPTY_SCHEMA_PATH,
allocator);
- final MapVector v = container.addOrGetMap("test");
+ final NullableMapVector v = container.addOrGetMap("test");
final PromotableWriter writer = new PromotableWriter(v, container)) {
container.allocateNew();
diff --git
a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
index bd8489eb2..29d39aabe 100644
---
a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
+++
b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
@@ -34,7 +34,7 @@
import org.apache.arrow.vector.IntVector;
import org.apache.arrow.vector.complex.ListVector;
import org.apache.arrow.vector.complex.MapVector;
-import org.apache.arrow.vector.complex.NonNullableMapVector;
+import org.apache.arrow.vector.complex.NullableMapVector;
import org.apache.arrow.vector.complex.UnionVector;
import org.apache.arrow.vector.complex.impl.ComplexWriterImpl;
import org.apache.arrow.vector.complex.impl.SingleMapReaderImpl;
@@ -830,7 +830,7 @@ public void complexCopierWithList() {
rootWriter.end();
writer.setValueCount(1);
- MapVector mapVector = (MapVector) parent.getChild("root");
+ NullableMapVector mapVector = (NullableMapVector) parent.getChild("root");
TransferPair tp = mapVector.getTransferPair(allocator);
tp.splitAndTransfer(0, 1);
MapVector toMapVector = (MapVector) tp.getTo();
@@ -849,7 +849,7 @@ public void testSingleMapWriter1() {
/* initialize a SingleMapWriter with empty MapVector and then lazily
* create all vectors with expected initialCapacity.
*/
- NonNullableMapVector parent = NonNullableMapVector.empty("parent",
allocator);
+ MapVector parent = MapVector.empty("parent", allocator);
SingleMapWriter singleMapWriter = new SingleMapWriter(parent);
int initialCapacity = 1024;
diff --git
a/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java
b/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java
index 1a7f71c72..9eb55c9bb 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java
@@ -36,7 +36,7 @@
import org.apache.arrow.vector.VarCharVector;
import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.complex.ListVector;
-import org.apache.arrow.vector.complex.MapVector;
+import org.apache.arrow.vector.complex.NullableMapVector;
import org.apache.arrow.vector.complex.impl.ComplexWriterImpl;
import org.apache.arrow.vector.complex.impl.UnionListWriter;
import org.apache.arrow.vector.complex.reader.FieldReader;
@@ -94,7 +94,7 @@ public void tearDown() {
DateTimeZone.setDefault(defaultTimezone);
}
- protected void writeData(int count, MapVector parent) {
+ protected void writeData(int count, NullableMapVector parent) {
ComplexWriter writer = new ComplexWriterImpl("root", parent);
MapWriter rootWriter = writer.rootAsMap();
IntWriter intWriter = rootWriter.integer("int");
@@ -119,7 +119,7 @@ protected void validateContent(int count, VectorSchemaRoot
root) {
}
}
- protected void writeComplexData(int count, MapVector parent) {
+ protected void writeComplexData(int count, NullableMapVector parent) {
ArrowBuf varchar = allocator.buffer(3);
varchar.readerIndex(0);
varchar.setByte(0, 'a');
@@ -189,7 +189,7 @@ private LocalDateTime makeDateTimeFromCount(int i) {
return new LocalDateTime(2000 + i, 1 + i, 1 + i, i, i, i, i);
}
- protected void writeDateTimeData(int count, MapVector parent) {
+ protected void writeDateTimeData(int count, NullableMapVector parent) {
Assert.assertTrue(count < 100);
ComplexWriter writer = new ComplexWriterImpl("root", parent);
MapWriter rootWriter = writer.rootAsMap();
@@ -494,7 +494,7 @@ public void validateUnionData(int count, VectorSchemaRoot
root) {
}
}
- public void writeUnionData(int count, MapVector parent) {
+ public void writeUnionData(int count, NullableMapVector parent) {
ArrowBuf varchar = allocator.buffer(3);
varchar.readerIndex(0);
varchar.setByte(0, 'a');
@@ -537,7 +537,7 @@ public void writeUnionData(int count, MapVector parent) {
varchar.release();
}
- protected void writeVarBinaryData(int count, MapVector parent) {
+ protected void writeVarBinaryData(int count, NullableMapVector parent) {
Assert.assertTrue(count < 100);
ComplexWriter writer = new ComplexWriterImpl("root", parent);
MapWriter rootWriter = writer.rootAsMap();
diff --git
a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowFile.java
b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowFile.java
index 0cfc9ba91..055c34e70 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowFile.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowFile.java
@@ -42,8 +42,7 @@
import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.VectorUnloader;
import org.apache.arrow.vector.complex.FixedSizeListVector;
-import org.apache.arrow.vector.complex.MapVector;
-import org.apache.arrow.vector.complex.MapVector;
+import org.apache.arrow.vector.complex.NullableMapVector;
import
org.apache.arrow.vector.dictionary.DictionaryProvider.MapDictionaryProvider;
import org.apache.arrow.vector.ipc.message.ArrowBlock;
import org.apache.arrow.vector.ipc.message.ArrowBuffer;
@@ -70,7 +69,7 @@ public void testWrite() throws IOException {
int count = COUNT;
try (
BufferAllocator vectorAllocator =
allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
- MapVector parent = MapVector.empty("parent", vectorAllocator)) {
+ NullableMapVector parent = NullableMapVector.empty("parent",
vectorAllocator)) {
writeData(count, parent);
write(parent.getChild("root"), file, new ByteArrayOutputStream());
}
@@ -82,7 +81,7 @@ public void testWriteComplex() throws IOException {
int count = COUNT;
try (
BufferAllocator vectorAllocator =
allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
- MapVector parent = MapVector.empty("parent", vectorAllocator)) {
+ NullableMapVector parent = NullableMapVector.empty("parent",
vectorAllocator)) {
writeComplexData(count, parent);
FieldVector root = parent.getChild("root");
validateComplexContent(count, new VectorSchemaRoot(root));
@@ -98,7 +97,7 @@ public void testWriteRead() throws IOException {
// write
try (BufferAllocator originalVectorAllocator =
allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
- MapVector parent = MapVector.empty("parent",
originalVectorAllocator)) {
+ NullableMapVector parent = NullableMapVector.empty("parent",
originalVectorAllocator)) {
writeData(count, parent);
write(parent.getChild("root"), file, stream);
}
@@ -154,7 +153,7 @@ public void testWriteReadComplex() throws IOException {
// write
try (BufferAllocator originalVectorAllocator =
allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
- MapVector parent = MapVector.empty("parent",
originalVectorAllocator)) {
+ NullableMapVector parent = NullableMapVector.empty("parent",
originalVectorAllocator)) {
writeComplexData(count, parent);
write(parent.getChild("root"), file, stream);
}
@@ -195,7 +194,7 @@ public void testWriteReadMultipleRBs() throws IOException {
// write
try (BufferAllocator originalVectorAllocator =
allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
- MapVector parent = MapVector.empty("parent", originalVectorAllocator);
+ NullableMapVector parent = NullableMapVector.empty("parent",
originalVectorAllocator);
FileOutputStream fileOutputStream = new FileOutputStream(file)) {
writeData(counts[0], parent);
VectorSchemaRoot root = new VectorSchemaRoot(parent.getChild("root"));
@@ -268,7 +267,7 @@ public void testWriteReadUnion() throws IOException {
// write
try (BufferAllocator vectorAllocator =
allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
- MapVector parent = MapVector.empty("parent", vectorAllocator)) {
+ NullableMapVector parent = NullableMapVector.empty("parent",
vectorAllocator)) {
writeUnionData(count, parent);
validateUnionData(count, new VectorSchemaRoot(parent.getChild("root")));
write(parent.getChild("root"), file, stream);
@@ -382,7 +381,7 @@ public void testWriteReadMetadata() throws IOException {
// write
try (BufferAllocator originalVectorAllocator =
allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
- MapVector vector = (MapVector)
field.createVector(originalVectorAllocator)) {
+ NullableMapVector vector = (NullableMapVector)
field.createVector(originalVectorAllocator)) {
vector.allocateNewSafe();
vector.setValueCount(0);
@@ -558,7 +557,7 @@ public void testWriteReadFixedSizeList() throws IOException
{
// write
try (BufferAllocator originalVectorAllocator =
allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
- MapVector parent = MapVector.empty("parent",
originalVectorAllocator)) {
+ NullableMapVector parent = NullableMapVector.empty("parent",
originalVectorAllocator)) {
FixedSizeListVector tuples = parent.addOrGet("float-pairs",
FieldType.nullable(new FixedSizeList(2)), FixedSizeListVector.class);
Float4Vector floats = (Float4Vector)
tuples.addOrGetVector(FieldType.nullable(MinorType.FLOAT4.getType())).getVector();
IntVector ints = parent.addOrGet("ints", FieldType.nullable(new Int(32,
true)), IntVector.class);
@@ -618,7 +617,7 @@ public void testWriteReadVarBin() throws IOException {
// write
try (
BufferAllocator vectorAllocator =
allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
- MapVector parent = MapVector.empty("parent", vectorAllocator)) {
+ NullableMapVector parent = NullableMapVector.empty("parent",
vectorAllocator)) {
writeVarBinaryData(count, parent);
VectorSchemaRoot root = new VectorSchemaRoot(parent.getChild("root"));
validateVarBinary(count, root);
diff --git
a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java
b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java
index 4f9093b8c..6a23a8438 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java
@@ -24,8 +24,7 @@
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.vector.FieldVector;
import org.apache.arrow.vector.VectorSchemaRoot;
-import org.apache.arrow.vector.complex.MapVector;
-import org.apache.arrow.vector.complex.MapVector;
+import org.apache.arrow.vector.complex.NullableMapVector;
import org.apache.arrow.vector.dictionary.DictionaryProvider;
import
org.apache.arrow.vector.dictionary.DictionaryProvider.MapDictionaryProvider;
import org.apache.arrow.vector.types.pojo.Schema;
@@ -45,7 +44,7 @@ public void testWriteRead() throws IOException {
// write
try (BufferAllocator originalVectorAllocator =
allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
- MapVector parent = MapVector.empty("parent",
originalVectorAllocator)) {
+ NullableMapVector parent = NullableMapVector.empty("parent",
originalVectorAllocator)) {
writeData(count, parent);
writeJSON(file, new VectorSchemaRoot(parent.getChild("root")), null);
}
@@ -73,7 +72,7 @@ public void testWriteReadComplexJSON() throws IOException {
// write
try (
BufferAllocator originalVectorAllocator =
allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
- MapVector parent = MapVector.empty("parent", originalVectorAllocator))
{
+ NullableMapVector parent = NullableMapVector.empty("parent",
originalVectorAllocator)) {
writeComplexData(count, parent);
writeJSON(file, new VectorSchemaRoot(parent.getChild("root")), null);
}
@@ -100,7 +99,7 @@ public void testWriteComplexJSON() throws IOException {
int count = COUNT;
try (
BufferAllocator vectorAllocator =
allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
- MapVector parent = MapVector.empty("parent", vectorAllocator)) {
+ NullableMapVector parent = NullableMapVector.empty("parent",
vectorAllocator)) {
writeComplexData(count, parent);
VectorSchemaRoot root = new VectorSchemaRoot(parent.getChild("root"));
validateComplexContent(root.getRowCount(), root);
@@ -122,7 +121,7 @@ public void testWriteReadUnionJSON() throws IOException {
int count = COUNT;
try (
BufferAllocator vectorAllocator =
allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
- MapVector parent = MapVector.empty("parent", vectorAllocator)) {
+ NullableMapVector parent = NullableMapVector.empty("parent",
vectorAllocator)) {
writeUnionData(count, parent);
printVectors(parent.getChildrenFromFields());
@@ -154,7 +153,7 @@ public void testWriteReadDateTimeJSON() throws IOException {
// write
try (
BufferAllocator vectorAllocator =
allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
- MapVector parent = MapVector.empty("parent", vectorAllocator)) {
+ NullableMapVector parent = NullableMapVector.empty("parent",
vectorAllocator)) {
writeDateTimeData(count, parent);
@@ -319,7 +318,7 @@ public void testWriteReadVarBinJSON() throws IOException {
// write
try (
BufferAllocator vectorAllocator =
allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
- MapVector parent = MapVector.empty("parent", vectorAllocator)) {
+ NullableMapVector parent = NullableMapVector.empty("parent",
vectorAllocator)) {
writeVarBinaryData(count, parent);
VectorSchemaRoot root = new VectorSchemaRoot(parent.getChild("root"));
validateVarBinary(count, root);
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
> [Java] Restore previous MapVector class names
> ---------------------------------------------
>
> Key: ARROW-1885
> URL: https://issues.apache.org/jira/browse/ARROW-1885
> Project: Apache Arrow
> Issue Type: Sub-task
> Components: Java - Vectors
> Reporter: Bryan Cutler
> Assignee: Bryan Cutler
> Labels: pull-request-available
> Fix For: 0.8.0
>
>
> Making changes to the MapVector class was not part of refactoring plan and
> should not have been done in ARROW-1710. This will restore class names to
> the previous state of {{NullableMapVector}} -> {{MapVector}} ->
> {{AbstractMapVector}}
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
