Repository: arrow Updated Branches: refs/heads/master 01114d831 -> b6033378c
ARROW-815 [Java] Exposing reAlloc for ValueVector Author: Emilio Lahr-Vivaz <[email protected]> Closes #534 from elahrvivaz/ARROW-815 and squashes the following commits: cf14944 [Emilio Lahr-Vivaz] unit test 45fa773 [Emilio Lahr-Vivaz] ARROW-815 [Java] Exposing reAlloc for ValueVector Project: http://git-wip-us.apache.org/repos/asf/arrow/repo Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/b6033378 Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/b6033378 Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/b6033378 Branch: refs/heads/master Commit: b6033378c2533ed7b396f111cc5aed10450907fb Parents: 01114d8 Author: Emilio Lahr-Vivaz <[email protected]> Authored: Fri Apr 14 16:37:25 2017 -0400 Committer: Wes McKinney <[email protected]> Committed: Fri Apr 14 16:37:25 2017 -0400 ---------------------------------------------------------------------- .../codegen/templates/NullableValueVectors.java | 6 + .../src/main/codegen/templates/UnionVector.java | 6 + .../org/apache/arrow/vector/ValueVector.java | 6 + .../org/apache/arrow/vector/ZeroVector.java | 3 + .../arrow/vector/complex/AbstractMapVector.java | 7 + .../vector/complex/BaseRepeatedValueVector.java | 14 +- .../apache/arrow/vector/complex/ListVector.java | 6 + .../arrow/vector/complex/NullableMapVector.java | 7 + .../apache/arrow/vector/TestVectorReAlloc.java | 144 +++++++++++++++++++ 9 files changed, 194 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/arrow/blob/b6033378/java/vector/src/main/codegen/templates/NullableValueVectors.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java index e5257ce..acee6cb 100644 --- a/java/vector/src/main/codegen/templates/NullableValueVectors.java +++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java @@ -242,6 +242,12 @@ public final class ${className} extends BaseDataValueVector implements <#if type return success; } + @Override + public void reAlloc() { + bits.reAlloc(); + values.reAlloc(); + } + <#if type.major == "VarLen"> @Override public void allocateNew(int totalBytes, int valueCount) { http://git-wip-us.apache.org/repos/asf/arrow/blob/b6033378/java/vector/src/main/codegen/templates/UnionVector.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java index 797b293..d70cbae 100644 --- a/java/vector/src/main/codegen/templates/UnionVector.java +++ b/java/vector/src/main/codegen/templates/UnionVector.java @@ -215,6 +215,12 @@ public class UnionVector implements FieldVector { } @Override + public void reAlloc() { + internalMap.reAlloc(); + typeVector.reAlloc(); + } + + @Override public void setInitialCapacity(int numRecords) { } http://git-wip-us.apache.org/repos/asf/arrow/blob/b6033378/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java index 8e35398..685b0be 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java @@ -68,6 +68,12 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> { */ boolean allocateNewSafe(); + /** + * Allocate new buffer with double capacity, and copy data into the new buffer. + * Replace vector's buffer with new buffer, and release old one + */ + void reAlloc(); + BufferAllocator getAllocator(); /** http://git-wip-us.apache.org/repos/asf/arrow/blob/b6033378/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java index 73f858e..e48788c 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java @@ -143,6 +143,9 @@ public class ZeroVector implements FieldVector { } @Override + public void reAlloc() {} + + @Override public BufferAllocator getAllocator() { throw new UnsupportedOperationException("Tried to get allocator from ZeroVector"); } http://git-wip-us.apache.org/repos/asf/arrow/blob/b6033378/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java index dc833ed..15e8a5b 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java @@ -81,6 +81,13 @@ public abstract class AbstractMapVector extends AbstractContainerVector { return true; } + @Override + public void reAlloc() { + for (final ValueVector v: vectors.values()) { + v.reAlloc(); + } + } + /** * Adds a new field with the given parameters or replaces the existing one and consequently returns the resultant * {@link org.apache.arrow.vector.ValueVector}. http://git-wip-us.apache.org/repos/asf/arrow/blob/b6033378/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java index 6b240c0..da221e3 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java @@ -20,6 +20,10 @@ package org.apache.arrow.vector.complex; import java.util.Collections; import java.util.Iterator; +import com.google.common.base.Preconditions; +import com.google.common.collect.ObjectArrays; + +import io.netty.buffer.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.AddOrGetResult; import org.apache.arrow.vector.BaseValueVector; @@ -31,11 +35,6 @@ import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.util.CallBack; import org.apache.arrow.vector.util.SchemaChangeRuntimeException; -import com.google.common.base.Preconditions; -import com.google.common.collect.ObjectArrays; - -import io.netty.buffer.ArrowBuf; - public abstract class BaseRepeatedValueVector extends BaseValueVector implements RepeatedValueVector { public final static FieldVector DEFAULT_DATA_VECTOR = ZeroVector.INSTANCE; @@ -79,6 +78,11 @@ public abstract class BaseRepeatedValueVector extends BaseValueVector implements return success; } + @Override + public void reAlloc() { + offsets.reAlloc(); + vector.reAlloc(); + } @Override public UInt4Vector getOffsetVector() { http://git-wip-us.apache.org/repos/asf/arrow/blob/b6033378/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java index 0461a8d..63235df 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java @@ -124,6 +124,12 @@ public class ListVector extends BaseRepeatedValueVector implements FieldVector { bits.allocateNewSafe(); } + @Override + public void reAlloc() { + super.reAlloc(); + bits.reAlloc(); + } + public void copyFromSafe(int inIndex, int outIndex, ListVector from) { copyFrom(inIndex, outIndex, from); } http://git-wip-us.apache.org/repos/asf/arrow/blob/b6033378/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java index 71fee67..647ab28 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java @@ -197,6 +197,13 @@ public class NullableMapVector extends MapVector implements FieldVector { bits.zeroVector(); return success; } + + @Override + public void reAlloc() { + bits.reAlloc(); + super.reAlloc(); + } + public final class Accessor extends MapVector.Accessor { final BitVector.Accessor bAccessor = bits.getAccessor(); http://git-wip-us.apache.org/repos/asf/arrow/blob/b6033378/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java ---------------------------------------------------------------------- diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java new file mode 100644 index 0000000..a7c35b6 --- /dev/null +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java @@ -0,0 +1,144 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.vector; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +import java.nio.charset.StandardCharsets; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.complex.ListVector; +import org.apache.arrow.vector.complex.NullableMapVector; +import org.apache.arrow.vector.types.Types.MinorType; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + + +public class TestVectorReAlloc { + + private BufferAllocator allocator; + + @Before + public void init() { + allocator = new RootAllocator(Long.MAX_VALUE); + } + + @After + public void terminate() throws Exception { + allocator.close(); + } + + @Test + public void testFixedType() { + try (final UInt4Vector vector = new UInt4Vector("", allocator)) { + final UInt4Vector.Mutator m = vector.getMutator(); + vector.setInitialCapacity(512); + vector.allocateNew(); + + assertEquals(512, vector.getValueCapacity()); + + try { + m.set(512, 0); + Assert.fail("Expected out of bounds exception"); + } catch (Exception e) { + // ok + } + + vector.reAlloc(); + assertEquals(1024, vector.getValueCapacity()); + + m.set(512, 100); + assertEquals(100, vector.getAccessor().get(512)); + } + } + + @Test + public void testNullableType() { + try (final NullableVarCharVector vector = new NullableVarCharVector("", allocator)) { + final NullableVarCharVector.Mutator m = vector.getMutator(); + vector.setInitialCapacity(512); + vector.allocateNew(); + + assertEquals(512, vector.getValueCapacity()); + + try { + m.set(512, "foo".getBytes(StandardCharsets.UTF_8)); + Assert.fail("Expected out of bounds exception"); + } catch (Exception e) { + // ok + } + + vector.reAlloc(); + assertEquals(1023, vector.getValueCapacity()); // note: size - 1 for some reason... + + m.set(512, "foo".getBytes(StandardCharsets.UTF_8)); + assertEquals("foo", new String(vector.getAccessor().get(512), StandardCharsets.UTF_8)); + } + } + + @Test + public void testListType() { + try (final ListVector vector = new ListVector("", allocator, null)) { + vector.addOrGetVector(FieldType.nullable(MinorType.INT.getType())); + + vector.setInitialCapacity(512); + vector.allocateNew(); + + assertEquals(1023, vector.getValueCapacity()); // TODO this doubles for some reason... + + try { + vector.getOffsetVector().getAccessor().get(2014); + Assert.fail("Expected out of bounds exception"); + } catch (Exception e) { + // ok + } + + vector.reAlloc(); + assertEquals(2047, vector.getValueCapacity()); // note: size - 1 + assertEquals(0, vector.getOffsetVector().getAccessor().get(2014)); + } + } + + @Test + public void testMapType() { + try (final NullableMapVector vector = new NullableMapVector("", allocator, null)) { + vector.addOrGet("", FieldType.nullable(MinorType.INT.getType()), NullableIntVector.class); + + vector.setInitialCapacity(512); + vector.allocateNew(); + + assertEquals(512, vector.getValueCapacity()); + + try { + vector.getAccessor().getObject(513); + Assert.fail("Expected out of bounds exception"); + } catch (Exception e) { + // ok + } + + vector.reAlloc(); + assertEquals(1024, vector.getValueCapacity()); + assertNull(vector.getAccessor().getObject(513)); + } + } +}
