Repository: arrow Updated Branches: refs/heads/master e0fb3698e -> e316b3f76
Make BaseValueVector#MAX_ALLOCATION_SIZE configurable This closes #65 Some of the tests are based on the assumption that the JVM can allocate at least 2GB of memory, which is not a common occurence (JVM usually defaults at 512MB). Current Travis CI VM only have 3GB of memory total, which would have make challenging to run some of the tests on them Add a system property to change BaseValueVector.MAX_ALLOCATION_SIZE to allow to use a much smaller value during tests. Project: http://git-wip-us.apache.org/repos/asf/arrow/repo Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/c0985a47 Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/c0985a47 Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/c0985a47 Branch: refs/heads/master Commit: c0985a47665f8ce8847a6a0215e6e3c0f1db28f4 Parents: e0fb369 Author: Laurent Goujon <[email protected]> Authored: Mon Apr 18 11:07:22 2016 -0700 Committer: Steven Phillips <[email protected]> Committed: Thu May 19 13:17:03 2016 -0700 ---------------------------------------------------------------------- .../apache/arrow/vector/BaseValueVector.java | 14 ++++---- .../apache/arrow/vector/TestValueVector.java | 36 ++++++++++++++++---- 2 files changed, 38 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/arrow/blob/c0985a47/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java index 8bca3c0..932e6f1 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java @@ -17,23 +17,24 @@ */ package org.apache.arrow.vector; -import io.netty.buffer.ArrowBuf; - import java.util.Iterator; -import com.google.common.base.Preconditions; -import com.google.common.collect.Iterators; - import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.types.MaterializedField; import org.apache.arrow.vector.util.TransferPair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.base.Preconditions; +import com.google.common.collect.Iterators; + +import io.netty.buffer.ArrowBuf; + public abstract class BaseValueVector implements ValueVector { private static final Logger logger = LoggerFactory.getLogger(BaseValueVector.class); - public static final int MAX_ALLOCATION_SIZE = Integer.MAX_VALUE; + public static final String MAX_ALLOCATION_SIZE_PROPERTY = "arrow.vector.max_allocation_bytes"; + public static final int MAX_ALLOCATION_SIZE = Integer.getInteger(MAX_ALLOCATION_SIZE_PROPERTY, Integer.MAX_VALUE); public static final int INITIAL_VALUE_ALLOCATION = 4096; protected final BufferAllocator allocator; @@ -99,6 +100,7 @@ public abstract class BaseValueVector implements ValueVector { public void generateTestData(int values) {} //TODO: consider making mutator stateless(if possible) on another issue. + @Override public void reset() {} } http://git-wip-us.apache.org/repos/asf/arrow/blob/c0985a47/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java ---------------------------------------------------------------------- diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java index ac3eebe..b5c4509 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java @@ -23,16 +23,12 @@ import static org.junit.Assert.assertTrue; import java.nio.charset.Charset; +import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.complex.ListVector; import org.apache.arrow.vector.complex.MapVector; import org.apache.arrow.vector.complex.RepeatedListVector; import org.apache.arrow.vector.complex.RepeatedMapVector; -import org.apache.arrow.vector.types.MaterializedField; -import org.apache.arrow.vector.types.Types; -import org.apache.arrow.vector.types.Types.MinorType; -import org.apache.arrow.vector.util.BasicTypeHelper; -import org.apache.arrow.vector.util.OversizedAllocationException; import org.apache.arrow.vector.holders.BitHolder; import org.apache.arrow.vector.holders.IntHolder; import org.apache.arrow.vector.holders.NullableFloat4Holder; @@ -44,10 +40,16 @@ import org.apache.arrow.vector.holders.RepeatedIntHolder; import org.apache.arrow.vector.holders.RepeatedVarBinaryHolder; import org.apache.arrow.vector.holders.UInt4Holder; import org.apache.arrow.vector.holders.VarCharHolder; -import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.types.MaterializedField; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.Types.MinorType; +import org.apache.arrow.vector.util.BasicTypeHelper; +import org.apache.arrow.vector.util.OversizedAllocationException; import org.junit.After; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExternalResource; public class TestValueVector { @@ -57,6 +59,28 @@ public class TestValueVector { private BufferAllocator allocator; + // Rule to adjust MAX_ALLOCATION_SIZE and restore it back after the tests + @Rule + public final ExternalResource rule = new ExternalResource() { + private final String systemValue = System.getProperty(BaseValueVector.MAX_ALLOCATION_SIZE_PROPERTY); + private final String testValue = Long.toString(32*1024*1024); + + @Override + protected void before() throws Throwable { + System.setProperty(BaseValueVector.MAX_ALLOCATION_SIZE_PROPERTY, testValue); + } + + @Override + protected void after() { + if (systemValue != null) { + System.setProperty(BaseValueVector.MAX_ALLOCATION_SIZE_PROPERTY, systemValue); + } + else { + System.clearProperty(BaseValueVector.MAX_ALLOCATION_SIZE_PROPERTY); + } + } + }; + @Before public void init() { allocator = new RootAllocator(Long.MAX_VALUE);
