This is an automated email from the ASF dual-hosted git repository.
cutlerb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 463161d ARROW-9371: [Java] Run vector tests for both allocators
463161d is described below
commit 463161dba7caa5f5f4d9abc9ea4a14cca40f719f
Author: Ryan Murray <[email protected]>
AuthorDate: Fri Jul 24 17:11:59 2020 -0700
ARROW-9371: [Java] Run vector tests for both allocators
As per #7619 (comment) the vector tests should be run for both netty and
unsafe allocators. The default tests are run with the Netty allocator, and
`run-unsafe` tests are done with the Unsafe Allocator.
Closes #7676 from rymurr/ARROW-9371
Authored-by: Ryan Murray <[email protected]>
Signed-off-by: Bryan Cutler <[email protected]>
---
java/vector/pom.xml | 131 ++++++++++++++-------
.../apache/arrow/vector/TestBitVectorHelper.java | 73 ++++++------
2 files changed, 126 insertions(+), 78 deletions(-)
diff --git a/java/vector/pom.xml b/java/vector/pom.xml
index 6b10137..aec4f7e 100644
--- a/java/vector/pom.xml
+++ b/java/vector/pom.xml
@@ -56,6 +56,12 @@
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>org.apache.arrow</groupId>
+ <artifactId>arrow-memory-unsafe</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>io.netty</groupId>
<artifactId>netty-common</artifactId>
</dependency>
@@ -70,19 +76,19 @@
</dependency>
</dependencies>
- <pluginRepositories>
- <pluginRepository>
- <id>apache</id>
- <name>apache</name>
- <url>https://repo.maven.apache.org/maven2/</url>
- <releases>
- <enabled>true</enabled>
- </releases>
- <snapshots>
- <enabled>false</enabled>
- </snapshots>
- </pluginRepository>
- </pluginRepositories>
+ <pluginRepositories>
+ <pluginRepository>
+ <id>apache</id>
+ <name>apache</name>
+ <url>https://repo.maven.apache.org/maven2/</url>
+ <releases>
+ <enabled>true</enabled>
+ </releases>
+ <snapshots>
+ <enabled>false</enabled>
+ </snapshots>
+ </pluginRepository>
+ </pluginRepositories>
<build>
@@ -97,6 +103,47 @@
<plugins>
<plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <enableAssertions>true</enableAssertions>
+ <childDelegation>true</childDelegation>
+ <forkCount>${forkCount}</forkCount>
+ <reuseForks>true</reuseForks>
+ <systemPropertyVariables>
+ <java.io.tmpdir>${project.build.directory}</java.io.tmpdir>
+
<io.netty.tryReflectionSetAccessible>true</io.netty.tryReflectionSetAccessible>
+ <user.timezone>UTC</user.timezone>
+ </systemPropertyVariables>
+ <!-- Note: changing the below configuration might increase the max
allocation size for a vector
+ which in turn can cause OOM. -->
+ <argLine>-Darrow.vector.max_allocation_bytes=1048576</argLine>
+ </configuration>
+ <executions>
+ <execution>
+ <id>default-test</id>
+ <phase>test</phase>
+ <configuration>
+ <classpathDependencyExcludes>
+
<classpathDependencyExclude>org.apache.arrow:arrow-memory-unsafe</classpathDependencyExclude>
+ </classpathDependencyExcludes>
+ </configuration>
+ </execution>
+ <execution>
+ <id>run-unsafe</id>
+ <phase>test</phase>
+ <goals>
+ <goal>test</goal>
+ </goals>
+ <configuration>
+ <classpathDependencyExcludes>
+
<classpathDependencyExclude>org.apache.arrow:arrow-memory-netty</classpathDependencyExclude>
+ </classpathDependencyExcludes>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
<artifactId>maven-resources-plugin</artifactId>
<executions>
<execution> <!-- copy all templates in the same location to compile
them at once -->
@@ -137,35 +184,35 @@
</executions>
</plugin>
<plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <version>3.1.1</version>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <artifactSet>
- <includes>
- <include>org.apache.arrow:arrow-format</include>
- <include>com.google.flatbuffers:*</include>
- </includes>
- </artifactSet>
- <shadedArtifactAttached>true</shadedArtifactAttached>
-
<shadedClassifierName>shade-format-flatbuffers</shadedClassifierName>
-
<keepDependenciesWithProvidedScope>true</keepDependenciesWithProvidedScope>
-
<promoteTransitiveDependencies>true</promoteTransitiveDependencies>
- <relocations>
- <relocation>
- <pattern>com.google.flatbuffers</pattern>
-
<shadedPattern>arrow.vector.com.google.flatbuffers</shadedPattern>
- </relocation>
- </relocations>
- </configuration>
- </execution>
- </executions>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>3.1.1</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <artifactSet>
+ <includes>
+ <include>org.apache.arrow:arrow-format</include>
+ <include>com.google.flatbuffers:*</include>
+ </includes>
+ </artifactSet>
+ <shadedArtifactAttached>true</shadedArtifactAttached>
+
<shadedClassifierName>shade-format-flatbuffers</shadedClassifierName>
+
<keepDependenciesWithProvidedScope>true</keepDependenciesWithProvidedScope>
+
<promoteTransitiveDependencies>true</promoteTransitiveDependencies>
+ <relocations>
+ <relocation>
+ <pattern>com.google.flatbuffers</pattern>
+
<shadedPattern>arrow.vector.com.google.flatbuffers</shadedPattern>
+ </relocation>
+ </relocations>
+ </configuration>
+ </execution>
+ </executions>
</plugin>
</plugins>
<pluginManagement>
diff --git
a/java/vector/src/test/java/org/apache/arrow/vector/TestBitVectorHelper.java
b/java/vector/src/test/java/org/apache/arrow/vector/TestBitVectorHelper.java
index 8cc697e..4b48876 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestBitVectorHelper.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestBitVectorHelper.java
@@ -22,50 +22,51 @@ import static org.junit.Assert.assertTrue;
import static org.junit.jupiter.api.Assertions.assertFalse;
import org.apache.arrow.memory.ArrowBuf;
-import org.apache.arrow.memory.ReferenceManager;
+import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
import org.junit.Test;
-import io.netty.buffer.PooledByteBufAllocatorL;
import io.netty.util.internal.PlatformDependent;
public class TestBitVectorHelper {
@Test
public void testGetNullCount() throws Exception {
- // test case 1, 1 null value for 0b110
- ArrowBuf validityBuffer = new ArrowBuf(
- ReferenceManager.NO_OP, null, 3, new
PooledByteBufAllocatorL().empty.memoryAddress());
- // we set validity buffer to be 0b10110, but only have 3 items with 1st
item is null
- validityBuffer.setByte(0, 0b10110);
-
- // we will only consider 0b110 here, since we only 3 items and only one is
null
- int count = BitVectorHelper.getNullCount(validityBuffer, 3);
- assertEquals(count, 1);
-
- // test case 2, no null value for 0xFF
- validityBuffer = new ArrowBuf(
- ReferenceManager.NO_OP, null, 8, new
PooledByteBufAllocatorL().empty.memoryAddress());
- validityBuffer.setByte(0, 0xFF);
-
- count = BitVectorHelper.getNullCount(validityBuffer, 8);
- assertEquals(count, 0);
-
- // test case 3, 1 null value for 0x7F
- validityBuffer = new ArrowBuf(
- ReferenceManager.NO_OP, null, 8, new
PooledByteBufAllocatorL().empty.memoryAddress());
- validityBuffer.setByte(0, 0x7F);
-
- count = BitVectorHelper.getNullCount(validityBuffer, 8);
- assertEquals(count, 1);
-
- // test case 4, validity buffer has multiple bytes, 11 items
- validityBuffer = new ArrowBuf(
- ReferenceManager.NO_OP, null, 11, new
PooledByteBufAllocatorL().empty.memoryAddress());
- validityBuffer.setByte(0, 0b10101010);
- validityBuffer.setByte(1, 0b01010101);
-
- count = BitVectorHelper.getNullCount(validityBuffer, 11);
- assertEquals(count, 5);
+ try (BufferAllocator root = new RootAllocator()) {
+ // test case 1, 1 null value for 0b110
+ ArrowBuf validityBuffer = root.buffer(3);
+ // we set validity buffer to be 0b10110, but only have 3 items with 1st
item is null
+ validityBuffer.setByte(0, 0b10110);
+
+ // we will only consider 0b110 here, since we only 3 items and only one
is null
+ int count = BitVectorHelper.getNullCount(validityBuffer, 3);
+ assertEquals(count, 1);
+ validityBuffer.close();
+
+ // test case 2, no null value for 0xFF
+ validityBuffer = root.buffer(8);
+ validityBuffer.setByte(0, 0xFF);
+
+ count = BitVectorHelper.getNullCount(validityBuffer, 8);
+ assertEquals(count, 0);
+ validityBuffer.close();
+
+ // test case 3, 1 null value for 0x7F
+ validityBuffer = root.buffer(8);
+ validityBuffer.setByte(0, 0x7F);
+
+ count = BitVectorHelper.getNullCount(validityBuffer, 8);
+ assertEquals(count, 1);
+ validityBuffer.close();
+
+ // test case 4, validity buffer has multiple bytes, 11 items
+ validityBuffer = root.buffer(11);
+ validityBuffer.setByte(0, 0b10101010);
+ validityBuffer.setByte(1, 0b01010101);
+
+ count = BitVectorHelper.getNullCount(validityBuffer, 11);
+ assertEquals(count, 5);
+ validityBuffer.close();
+ }
}
@Test
