DRILL-4410: ListVector should initialize bits in allocateNew Remove large arrays.json files and generate it in /tmp, Check test results Add another unit test for ListVector in TestValueVector, and use tempDir in TestComplexTypeReader
Closes #380 Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/ae46d268 Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/ae46d268 Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/ae46d268 Branch: refs/heads/master Commit: ae46d268b318b43f178250e34fc58028c016fc70 Parents: c95b543 Author: Minji Kim <[email protected]> Authored: Wed Feb 17 22:29:11 2016 -0800 Committer: vkorukanti <[email protected]> Committed: Wed Mar 2 14:08:51 2016 -0800 ---------------------------------------------------------------------- .../exec/record/vector/TestValueVector.java | 23 +++++++- .../complex/writer/TestComplexTypeReader.java | 58 ++++++++++++++++++++ .../drill/exec/vector/complex/ListVector.java | 1 + 3 files changed, 81 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/drill/blob/ae46d268/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java b/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java index f6bcf97..f6044e1 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java @@ -25,7 +25,7 @@ import java.nio.charset.Charset; import org.apache.drill.common.AutoCloseables; import org.apache.drill.common.config.DrillConfig; -import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.types.MinorType; import org.apache.drill.common.types.TypeProtos; import org.apache.drill.common.types.Types; import org.apache.drill.exec.ExecTest; @@ -47,6 +47,7 @@ import org.apache.drill.exec.memory.BufferAllocator; import org.apache.drill.exec.memory.RootAllocatorFactory; import org.apache.drill.exec.proto.UserBitShared; import org.apache.drill.exec.record.MaterializedField; +import org.apache.drill.exec.record.VectorWrapper; import org.apache.drill.exec.vector.BaseValueVector; import org.apache.drill.exec.vector.BitVector; import org.apache.drill.exec.vector.NullableFloat4Vector; @@ -56,6 +57,7 @@ import org.apache.drill.exec.vector.RepeatedIntVector; import org.apache.drill.exec.vector.UInt4Vector; import org.apache.drill.exec.vector.ValueVector; import org.apache.drill.exec.vector.VarCharVector; +import org.apache.drill.exec.vector.complex.ListVector; import org.apache.drill.exec.vector.complex.MapVector; import org.apache.drill.exec.vector.complex.RepeatedListVector; import org.apache.drill.exec.vector.complex.RepeatedMapVector; @@ -789,4 +791,23 @@ the interface to load has changed } }); } + + @Test + public void testListVectorShouldNotThrowOversizedAllocationException() throws Exception { + final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, + Types.optional(TypeProtos.MinorType.LIST)); + ListVector vector = new ListVector(field, allocator, null); + ListVector vectorFrom = new ListVector(field, allocator, null); + vectorFrom.allocateNew(); + + for (int i = 0; i < 10000; i++) { + vector.allocateNew(); + vector.copyFromSafe(0, 0, vectorFrom); + vector.clear(); + } + + vectorFrom.clear(); + vector.clear(); + } + } http://git-wip-us.apache.org/repos/asf/drill/blob/ae46d268/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java index 077a3ca..c3cfb68 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java @@ -19,10 +19,21 @@ package org.apache.drill.exec.vector.complex.writer; import org.apache.drill.BaseTestQuery; +import org.apache.drill.TestBuilder; import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Test; +import java.io.BufferedWriter; +import java.io.FileWriter; +import java.io.IOException; +import java.io.PrintWriter; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import static org.apache.drill.TestBuilder.listOf; + public class TestComplexTypeReader extends BaseTestQuery{ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestComplexTypeReader.class); @@ -241,4 +252,51 @@ public class TestComplexTypeReader extends BaseTestQuery{ .go(); } + @Test // DRILL-4410 + // ListVector allocation + public void test_array() throws Exception{ + + long numRecords = 100000; + + String tempDir = BaseTestQuery.getTempDir("ComplexTypeWriter"); + String file1 = tempDir + TestComplexTypeReader.class.getName() + "arrays1.json"; + String file2 = tempDir + TestComplexTypeReader.class.getName() + "arrays2.json"; + Path path1 = Paths.get(file1); + Path path2 = Paths.get(file2); + + String arrayString = "[ \"abcdef\", \"ghijkl\", \"mnopqr\", \"stuvwx\", \"yz1234\", \"567890\" ] "; + Files.deleteIfExists(path1); + Files.deleteIfExists(path2); + Files.createFile(path1); + Files.createFile(path2); + + try(PrintWriter out = new PrintWriter(new BufferedWriter(new FileWriter(file1, true)))) { + for (long i = 0; i < numRecords; i++) { + out.println("{ \"id\" : " + i + ", \"array\" : " + arrayString + "}"); + } + }catch (IOException e) { + throw e; + } + + try(PrintWriter out = new PrintWriter(new BufferedWriter(new FileWriter(file2, true)))) { + for (long i = 0; i < numRecords; i++) { + out.println("{ \"id\" : " + i + ", \"array\" : " + arrayString + "}"); + } + }catch (IOException e) { + throw e; + } + + String queryString = "select * from dfs.`" + file1 + "` `arrays1` INNER JOIN dfs.`" + file2 + "` `arrays2` ON " + + "(`arrays1`.id = `arrays2`.id)"; + TestBuilder testBuilder = testBuilder().sqlQuery(queryString).unOrdered(); + testBuilder.baselineColumns("id", "id0", "array", "array0"); + for (long i = 0; i < numRecords; i++) { + testBuilder.baselineValues(i, i, listOf("abcdef", "ghijkl", "mnopqr", "stuvwx", "yz1234", "567890"), + listOf("abcdef", "ghijkl", "mnopqr", "stuvwx", "yz1234", "567890")); + } + testBuilder.go(); + + Files.deleteIfExists(path1); + Files.deleteIfExists(path2); + } } http://git-wip-us.apache.org/repos/asf/drill/blob/ae46d268/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java ---------------------------------------------------------------------- diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java index ba30241..33d6ddc 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java @@ -70,6 +70,7 @@ public class ListVector extends BaseRepeatedValueVector { @Override public void allocateNew() throws OutOfMemoryException { super.allocateNewSafe(); + bits.allocateNewSafe(); } public void transferTo(ListVector target) {
