Repository: arrow Updated Branches: refs/heads/master d06c49144 -> 14ed1be2d
ARROW-400: set struct length on load Adds unit test, closes #233 Author: Julien Le Dem <[email protected]> Author: Wes McKinney <[email protected]> Closes #234 from wesm/ARROW-400 and squashes the following commits: f516ba1 [Wes McKinney] Add unit test for ARROW-400 741ff71 [Julien Le Dem] ARROW-400: set struct length on json load Project: http://git-wip-us.apache.org/repos/asf/arrow/repo Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/14ed1be2 Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/14ed1be2 Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/14ed1be2 Branch: refs/heads/master Commit: 14ed1be2d89fedc31f4015456cda28216f926dcc Parents: d06c491 Author: Julien Le Dem <[email protected]> Authored: Fri Dec 9 12:04:25 2016 -0500 Committer: Wes McKinney <[email protected]> Committed: Fri Dec 9 12:04:25 2016 -0500 ---------------------------------------------------------------------- integration/data/struct_example.json | 237 +++++++++++++++++++ .../apache/arrow/vector/complex/MapVector.java | 2 +- .../arrow/vector/file/json/JsonFileReader.java | 4 + .../arrow/vector/file/json/TestJSONFile.java | 20 ++ 4 files changed, 262 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/arrow/blob/14ed1be2/integration/data/struct_example.json ---------------------------------------------------------------------- diff --git a/integration/data/struct_example.json b/integration/data/struct_example.json new file mode 100644 index 0000000..3ea062d --- /dev/null +++ b/integration/data/struct_example.json @@ -0,0 +1,237 @@ +{ + "schema": { + "fields": [ + { + "name": "struct_nullable", + "type": { + "name": "struct" + }, + "nullable": true, + "children": [ + { + "name": "f1", + "type": { + "name": "int", + "isSigned": true, + "bitWidth": 32 + }, + "nullable": true, + "children": [], + "typeLayout": { + "vectors": [ + { + "type": "VALIDITY", + "typeBitWidth": 1 + }, + { + "type": "DATA", + "typeBitWidth": 32 + } + ] + } + }, + { + "name": "f2", + "type": { + "name": "utf8" + }, + "nullable": true, + "children": [], + "typeLayout": { + "vectors": [ + { + "type": "VALIDITY", + "typeBitWidth": 1 + }, + { + "type": "OFFSET", + "typeBitWidth": 32 + }, + { + "type": "DATA", + "typeBitWidth": 8 + } + ] + } + } + ], + "typeLayout": { + "vectors": [ + { + "type": "VALIDITY", + "typeBitWidth": 1 + } + ] + } + } + ] + }, + "batches": [ + { + "count": 7, + "columns": [ + { + "name": "struct_nullable", + "count": 7, + "VALIDITY": [ + 0, + 1, + 1, + 1, + 0, + 1, + 0 + ], + "children": [ + { + "name": "f1", + "count": 7, + "VALIDITY": [ + 1, + 0, + 1, + 1, + 1, + 0, + 0 + ], + "DATA": [ + 1402032511, + 290876774, + 137773603, + 410361374, + 1959836418, + 1995074679, + -163525262 + ] + }, + { + "name": "f2", + "count": 7, + "VALIDITY": [ + 0, + 1, + 1, + 1, + 0, + 1, + 0 + ], + "OFFSET": [ + 0, + 0, + 7, + 14, + 21, + 21, + 28, + 28 + ], + "DATA": [ + "", + "MhRNxD4", + "3F9HBxK", + "aVd88fp", + "", + "3loZrRf", + "" + ] + } + ] + } + ] + }, + { + "count": 10, + "columns": [ + { + "name": "struct_nullable", + "count": 10, + "VALIDITY": [ + 0, + 1, + 1, + 0, + 1, + 0, + 0, + 1, + 1, + 1 + ], + "children": [ + { + "name": "f1", + "count": 10, + "VALIDITY": [ + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 0, + 0, + 0 + ], + "DATA": [ + -2041500147, + 1715692943, + -35444996, + 1425496657, + 112765084, + 1760754983, + 413888857, + 2039738337, + -1924327700, + 670528518 + ] + }, + { + "name": "f2", + "count": 10, + "VALIDITY": [ + 1, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 0 + ], + "OFFSET": [ + 0, + 7, + 7, + 7, + 14, + 21, + 28, + 35, + 42, + 49, + 49 + ], + "DATA": [ + "AS5oARE", + "", + "", + "JGdagcX", + "78SLiRw", + "vbGf7OY", + "5uh5fTs", + "0ilsf82", + "LjS9MbU", + "" + ] + } + ] + } + ] + } + ] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/arrow/blob/14ed1be2/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java index c2f216b..31a1bb7 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java @@ -50,7 +50,7 @@ public class MapVector extends AbstractMapVector { private final SingleMapReaderImpl reader = new SingleMapReaderImpl(this); private final Accessor accessor = new Accessor(); private final Mutator mutator = new Mutator(); - int valueCount; + public int valueCount; public MapVector(String name, BufferAllocator allocator, CallBack callBack) { super(name, allocator, callBack); http://git-wip-us.apache.org/repos/asf/arrow/blob/14ed1be2/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java index 26dd3f6..152867c 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java @@ -47,6 +47,7 @@ import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.ValueVector.Mutator; import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.complex.NullableMapVector; import org.apache.arrow.vector.schema.ArrowVectorType; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.Schema; @@ -153,6 +154,9 @@ public class JsonFileReader implements AutoCloseable { } readToken(END_ARRAY); } + if (vector instanceof NullableMapVector) { + ((NullableMapVector)vector).valueCount = count; + } } readToken(END_OBJECT); } http://git-wip-us.apache.org/repos/asf/arrow/blob/14ed1be2/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java ---------------------------------------------------------------------- diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java b/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java index 7d25003..3720a13 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java @@ -21,11 +21,13 @@ import java.io.File; import java.io.IOException; import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.complex.MapVector; import org.apache.arrow.vector.complex.NullableMapVector; import org.apache.arrow.vector.file.BaseFileTest; import org.apache.arrow.vector.types.pojo.Schema; +import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -117,4 +119,22 @@ public class TestJSONFile extends BaseFileTest { } } + @Test + public void testSetStructLength() throws IOException { + File file = new File("../../integration/data/struct_example.json"); + try ( + BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE); + ) { + JsonFileReader reader = new JsonFileReader(file, readerAllocator); + Schema schema = reader.start(); + LOGGER.debug("reading schema: " + schema); + + // initialize vectors + try (VectorSchemaRoot root = reader.read();) { + FieldVector vector = root.getVector("struct_nullable"); + Assert.assertEquals(7, vector.getAccessor().getValueCount()); + } + } + } + }
