DRILL-1660: adding a test case to cover an array of empty and null valued objects. i) remove redundant guard statement from JsonRecordReader2 ii) rename unit test suite
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/0cddfc22 Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/0cddfc22 Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/0cddfc22 Branch: refs/heads/master Commit: 0cddfc2228859bb636dcf6ed03a227626f69e480 Parents: 23d5c38 Author: Hanifi Gunes <hgu...@maprtech.com> Authored: Tue Nov 11 15:46:10 2014 -0800 Committer: Hanifi Gunes <hgu...@maprtech.com> Committed: Tue Nov 11 16:14:43 2014 -0800 ---------------------------------------------------------------------- .../exec/store/easy/json/JSONRecordReader2.java | 2 +- .../exec/vector/complex/fn/JsonReaderTests.java | 127 ------------------- .../fn/TestJsonReaderWithSparseFiles.java | 127 +++++++++++++++++++ .../vector/complex/fn/nested-with-nulls.json | 3 +- 4 files changed, 130 insertions(+), 129 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0cddfc22/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader2.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader2.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader2.java index ff7d3f2..d1502d4 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader2.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader2.java @@ -105,7 +105,7 @@ public class JSONRecordReader2 extends AbstractRecordReader { recordCount = 0; try{ - outside: while(true && recordCount < BaseValueVector.INITIAL_VALUE_ALLOCATION){ + outside: while(recordCount < BaseValueVector.INITIAL_VALUE_ALLOCATION){ writer.setPosition(recordCount); switch(jsonReader.write(writer)){ http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0cddfc22/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/fn/JsonReaderTests.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/fn/JsonReaderTests.java b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/fn/JsonReaderTests.java deleted file mode 100644 index 9a8eedf..0000000 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/fn/JsonReaderTests.java +++ /dev/null @@ -1,127 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.vector.complex.fn; - - -import java.util.List; -import java.util.Objects; - -import org.apache.drill.BaseTestQuery; -import org.apache.drill.common.expression.SchemaPath; -import org.apache.drill.exec.proto.UserBitShared; -import org.apache.drill.exec.proto.beans.QueryResult; -import org.apache.drill.exec.record.RecordBatchLoader; -import org.apache.drill.exec.rpc.user.QueryResultBatch; -import org.apache.drill.exec.vector.ValueVector; -import org.junit.Test; - -public class JsonReaderTests extends BaseTestQuery { - - static interface Function<T> { - void apply(T param); - } - - protected void query(final String query, final Function<RecordBatchLoader> testBody) throws Exception { - List<QueryResultBatch> batches = testSqlWithResults(query); - RecordBatchLoader loader = new RecordBatchLoader(client.getAllocator()); - try { - QueryResultBatch batch = batches.get(0); - loader.load(batch.getHeader().getDef(), batch.getData()); - - testBody.apply(loader); - - } finally { - for (QueryResultBatch batch:batches) { - batch.release(); - } - loader.clear(); - } - } - - @Test - public void testIfDrillCanReadSparseRecords() throws Exception { - final String sql = "select * from cp.`vector/complex/fn/sparse.json`"; - query(sql, new Function<RecordBatchLoader>() { - @Override - public void apply(RecordBatchLoader loader) { - assert loader.getRecordCount() == 4 : "invalid record count returned"; - - //XXX: make sure value order matches vector order - final Object[][] values = new Object[][] { - {null, null}, - {1L, null}, - {null, 2L}, - {3L, 3L} - }; - - Object[] row; - Object expected; - Object actual; - for (int r=0;r<values.length;r++) { - row = values[r]; - for (int c=0; c<values[r].length; c++) { - expected = row[c]; - actual = loader.getValueAccessorById(ValueVector.class, c).getValueVector().getAccessor().getObject(r); - assert Objects.equals(expected, actual) : String.format("row:%d - col:%d - expected:%s[%s] - actual:%s[%s]", - r, c, - expected, - expected==null?"null":expected.getClass().getSimpleName(), - actual, - actual==null?"null":actual.getClass().getSimpleName()); - } - } - } - }); - } - - @Test - public void testIfDrillCanReadSparseNestedRecordsWithoutRaisingException() throws Exception { - final String sql = "select * from cp.`vector/complex/fn/nested-with-nulls.json`"; - query(sql, new Function<RecordBatchLoader>() { - @Override - public void apply(RecordBatchLoader loader) { - assert loader.getRecordCount() == 3 : "invalid record count returned"; - - //XXX: make sure value order matches vector order - final Object[][] values = new Object[][] { - {"[{},{},{},{\"name\":\"doe\"},{}]"}, - {"[]"}, - {"[{\"name\":\"john\",\"id\":10}]"}, - }; - - Object[] row; - Object expected; - Object actual; - for (int r=0;r<values.length;r++) { - row = values[r]; - for (int c = 0; c < values[r].length; c++) { - expected = row[c]; - actual = loader.getValueAccessorById(ValueVector.class, c).getValueVector().getAccessor().getObject(r); - assert Objects.equals(actual, expected) : String.format("row:%d - col:%d - expected:%s[%s] - actual:%s[%s]", - r, c, - expected, - expected == null ? "null" : expected.getClass().getSimpleName(), - actual, - actual == null ? "null" : actual.getClass().getSimpleName()); - } - } - } - }); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0cddfc22/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/fn/TestJsonReaderWithSparseFiles.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/fn/TestJsonReaderWithSparseFiles.java b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/fn/TestJsonReaderWithSparseFiles.java new file mode 100644 index 0000000..3cfdc1d --- /dev/null +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/fn/TestJsonReaderWithSparseFiles.java @@ -0,0 +1,127 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.vector.complex.fn; + + +import java.util.List; +import java.util.Objects; + +import org.apache.drill.BaseTestQuery; +import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.exec.proto.UserBitShared; +import org.apache.drill.exec.proto.beans.QueryResult; +import org.apache.drill.exec.record.RecordBatchLoader; +import org.apache.drill.exec.rpc.user.QueryResultBatch; +import org.apache.drill.exec.vector.ValueVector; +import org.junit.Test; + +public class TestJsonReaderWithSparseFiles extends BaseTestQuery { + + static interface Function<T> { + void apply(T param); + } + + protected void query(final String query, final Function<RecordBatchLoader> testBody) throws Exception { + List<QueryResultBatch> batches = testSqlWithResults(query); + RecordBatchLoader loader = new RecordBatchLoader(client.getAllocator()); + try { + // first batch at index 0 is empty and used for fast schema return. Load the second one for the tests + QueryResultBatch batch = batches.get(1); + loader.load(batch.getHeader().getDef(), batch.getData()); + testBody.apply(loader); + } finally { + for (QueryResultBatch batch:batches) { + batch.release(); + } + loader.clear(); + } + } + + @Test + public void testIfDrillCanReadSparseRecords() throws Exception { + final String sql = "select * from cp.`vector/complex/fn/sparse.json`"; + query(sql, new Function<RecordBatchLoader>() { + @Override + public void apply(RecordBatchLoader loader) { + assert loader.getRecordCount() == 4 : "invalid record count returned"; + + //XXX: make sure value order matches vector order + final Object[][] values = new Object[][] { + {null, null}, + {1L, null}, + {null, 2L}, + {3L, 3L} + }; + + Object[] row; + Object expected; + Object actual; + for (int r=0;r<values.length;r++) { + row = values[r]; + for (int c=0; c<values[r].length; c++) { + expected = row[c]; + actual = loader.getValueAccessorById(ValueVector.class, c).getValueVector().getAccessor().getObject(r); + assert Objects.equals(expected, actual) : String.format("row:%d - col:%d - expected:%s[%s] - actual:%s[%s]", + r, c, + expected, + expected==null?"null":expected.getClass().getSimpleName(), + actual, + actual==null?"null":actual.getClass().getSimpleName()); + } + } + } + }); + } + + @Test + public void testIfDrillCanReadSparseNestedRecordsWithoutRaisingException() throws Exception { + final String sql = "select * from cp.`vector/complex/fn/nested-with-nulls.json`"; + query(sql, new Function<RecordBatchLoader>() { + @Override + public void apply(RecordBatchLoader loader) { + assert loader.getRecordCount() == 4 : "invalid record count returned"; + + //XXX: make sure value order matches vector order + final Object[][] values = new Object[][] { + {"[{},{},{},{\"name\":\"doe\"},{}]"}, + {"[]"}, + {"[{\"name\":\"john\",\"id\":10}]"}, + {"[{},{}]"}, + }; + + Object[] row; + Object expected; + Object actual; + for (int r=0;r<values.length;r++) { + row = values[r]; + for (int c = 0; c < values[r].length; c++) { + expected = row[c]; + actual = loader.getValueAccessorById(ValueVector.class, c).getValueVector().getAccessor().getObject(r); + assert Objects.equals(actual, expected) : String.format("row:%d - col:%d - expected:%s[%s] - actual:%s[%s]", + r, c, + expected, + expected == null ? "null" : expected.getClass().getSimpleName(), + actual, + actual == null ? "null" : actual.getClass().getSimpleName()); + } + } + } + }); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0cddfc22/exec/java-exec/src/test/resources/vector/complex/fn/nested-with-nulls.json ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/vector/complex/fn/nested-with-nulls.json b/exec/java-exec/src/test/resources/vector/complex/fn/nested-with-nulls.json index 1514a47..7ce785c 100644 --- a/exec/java-exec/src/test/resources/vector/complex/fn/nested-with-nulls.json +++ b/exec/java-exec/src/test/resources/vector/complex/fn/nested-with-nulls.json @@ -1,3 +1,4 @@ {"users":[{}, {"id": null, "name":null}, {}, {"name": "doe"}, {}]} {} -{"users":[{"id": 10, "name":"john"}]} \ No newline at end of file +{"users":[{"id": 10, "name":"john"}]} +{"users":[{}, {"nothing":null}]} \ No newline at end of file
