Does this means any complex type will be displayed JSON strings to the user from the interactive shell as well?
Tim On Tue, May 13, 2014 at 5:26 PM, <[email protected]> wrote: > DRILL-718: Return complex types as JSON string. > > > Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo > Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/ea72a380 > Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/ea72a380 > Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/ea72a380 > > Branch: refs/heads/master > Commit: ea72a380e875aaaa126510116742be659016d98f > Parents: 4b57652 > Author: Mehant Baid <[email protected]> > Authored: Tue May 13 15:07:54 2014 -0700 > Committer: Aditya Kishore <[email protected]> > Committed: Tue May 13 16:06:03 2014 -0700 > > ---------------------------------------------------------------------- > .../src/main/codegen/includes/vv_imports.ftl | 1 + > .../codegen/templates/RepeatedValueVectors.java | 2 +- > .../src/main/codegen/templates/TypeHelper.java | 5 ++ > .../drill/exec/util/JsonStringArrayList.java | 46 ++++++++++++++++++ > .../drill/exec/util/JsonStringHashMap.java | 50 ++++++++++++++++++++ > .../apache/drill/exec/util/TextSerializer.java | 38 +++++++++++++++ > .../drill/exec/vector/complex/MapVector.java | 3 +- > .../exec/vector/complex/RepeatedListVector.java | 3 +- > .../exec/vector/complex/RepeatedMapVector.java | 3 +- > .../exec/vector/complex/fn/JsonReader.java | 2 +- > .../exec/store/json/JsonRecordReader2Test.java | 2 + > 11 files changed, 150 insertions(+), 5 deletions(-) > ---------------------------------------------------------------------- > > > http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/codegen/includes/vv_imports.ftl > ---------------------------------------------------------------------- > diff --git a/exec/java-exec/src/main/codegen/includes/vv_imports.ftl > b/exec/java-exec/src/main/codegen/includes/vv_imports.ftl > index 71404d3..114ecbe 100644 > --- a/exec/java-exec/src/main/codegen/includes/vv_imports.ftl > +++ b/exec/java-exec/src/main/codegen/includes/vv_imports.ftl > @@ -38,6 +38,7 @@ import org.apache.drill.exec.vector.complex.impl.*; > import org.apache.drill.exec.vector.complex.writer.*; > import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter; > import org.apache.drill.exec.vector.complex.writer.BaseWriter.ListWriter; > +import org.apache.drill.exec.util.JsonStringArrayList; > > import org.apache.drill.exec.memory.OutOfMemoryRuntimeException; > > > http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java > ---------------------------------------------------------------------- > diff --git > a/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java > b/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java > index 537c7fd..cac87c5 100644 > --- a/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java > +++ b/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java > @@ -296,7 +296,7 @@ package org.apache.drill.exec.vector; > } > > public List<${friendlyType}> getObject(int index) { > - List<${friendlyType}> vals = Lists.newArrayList(); > + List<${friendlyType}> vals = new JsonStringArrayList(); > int start = offsets.getAccessor().get(index) - sliceOffset; > int end = offsets.getAccessor().get(index+1) - sliceOffset; > for(int i = start; i < end; i++){ > > http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/codegen/templates/TypeHelper.java > ---------------------------------------------------------------------- > diff --git a/exec/java-exec/src/main/codegen/templates/TypeHelper.java > b/exec/java-exec/src/main/codegen/templates/TypeHelper.java > index 4755e92..f2d66d2 100644 > --- a/exec/java-exec/src/main/codegen/templates/TypeHelper.java > +++ b/exec/java-exec/src/main/codegen/templates/TypeHelper.java > @@ -66,7 +66,12 @@ public class TypeHelper { > } > </#list> > </#list> > + case MAP: > + case REPEATMAP: > + case LIST: > + return new GenericAccessor(vector); > } > + > throw new UnsupportedOperationException(); > } > > > http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringArrayList.java > ---------------------------------------------------------------------- > diff --git > a/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringArrayList.java > > b/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringArrayList.java > new file mode 100644 > index 0000000..bc378a1 > --- /dev/null > +++ > b/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringArrayList.java > @@ -0,0 +1,46 @@ > +/** > + * Licensed to the Apache Software Foundation (ASF) under one > + * or more contributor license agreements. See the NOTICE file > + * distributed with this work for additional information > + * regarding copyright ownership. The ASF licenses this file > + * to you under the Apache License, Version 2.0 (the > + * "License"); you may not use this file except in compliance > + * with the License. You may obtain a copy of the License at > + * > + * http://www.apache.org/licenses/LICENSE-2.0 > + * > + * Unless required by applicable law or agreed to in writing, software > + * distributed under the License is distributed on an "AS IS" BASIS, > + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. > + * See the License for the specific language governing permissions and > + * limitations under the License. > + */ > +package org.apache.drill.exec.util; > + > +import com.fasterxml.jackson.core.JsonProcessingException; > +import com.fasterxml.jackson.databind.ObjectMapper; > +import com.fasterxml.jackson.databind.module.SimpleModule; > +import org.apache.hadoop.io.Text; > + > +import java.util.ArrayList; > + > +public class JsonStringArrayList extends ArrayList { > + > + private static ObjectMapper mapper; > + > + static { > + mapper = new ObjectMapper(); > + SimpleModule serializer = new SimpleModule("TextSerializer") > + .addSerializer(Text.class, new TextSerializer()); > + mapper.registerModule(serializer); > + } > + > + @Override > + public final String toString() { > + try { > + return mapper.writeValueAsString(this); > + } catch(JsonProcessingException e) { > + throw new IllegalStateException("Cannot serialize array list to JSON > string", e); > + } > + } > +} > > http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringHashMap.java > ---------------------------------------------------------------------- > diff --git > a/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringHashMap.java > > b/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringHashMap.java > new file mode 100644 > index 0000000..124252a > --- /dev/null > +++ > b/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringHashMap.java > @@ -0,0 +1,50 @@ > +/** > + * Licensed to the Apache Software Foundation (ASF) under one > + * or more contributor license agreements. See the NOTICE file > + * distributed with this work for additional information > + * regarding copyright ownership. The ASF licenses this file > + * to you under the Apache License, Version 2.0 (the > + * "License"); you may not use this file except in compliance > + * with the License. You may obtain a copy of the License at > + * > + * http://www.apache.org/licenses/LICENSE-2.0 > + * > + * Unless required by applicable law or agreed to in writing, software > + * distributed under the License is distributed on an "AS IS" BASIS, > + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. > + * See the License for the specific language governing permissions and > + * limitations under the License. > + */ > +package org.apache.drill.exec.util; > + > +import com.fasterxml.jackson.core.JsonProcessingException; > +import com.fasterxml.jackson.databind.ObjectMapper; > +import com.fasterxml.jackson.databind.module.SimpleModule; > +import org.apache.hadoop.io.Text; > + > +import java.util.HashMap; > + > +/* > + * Simple class that extends the regular java.util.HashMap but overrides the > + * toString() method of the HashMap class to produce a JSON string instead > + */ > +public class JsonStringHashMap extends HashMap { > + > + private static ObjectMapper mapper; > + > + static { > + mapper = new ObjectMapper(); > + SimpleModule serializer = new SimpleModule("TextSerializer") > + .addSerializer(Text.class, new TextSerializer()); > + mapper.registerModule(serializer); > + } > + > + @Override > + public final String toString() { > + try { > + return mapper.writeValueAsString(this); > + } catch(JsonProcessingException e) { > + throw new IllegalStateException("Cannot serialize hash map to JSON > string", e); > + } > + } > +} > > http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/java/org/apache/drill/exec/util/TextSerializer.java > ---------------------------------------------------------------------- > diff --git > a/exec/java-exec/src/main/java/org/apache/drill/exec/util/TextSerializer.java > b/exec/java-exec/src/main/java/org/apache/drill/exec/util/TextSerializer.java > new file mode 100644 > index 0000000..3070592 > --- /dev/null > +++ > b/exec/java-exec/src/main/java/org/apache/drill/exec/util/TextSerializer.java > @@ -0,0 +1,38 @@ > +/** > + * Licensed to the Apache Software Foundation (ASF) under one > + * or more contributor license agreements. See the NOTICE file > + * distributed with this work for additional information > + * regarding copyright ownership. The ASF licenses this file > + * to you under the Apache License, Version 2.0 (the > + * "License"); you may not use this file except in compliance > + * with the License. You may obtain a copy of the License at > + * > + * http://www.apache.org/licenses/LICENSE-2.0 > + * > + * Unless required by applicable law or agreed to in writing, software > + * distributed under the License is distributed on an "AS IS" BASIS, > + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. > + * See the License for the specific language governing permissions and > + * limitations under the License. > + */ > +package org.apache.drill.exec.util; > + > +import com.fasterxml.jackson.core.JsonGenerationException; > +import com.fasterxml.jackson.core.JsonGenerator; > +import com.fasterxml.jackson.databind.SerializerProvider; > +import com.fasterxml.jackson.databind.ser.std.StdSerializer; > +import org.apache.hadoop.io.Text; > + > +import java.io.IOException; > + > +public class TextSerializer extends StdSerializer<Text> { > + > + public TextSerializer() { > + super(Text.class); > + } > + > + @Override > + public void serialize(Text text, JsonGenerator jsonGenerator, > SerializerProvider serializerProvider) throws IOException, > JsonGenerationException { > + jsonGenerator.writeString(text.toString()); > + } > +} > > http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java > ---------------------------------------------------------------------- > diff --git > a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java > > b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java > index 91c0be5..9b2e119 100644 > --- > a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java > +++ > b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java > @@ -36,6 +36,7 @@ import > org.apache.drill.exec.memory.OutOfMemoryRuntimeException; > import org.apache.drill.exec.proto.UserBitShared.SerializedField; > import org.apache.drill.exec.record.MaterializedField; > import org.apache.drill.exec.record.TransferPair; > +import org.apache.drill.exec.util.JsonStringHashMap; > import org.apache.drill.exec.vector.ValueVector; > import > org.apache.drill.exec.vector.complex.RepeatedMapVector.MapSingleCopier; > import org.apache.drill.exec.vector.complex.impl.SingleMapReaderImpl; > @@ -316,7 +317,7 @@ public class MapVector extends AbstractContainerVector { > > @Override > public Object getObject(int index) { > - Map<String, Object> vv = Maps.newHashMap(); > + Map<String, Object> vv = new JsonStringHashMap(); > for(Map.Entry<String, ValueVector> e : vectors.entrySet()){ > ValueVector v = e.getValue(); > String k = e.getKey(); > > http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java > ---------------------------------------------------------------------- > diff --git > a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java > > b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java > index 93930b5..88d858a 100644 > --- > a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java > +++ > b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java > @@ -37,6 +37,7 @@ import > org.apache.drill.exec.memory.OutOfMemoryRuntimeException; > import org.apache.drill.exec.proto.UserBitShared.SerializedField; > import org.apache.drill.exec.record.MaterializedField; > import org.apache.drill.exec.record.TransferPair; > +import org.apache.drill.exec.util.JsonStringArrayList; > import org.apache.drill.exec.vector.RepeatedFixedWidthVector; > import org.apache.drill.exec.vector.UInt4Vector; > import org.apache.drill.exec.vector.ValueVector; > @@ -155,7 +156,7 @@ public class RepeatedListVector extends > AbstractContainerVector implements Repea > > @Override > public Object getObject(int index) { > - List<Object> l = Lists.newArrayList(); > + List<Object> l = new JsonStringArrayList(); > int end = offsets.getAccessor().get(index+1); > for(int i = offsets.getAccessor().get(index); i < end; i++){ > l.add(vector.getAccessor().getObject(i)); > > http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java > ---------------------------------------------------------------------- > diff --git > a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java > > b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java > index 2492cc8..22471f0 100644 > --- > a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java > +++ > b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java > @@ -36,6 +36,7 @@ import > org.apache.drill.exec.memory.OutOfMemoryRuntimeException; > import org.apache.drill.exec.proto.UserBitShared.SerializedField; > import org.apache.drill.exec.record.MaterializedField; > import org.apache.drill.exec.record.TransferPair; > +import org.apache.drill.exec.util.JsonStringArrayList; > import org.apache.drill.exec.vector.RepeatedFixedWidthVector; > import org.apache.drill.exec.vector.UInt4Vector; > import org.apache.drill.exec.vector.ValueVector; > @@ -348,7 +349,7 @@ public class RepeatedMapVector extends > AbstractContainerVector implements Repeat > > @Override > public Object getObject(int index) { > - List<Object> l = Lists.newArrayList(); > + List<Object> l = new JsonStringArrayList(); > int end = offsets.getAccessor().get(index+1); > for(int i = offsets.getAccessor().get(index); i < end; i++){ > Map<String, Object> vv = Maps.newHashMap(); > > http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java > ---------------------------------------------------------------------- > diff --git > a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java > > b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java > index 761bc79..e5d4b5b 100644 > --- > a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java > +++ > b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java > @@ -221,7 +221,7 @@ public class JsonReader { > vh.start = 0; > vh.end = b.length; > list.varChar().write(vh); > - > + break; > default: > throw new IllegalStateException("Unexpected token " + > parser.getCurrentToken()); > } > > http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JsonRecordReader2Test.java > ---------------------------------------------------------------------- > diff --git > a/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JsonRecordReader2Test.java > > b/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JsonRecordReader2Test.java > index 1372219..ad46326 100644 > --- > a/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JsonRecordReader2Test.java > +++ > b/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JsonRecordReader2Test.java > @@ -27,6 +27,8 @@ public class JsonRecordReader2Test extends BaseTestQuery{ > public void testComplexJsonInput() throws Exception{ > // test("select z[0]['orange'] from cp.`jsoninput/input2.json` limit 10"); > test("select `integer`, x['y'] as x1, x['y'] as x2, z[0], > z[0]['orange'], z[1]['pink'] from cp.`jsoninput/input2.json` limit 10"); > + test("select x from cp.`jsoninput/input2.json`"); > + > // test("select z[0] from cp.`jsoninput/input2.json` limit 10"); > } > >
