Added: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestDeepParquetHiveMapInspector.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestDeepParquetHiveMapInspector.java?rev=1566173&view=auto ============================================================================== --- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestDeepParquetHiveMapInspector.java (added) +++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestDeepParquetHiveMapInspector.java Sun Feb 9 00:23:49 2014 @@ -0,0 +1,90 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.io.parquet.serde; + +import java.util.HashMap; +import java.util.Map; + +import junit.framework.TestCase; + +import org.apache.hadoop.hive.ql.io.parquet.serde.primitive.ParquetPrimitiveInspectorFactory; +import org.apache.hadoop.hive.serde2.io.ShortWritable; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.io.ArrayWritable; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.Writable; +import org.junit.Test; + +public class TestDeepParquetHiveMapInspector extends TestCase { + + private DeepParquetHiveMapInspector inspector; + + @Override + public void setUp() { + inspector = new DeepParquetHiveMapInspector(ParquetPrimitiveInspectorFactory.parquetShortInspector, + PrimitiveObjectInspectorFactory.javaIntObjectInspector); + } + + @Test + public void testNullMap() { + assertNull("Should be null", inspector.getMapValueElement(null, new ShortWritable((short) 0))); + } + + @Test + public void testNullContainer() { + final ArrayWritable map = new ArrayWritable(ArrayWritable.class, null); + assertNull("Should be null", inspector.getMapValueElement(map, new ShortWritable((short) 0))); + } + + @Test + public void testEmptyContainer() { + final ArrayWritable map = new ArrayWritable(ArrayWritable.class, new ArrayWritable[0]); + assertNull("Should be null", inspector.getMapValueElement(map, new ShortWritable((short) 0))); + } + + @Test + public void testRegularMap() { + final Writable[] entry1 = new Writable[]{new IntWritable(0), new IntWritable(1)}; + final Writable[] entry2 = new Writable[]{new IntWritable(2), new IntWritable(3)}; + + final ArrayWritable internalMap = new ArrayWritable(ArrayWritable.class, new Writable[]{ + new ArrayWritable(Writable.class, entry1), new ArrayWritable(Writable.class, entry2)}); + + final ArrayWritable map = new ArrayWritable(ArrayWritable.class, new Writable[]{internalMap}); + + assertEquals("Wrong result of inspection", new IntWritable(1), inspector.getMapValueElement(map, new IntWritable(0))); + assertEquals("Wrong result of inspection", new IntWritable(3), inspector.getMapValueElement(map, new IntWritable(2))); + assertEquals("Wrong result of inspection", new IntWritable(1), inspector.getMapValueElement(map, new ShortWritable((short) 0))); + assertEquals("Wrong result of inspection", new IntWritable(3), inspector.getMapValueElement(map, new ShortWritable((short) 2))); + } + + @Test + public void testHashMap() { + final Map<Writable, Writable> map = new HashMap<Writable, Writable>(); + map.put(new IntWritable(0), new IntWritable(1)); + map.put(new IntWritable(2), new IntWritable(3)); + map.put(new IntWritable(4), new IntWritable(5)); + map.put(new IntWritable(6), new IntWritable(7)); + + + assertEquals("Wrong result of inspection", new IntWritable(1), inspector.getMapValueElement(map, new IntWritable(0))); + assertEquals("Wrong result of inspection", new IntWritable(3), inspector.getMapValueElement(map, new IntWritable(2))); + assertEquals("Wrong result of inspection", new IntWritable(5), inspector.getMapValueElement(map, new IntWritable(4))); + assertEquals("Wrong result of inspection", new IntWritable(7), inspector.getMapValueElement(map, new IntWritable(6))); + assertEquals("Wrong result of inspection", new IntWritable(1), inspector.getMapValueElement(map, new ShortWritable((short) 0))); + assertEquals("Wrong result of inspection", new IntWritable(3), inspector.getMapValueElement(map, new ShortWritable((short) 2))); + assertEquals("Wrong result of inspection", new IntWritable(5), inspector.getMapValueElement(map, new ShortWritable((short) 4))); + assertEquals("Wrong result of inspection", new IntWritable(7), inspector.getMapValueElement(map, new ShortWritable((short) 6))); + } +}
Added: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestParquetHiveArrayInspector.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestParquetHiveArrayInspector.java?rev=1566173&view=auto ============================================================================== --- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestParquetHiveArrayInspector.java (added) +++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestParquetHiveArrayInspector.java Sun Feb 9 00:23:49 2014 @@ -0,0 +1,80 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.io.parquet.serde; + +import java.util.ArrayList; +import java.util.List; + +import junit.framework.TestCase; + +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.io.ArrayWritable; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.Writable; +import org.junit.Test; + +public class TestParquetHiveArrayInspector extends TestCase { + + private ParquetHiveArrayInspector inspector; + + @Override + public void setUp() { + inspector = new ParquetHiveArrayInspector(PrimitiveObjectInspectorFactory.javaIntObjectInspector); + } + + @Test + public void testNullArray() { + assertEquals("Wrong size", -1, inspector.getListLength(null)); + assertNull("Should be null", inspector.getList(null)); + assertNull("Should be null", inspector.getListElement(null, 0)); + } + + @Test + public void testNullContainer() { + final ArrayWritable list = new ArrayWritable(ArrayWritable.class, null); + assertEquals("Wrong size", -1, inspector.getListLength(list)); + assertNull("Should be null", inspector.getList(list)); + assertNull("Should be null", inspector.getListElement(list, 0)); + } + + @Test + public void testEmptyContainer() { + final ArrayWritable list = new ArrayWritable(ArrayWritable.class, new ArrayWritable[0]); + assertEquals("Wrong size", -1, inspector.getListLength(list)); + assertNull("Should be null", inspector.getList(list)); + assertNull("Should be null", inspector.getListElement(list, 0)); + } + + @Test + public void testRegularList() { + final ArrayWritable internalList = new ArrayWritable(Writable.class, + new Writable[]{new IntWritable(3), new IntWritable(5), new IntWritable(1)}); + final ArrayWritable list = new ArrayWritable(ArrayWritable.class, new ArrayWritable[]{internalList}); + + final List<Writable> expected = new ArrayList<Writable>(); + expected.add(new IntWritable(3)); + expected.add(new IntWritable(5)); + expected.add(new IntWritable(1)); + + assertEquals("Wrong size", 3, inspector.getListLength(list)); + assertEquals("Wrong result of inspection", expected, inspector.getList(list)); + + for (int i = 0; i < expected.size(); ++i) { + assertEquals("Wrong result of inspection", expected.get(i), inspector.getListElement(list, i)); + + } + + assertNull("Should be null", inspector.getListElement(list, 3)); + } +} Added: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestStandardParquetHiveMapInspector.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestStandardParquetHiveMapInspector.java?rev=1566173&view=auto ============================================================================== --- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestStandardParquetHiveMapInspector.java (added) +++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestStandardParquetHiveMapInspector.java Sun Feb 9 00:23:49 2014 @@ -0,0 +1,88 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.io.parquet.serde; + +import java.util.HashMap; +import java.util.Map; + +import junit.framework.TestCase; + +import org.apache.hadoop.hive.serde2.io.ShortWritable; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.io.ArrayWritable; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.Writable; +import org.junit.Test; + +public class TestStandardParquetHiveMapInspector extends TestCase { + + private StandardParquetHiveMapInspector inspector; + + @Override + public void setUp() { + inspector = new StandardParquetHiveMapInspector(PrimitiveObjectInspectorFactory.javaIntObjectInspector, + PrimitiveObjectInspectorFactory.javaIntObjectInspector); + } + + @Test + public void testNullMap() { + assertNull("Should be null", inspector.getMapValueElement(null, new IntWritable(0))); + } + + @Test + public void testNullContainer() { + final ArrayWritable map = new ArrayWritable(ArrayWritable.class, null); + assertNull("Should be null", inspector.getMapValueElement(map, new IntWritable(0))); + } + + @Test + public void testEmptyContainer() { + final ArrayWritable map = new ArrayWritable(ArrayWritable.class, new ArrayWritable[0]); + assertNull("Should be null", inspector.getMapValueElement(map, new IntWritable(0))); + } + + @Test + public void testRegularMap() { + final Writable[] entry1 = new Writable[]{new IntWritable(0), new IntWritable(1)}; + final Writable[] entry2 = new Writable[]{new IntWritable(2), new IntWritable(3)}; + + final ArrayWritable internalMap = new ArrayWritable(ArrayWritable.class, new Writable[]{ + new ArrayWritable(Writable.class, entry1), new ArrayWritable(Writable.class, entry2)}); + + final ArrayWritable map = new ArrayWritable(ArrayWritable.class, new Writable[]{internalMap}); + + assertEquals("Wrong result of inspection", new IntWritable(1), inspector.getMapValueElement(map, new IntWritable(0))); + assertEquals("Wrong result of inspection", new IntWritable(3), inspector.getMapValueElement(map, new IntWritable(2))); + assertNull("Wrong result of inspection", inspector.getMapValueElement(map, new ShortWritable((short) 0))); + assertNull("Wrong result of inspection", inspector.getMapValueElement(map, new ShortWritable((short) 2))); + } + + @Test + public void testHashMap() { + final Map<Writable, Writable> map = new HashMap<Writable, Writable>(); + map.put(new IntWritable(0), new IntWritable(1)); + map.put(new IntWritable(2), new IntWritable(3)); + map.put(new IntWritable(4), new IntWritable(5)); + map.put(new IntWritable(6), new IntWritable(7)); + + assertEquals("Wrong result of inspection", new IntWritable(1), inspector.getMapValueElement(map, new IntWritable(0))); + assertEquals("Wrong result of inspection", new IntWritable(3), inspector.getMapValueElement(map, new IntWritable(2))); + assertEquals("Wrong result of inspection", new IntWritable(5), inspector.getMapValueElement(map, new IntWritable(4))); + assertEquals("Wrong result of inspection", new IntWritable(7), inspector.getMapValueElement(map, new IntWritable(6))); + assertNull("Wrong result of inspection", inspector.getMapValueElement(map, new ShortWritable((short) 0))); + assertNull("Wrong result of inspection", inspector.getMapValueElement(map, new ShortWritable((short) 2))); + assertNull("Wrong result of inspection", inspector.getMapValueElement(map, new ShortWritable((short) 4))); + assertNull("Wrong result of inspection", inspector.getMapValueElement(map, new ShortWritable((short) 6))); + } +} Added: hive/trunk/ql/src/test/queries/clientpositive/parquet_create.q URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/parquet_create.q?rev=1566173&view=auto ============================================================================== --- hive/trunk/ql/src/test/queries/clientpositive/parquet_create.q (added) +++ hive/trunk/ql/src/test/queries/clientpositive/parquet_create.q Sun Feb 9 00:23:49 2014 @@ -0,0 +1,36 @@ +DROP TABLE parquet_create_staging; +DROP TABLE parquet_create; + +CREATE TABLE parquet_create_staging ( + id int, + str string, + mp MAP<STRING,STRING>, + lst ARRAY<STRING>, + strct STRUCT<A:STRING,B:STRING> +) ROW FORMAT DELIMITED +FIELDS TERMINATED BY '|' +COLLECTION ITEMS TERMINATED BY ',' +MAP KEYS TERMINATED BY ':'; + +CREATE TABLE parquet_create ( + id int, + str string, + mp MAP<STRING,STRING>, + lst ARRAY<STRING>, + strct STRUCT<A:STRING,B:STRING> +) STORED AS PARQUET; + +DESCRIBE FORMATTED parquet_create; + +LOAD DATA LOCAL INPATH '../../data/files/parquet_create.txt' OVERWRITE INTO TABLE parquet_create_staging; + +SELECT * FROM parquet_create_staging; + +INSERT OVERWRITE TABLE parquet_create SELECT * FROM parquet_create_staging; + +SELECT * FROM parquet_create group by id; +SELECT id, count(0) FROM parquet_create group by id; +SELECT str from parquet_create; +SELECT mp from parquet_create; +SELECT lst from parquet_create; +SELECT strct from parquet_create; Added: hive/trunk/ql/src/test/queries/clientpositive/parquet_partitioned.q URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/parquet_partitioned.q?rev=1566173&view=auto ============================================================================== --- hive/trunk/ql/src/test/queries/clientpositive/parquet_partitioned.q (added) +++ hive/trunk/ql/src/test/queries/clientpositive/parquet_partitioned.q Sun Feb 9 00:23:49 2014 @@ -0,0 +1,34 @@ +set hive.exec.dynamic.partition.mode=nonstrict; +set hive.exec.dynamic.partition=true; + +DROP TABLE parquet_partitioned_staging; +DROP TABLE parquet_partitioned; + +CREATE TABLE parquet_partitioned_staging ( + id int, + str string, + part string +) ROW FORMAT DELIMITED +FIELDS TERMINATED BY '|'; + +CREATE TABLE parquet_partitioned ( + id int, + str string +) PARTITIONED BY (part string) +STORED AS PARQUET; + +DESCRIBE FORMATTED parquet_partitioned; + +LOAD DATA LOCAL INPATH '../../data/files/parquet_partitioned.txt' OVERWRITE INTO TABLE parquet_partitioned_staging; + +SELECT * FROM parquet_partitioned_staging; + +INSERT OVERWRITE TABLE parquet_partitioned PARTITION (part) SELECT * FROM parquet_partitioned_staging; + +set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; +SELECT * FROM parquet_partitioned; +SELECT part, COUNT(0) FROM parquet_partitioned GROUP BY part; + +set hive.input.format=org.apache.hadoop.hive.ql.io.CombineHiveInputFormat; +SELECT * FROM parquet_partitioned; +SELECT part, COUNT(0) FROM parquet_partitioned GROUP BY part; Modified: hive/trunk/ql/src/test/results/clientnegative/authorization_invalid_priv_v1.q.out URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientnegative/authorization_invalid_priv_v1.q.out?rev=1566173&r1=1566172&r2=1566173&view=diff ============================================================================== --- hive/trunk/ql/src/test/results/clientnegative/authorization_invalid_priv_v1.q.out (original) +++ hive/trunk/ql/src/test/results/clientnegative/authorization_invalid_priv_v1.q.out Sun Feb 9 00:23:49 2014 @@ -3,4 +3,4 @@ PREHOOK: type: CREATETABLE POSTHOOK: query: create table if not exists authorization_invalid_v1 (key int, value string) POSTHOOK: type: CREATETABLE POSTHOOK: Output: default@authorization_invalid_v1 -FAILED: SemanticException undefined privilege TOK_PRIV_DELETE +FAILED: SemanticException Undefined privilege Delete Added: hive/trunk/ql/src/test/results/clientpositive/parquet_create.q.out URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/parquet_create.q.out?rev=1566173&view=auto ============================================================================== --- hive/trunk/ql/src/test/results/clientpositive/parquet_create.q.out (added) +++ hive/trunk/ql/src/test/results/clientpositive/parquet_create.q.out Sun Feb 9 00:23:49 2014 @@ -0,0 +1,206 @@ +PREHOOK: query: DROP TABLE parquet_create_staging +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE parquet_create_staging +POSTHOOK: type: DROPTABLE +PREHOOK: query: DROP TABLE parquet_create +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE parquet_create +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE parquet_create_staging ( + id int, + str string, + mp MAP<STRING,STRING>, + lst ARRAY<STRING>, + strct STRUCT<A:STRING,B:STRING> +) ROW FORMAT DELIMITED +FIELDS TERMINATED BY '|' +COLLECTION ITEMS TERMINATED BY ',' +MAP KEYS TERMINATED BY ':' +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE parquet_create_staging ( + id int, + str string, + mp MAP<STRING,STRING>, + lst ARRAY<STRING>, + strct STRUCT<A:STRING,B:STRING> +) ROW FORMAT DELIMITED +FIELDS TERMINATED BY '|' +COLLECTION ITEMS TERMINATED BY ',' +MAP KEYS TERMINATED BY ':' +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@parquet_create_staging +PREHOOK: query: CREATE TABLE parquet_create ( + id int, + str string, + mp MAP<STRING,STRING>, + lst ARRAY<STRING>, + strct STRUCT<A:STRING,B:STRING> +) STORED AS PARQUET +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE parquet_create ( + id int, + str string, + mp MAP<STRING,STRING>, + lst ARRAY<STRING>, + strct STRUCT<A:STRING,B:STRING> +) STORED AS PARQUET +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@parquet_create +PREHOOK: query: DESCRIBE FORMATTED parquet_create +PREHOOK: type: DESCTABLE +POSTHOOK: query: DESCRIBE FORMATTED parquet_create +POSTHOOK: type: DESCTABLE +# col_name data_type comment + +id int from deserializer +str string from deserializer +mp map<string,string> from deserializer +lst array<string> from deserializer +strct struct<A:string,B:string> from deserializer + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe +InputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/parquet_create.txt' OVERWRITE INTO TABLE parquet_create_staging +PREHOOK: type: LOAD +PREHOOK: Output: default@parquet_create_staging +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/parquet_create.txt' OVERWRITE INTO TABLE parquet_create_staging +POSTHOOK: type: LOAD +POSTHOOK: Output: default@parquet_create_staging +PREHOOK: query: SELECT * FROM parquet_create_staging +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_create_staging +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM parquet_create_staging +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_create_staging +#### A masked pattern was here #### +1 foo line1 {"key11":"value11","key12":"value12","key13":"value13"} ["a","b","c"] {"a":"one","b":"two"} +2 bar line2 {"key21":"value21","key22":"value22","key23":"value23"} ["d","e","f"] {"a":"three","b":"four"} +3 baz line3 {"key31":"value31","key32":"value32","key33":"value33"} ["g","h","i"] {"a":"five","b":"six"} +PREHOOK: query: INSERT OVERWRITE TABLE parquet_create SELECT * FROM parquet_create_staging +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_create_staging +PREHOOK: Output: default@parquet_create +POSTHOOK: query: INSERT OVERWRITE TABLE parquet_create SELECT * FROM parquet_create_staging +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_create_staging +POSTHOOK: Output: default@parquet_create +POSTHOOK: Lineage: parquet_create.id SIMPLE [(parquet_create_staging)parquet_create_staging.FieldSchema(name:id, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_create.lst SIMPLE [(parquet_create_staging)parquet_create_staging.FieldSchema(name:lst, type:array<string>, comment:null), ] +POSTHOOK: Lineage: parquet_create.mp SIMPLE [(parquet_create_staging)parquet_create_staging.FieldSchema(name:mp, type:map<string,string>, comment:null), ] +POSTHOOK: Lineage: parquet_create.str SIMPLE [(parquet_create_staging)parquet_create_staging.FieldSchema(name:str, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_create.strct SIMPLE [(parquet_create_staging)parquet_create_staging.FieldSchema(name:strct, type:struct<A:string,B:string>, comment:null), ] +PREHOOK: query: SELECT * FROM parquet_create group by id +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_create +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM parquet_create group by id +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_create +#### A masked pattern was here #### +POSTHOOK: Lineage: parquet_create.id SIMPLE [(parquet_create_staging)parquet_create_staging.FieldSchema(name:id, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_create.lst SIMPLE [(parquet_create_staging)parquet_create_staging.FieldSchema(name:lst, type:array<string>, comment:null), ] +POSTHOOK: Lineage: parquet_create.mp SIMPLE [(parquet_create_staging)parquet_create_staging.FieldSchema(name:mp, type:map<string,string>, comment:null), ] +POSTHOOK: Lineage: parquet_create.str SIMPLE [(parquet_create_staging)parquet_create_staging.FieldSchema(name:str, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_create.strct SIMPLE [(parquet_create_staging)parquet_create_staging.FieldSchema(name:strct, type:struct<A:string,B:string>, comment:null), ] +1 +2 +3 +PREHOOK: query: SELECT id, count(0) FROM parquet_create group by id +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_create +#### A masked pattern was here #### +POSTHOOK: query: SELECT id, count(0) FROM parquet_create group by id +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_create +#### A masked pattern was here #### +POSTHOOK: Lineage: parquet_create.id SIMPLE [(parquet_create_staging)parquet_create_staging.FieldSchema(name:id, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_create.lst SIMPLE [(parquet_create_staging)parquet_create_staging.FieldSchema(name:lst, type:array<string>, comment:null), ] +POSTHOOK: Lineage: parquet_create.mp SIMPLE [(parquet_create_staging)parquet_create_staging.FieldSchema(name:mp, type:map<string,string>, comment:null), ] +POSTHOOK: Lineage: parquet_create.str SIMPLE [(parquet_create_staging)parquet_create_staging.FieldSchema(name:str, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_create.strct SIMPLE [(parquet_create_staging)parquet_create_staging.FieldSchema(name:strct, type:struct<A:string,B:string>, comment:null), ] +1 1 +2 1 +3 1 +PREHOOK: query: SELECT str from parquet_create +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_create +#### A masked pattern was here #### +POSTHOOK: query: SELECT str from parquet_create +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_create +#### A masked pattern was here #### +POSTHOOK: Lineage: parquet_create.id SIMPLE [(parquet_create_staging)parquet_create_staging.FieldSchema(name:id, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_create.lst SIMPLE [(parquet_create_staging)parquet_create_staging.FieldSchema(name:lst, type:array<string>, comment:null), ] +POSTHOOK: Lineage: parquet_create.mp SIMPLE [(parquet_create_staging)parquet_create_staging.FieldSchema(name:mp, type:map<string,string>, comment:null), ] +POSTHOOK: Lineage: parquet_create.str SIMPLE [(parquet_create_staging)parquet_create_staging.FieldSchema(name:str, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_create.strct SIMPLE [(parquet_create_staging)parquet_create_staging.FieldSchema(name:strct, type:struct<A:string,B:string>, comment:null), ] +foo line1 +bar line2 +baz line3 +PREHOOK: query: SELECT mp from parquet_create +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_create +#### A masked pattern was here #### +POSTHOOK: query: SELECT mp from parquet_create +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_create +#### A masked pattern was here #### +POSTHOOK: Lineage: parquet_create.id SIMPLE [(parquet_create_staging)parquet_create_staging.FieldSchema(name:id, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_create.lst SIMPLE [(parquet_create_staging)parquet_create_staging.FieldSchema(name:lst, type:array<string>, comment:null), ] +POSTHOOK: Lineage: parquet_create.mp SIMPLE [(parquet_create_staging)parquet_create_staging.FieldSchema(name:mp, type:map<string,string>, comment:null), ] +POSTHOOK: Lineage: parquet_create.str SIMPLE [(parquet_create_staging)parquet_create_staging.FieldSchema(name:str, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_create.strct SIMPLE [(parquet_create_staging)parquet_create_staging.FieldSchema(name:strct, type:struct<A:string,B:string>, comment:null), ] +{"key12":"value12","key11":"value11","key13":"value13"} +{"key21":"value21","key23":"value23","key22":"value22"} +{"key33":"value33","key31":"value31","key32":"value32"} +PREHOOK: query: SELECT lst from parquet_create +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_create +#### A masked pattern was here #### +POSTHOOK: query: SELECT lst from parquet_create +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_create +#### A masked pattern was here #### +POSTHOOK: Lineage: parquet_create.id SIMPLE [(parquet_create_staging)parquet_create_staging.FieldSchema(name:id, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_create.lst SIMPLE [(parquet_create_staging)parquet_create_staging.FieldSchema(name:lst, type:array<string>, comment:null), ] +POSTHOOK: Lineage: parquet_create.mp SIMPLE [(parquet_create_staging)parquet_create_staging.FieldSchema(name:mp, type:map<string,string>, comment:null), ] +POSTHOOK: Lineage: parquet_create.str SIMPLE [(parquet_create_staging)parquet_create_staging.FieldSchema(name:str, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_create.strct SIMPLE [(parquet_create_staging)parquet_create_staging.FieldSchema(name:strct, type:struct<A:string,B:string>, comment:null), ] +["a","b","c"] +["d","e","f"] +["g","h","i"] +PREHOOK: query: SELECT strct from parquet_create +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_create +#### A masked pattern was here #### +POSTHOOK: query: SELECT strct from parquet_create +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_create +#### A masked pattern was here #### +POSTHOOK: Lineage: parquet_create.id SIMPLE [(parquet_create_staging)parquet_create_staging.FieldSchema(name:id, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_create.lst SIMPLE [(parquet_create_staging)parquet_create_staging.FieldSchema(name:lst, type:array<string>, comment:null), ] +POSTHOOK: Lineage: parquet_create.mp SIMPLE [(parquet_create_staging)parquet_create_staging.FieldSchema(name:mp, type:map<string,string>, comment:null), ] +POSTHOOK: Lineage: parquet_create.str SIMPLE [(parquet_create_staging)parquet_create_staging.FieldSchema(name:str, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_create.strct SIMPLE [(parquet_create_staging)parquet_create_staging.FieldSchema(name:strct, type:struct<A:string,B:string>, comment:null), ] +{"a":"one","b":"two"} +{"a":"three","b":"four"} +{"a":"five","b":"six"} Added: hive/trunk/ql/src/test/results/clientpositive/parquet_partitioned.q.out URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/parquet_partitioned.q.out?rev=1566173&view=auto ============================================================================== --- hive/trunk/ql/src/test/results/clientpositive/parquet_partitioned.q.out (added) +++ hive/trunk/ql/src/test/results/clientpositive/parquet_partitioned.q.out Sun Feb 9 00:23:49 2014 @@ -0,0 +1,174 @@ +PREHOOK: query: DROP TABLE parquet_partitioned_staging +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE parquet_partitioned_staging +POSTHOOK: type: DROPTABLE +PREHOOK: query: DROP TABLE parquet_partitioned +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE parquet_partitioned +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE parquet_partitioned_staging ( + id int, + str string, + part string +) ROW FORMAT DELIMITED +FIELDS TERMINATED BY '|' +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE parquet_partitioned_staging ( + id int, + str string, + part string +) ROW FORMAT DELIMITED +FIELDS TERMINATED BY '|' +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@parquet_partitioned_staging +PREHOOK: query: CREATE TABLE parquet_partitioned ( + id int, + str string +) PARTITIONED BY (part string) +STORED AS PARQUET +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE parquet_partitioned ( + id int, + str string +) PARTITIONED BY (part string) +STORED AS PARQUET +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@parquet_partitioned +PREHOOK: query: DESCRIBE FORMATTED parquet_partitioned +PREHOOK: type: DESCTABLE +POSTHOOK: query: DESCRIBE FORMATTED parquet_partitioned +POSTHOOK: type: DESCTABLE +# col_name data_type comment + +id int from deserializer +str string from deserializer + +# Partition Information +# col_name data_type comment + +part string None + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe +InputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/parquet_partitioned.txt' OVERWRITE INTO TABLE parquet_partitioned_staging +PREHOOK: type: LOAD +PREHOOK: Output: default@parquet_partitioned_staging +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/parquet_partitioned.txt' OVERWRITE INTO TABLE parquet_partitioned_staging +POSTHOOK: type: LOAD +POSTHOOK: Output: default@parquet_partitioned_staging +PREHOOK: query: SELECT * FROM parquet_partitioned_staging +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_partitioned_staging +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM parquet_partitioned_staging +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_partitioned_staging +#### A masked pattern was here #### +1 foo part1 +2 bar part2 +3 baz part2 +PREHOOK: query: INSERT OVERWRITE TABLE parquet_partitioned PARTITION (part) SELECT * FROM parquet_partitioned_staging +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_partitioned_staging +PREHOOK: Output: default@parquet_partitioned +POSTHOOK: query: INSERT OVERWRITE TABLE parquet_partitioned PARTITION (part) SELECT * FROM parquet_partitioned_staging +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_partitioned_staging +POSTHOOK: Output: default@parquet_partitioned@part=part1 +POSTHOOK: Output: default@parquet_partitioned@part=part2 +POSTHOOK: Lineage: parquet_partitioned PARTITION(part=part1).id SIMPLE [(parquet_partitioned_staging)parquet_partitioned_staging.FieldSchema(name:id, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_partitioned PARTITION(part=part1).str SIMPLE [(parquet_partitioned_staging)parquet_partitioned_staging.FieldSchema(name:str, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_partitioned PARTITION(part=part2).id SIMPLE [(parquet_partitioned_staging)parquet_partitioned_staging.FieldSchema(name:id, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_partitioned PARTITION(part=part2).str SIMPLE [(parquet_partitioned_staging)parquet_partitioned_staging.FieldSchema(name:str, type:string, comment:null), ] +PREHOOK: query: SELECT * FROM parquet_partitioned +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_partitioned +PREHOOK: Input: default@parquet_partitioned@part=part1 +PREHOOK: Input: default@parquet_partitioned@part=part2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM parquet_partitioned +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_partitioned +POSTHOOK: Input: default@parquet_partitioned@part=part1 +POSTHOOK: Input: default@parquet_partitioned@part=part2 +#### A masked pattern was here #### +POSTHOOK: Lineage: parquet_partitioned PARTITION(part=part1).id SIMPLE [(parquet_partitioned_staging)parquet_partitioned_staging.FieldSchema(name:id, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_partitioned PARTITION(part=part1).str SIMPLE [(parquet_partitioned_staging)parquet_partitioned_staging.FieldSchema(name:str, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_partitioned PARTITION(part=part2).id SIMPLE [(parquet_partitioned_staging)parquet_partitioned_staging.FieldSchema(name:id, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_partitioned PARTITION(part=part2).str SIMPLE [(parquet_partitioned_staging)parquet_partitioned_staging.FieldSchema(name:str, type:string, comment:null), ] +1 foo part1 +2 bar part2 +3 baz part2 +PREHOOK: query: SELECT part, COUNT(0) FROM parquet_partitioned GROUP BY part +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_partitioned +PREHOOK: Input: default@parquet_partitioned@part=part1 +PREHOOK: Input: default@parquet_partitioned@part=part2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT part, COUNT(0) FROM parquet_partitioned GROUP BY part +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_partitioned +POSTHOOK: Input: default@parquet_partitioned@part=part1 +POSTHOOK: Input: default@parquet_partitioned@part=part2 +#### A masked pattern was here #### +POSTHOOK: Lineage: parquet_partitioned PARTITION(part=part1).id SIMPLE [(parquet_partitioned_staging)parquet_partitioned_staging.FieldSchema(name:id, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_partitioned PARTITION(part=part1).str SIMPLE [(parquet_partitioned_staging)parquet_partitioned_staging.FieldSchema(name:str, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_partitioned PARTITION(part=part2).id SIMPLE [(parquet_partitioned_staging)parquet_partitioned_staging.FieldSchema(name:id, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_partitioned PARTITION(part=part2).str SIMPLE [(parquet_partitioned_staging)parquet_partitioned_staging.FieldSchema(name:str, type:string, comment:null), ] +part1 1 +part2 2 +PREHOOK: query: SELECT * FROM parquet_partitioned +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_partitioned +PREHOOK: Input: default@parquet_partitioned@part=part1 +PREHOOK: Input: default@parquet_partitioned@part=part2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM parquet_partitioned +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_partitioned +POSTHOOK: Input: default@parquet_partitioned@part=part1 +POSTHOOK: Input: default@parquet_partitioned@part=part2 +#### A masked pattern was here #### +POSTHOOK: Lineage: parquet_partitioned PARTITION(part=part1).id SIMPLE [(parquet_partitioned_staging)parquet_partitioned_staging.FieldSchema(name:id, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_partitioned PARTITION(part=part1).str SIMPLE [(parquet_partitioned_staging)parquet_partitioned_staging.FieldSchema(name:str, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_partitioned PARTITION(part=part2).id SIMPLE [(parquet_partitioned_staging)parquet_partitioned_staging.FieldSchema(name:id, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_partitioned PARTITION(part=part2).str SIMPLE [(parquet_partitioned_staging)parquet_partitioned_staging.FieldSchema(name:str, type:string, comment:null), ] +1 foo part1 +2 bar part2 +3 baz part2 +PREHOOK: query: SELECT part, COUNT(0) FROM parquet_partitioned GROUP BY part +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_partitioned +PREHOOK: Input: default@parquet_partitioned@part=part1 +PREHOOK: Input: default@parquet_partitioned@part=part2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT part, COUNT(0) FROM parquet_partitioned GROUP BY part +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_partitioned +POSTHOOK: Input: default@parquet_partitioned@part=part1 +POSTHOOK: Input: default@parquet_partitioned@part=part2 +#### A masked pattern was here #### +POSTHOOK: Lineage: parquet_partitioned PARTITION(part=part1).id SIMPLE [(parquet_partitioned_staging)parquet_partitioned_staging.FieldSchema(name:id, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_partitioned PARTITION(part=part1).str SIMPLE [(parquet_partitioned_staging)parquet_partitioned_staging.FieldSchema(name:str, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_partitioned PARTITION(part=part2).id SIMPLE [(parquet_partitioned_staging)parquet_partitioned_staging.FieldSchema(name:id, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_partitioned PARTITION(part=part2).str SIMPLE [(parquet_partitioned_staging)parquet_partitioned_staging.FieldSchema(name:str, type:string, comment:null), ] +part1 1 +part2 2
