[ https://issues.apache.org/jira/browse/HIVE-7657?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Arkadiusz Gasior updated HIVE-7657: ----------------------------------- Description: Joining table which contains map of union of 3 types or more is causing serialization issues. Given tables: ag_map_of_uniontype: # col_name data_type comment hash string None custom_fields map<string,uniontype<bigint,double,string,boolean,int,array<uniontype<bigint,double,string,boolean,int>>,map<string,uniontype<bigint,double,string,boolean,int>>>> None ag_simple_join_table: # col_name data_type comment hash string None size int None Query will throw java.lang.RuntimeException: {code} select i.hash, i.custom_fields, b.size from ag_map_of_uniontype i LEFT OUTER JOIN ag_simple_join_table b ON (i.hash = b.hash); {code} {code} org.apache.hadoop.hive.ql.exec.MapJoinOperator: Unxpected exception: java.util.ArrayList cannot be cast to org.apache.hadoop.hive.serde2.objectinspector.UnionObject java.lang.ClassCastException: java.util.ArrayList cannot be cast to org.apache.hadoop.hive.serde2.objectinspector.UnionObject at org.apache.hadoop.hive.serde2.objectinspector.StandardUnionObjectInspector.getTag(StandardUnionObjectInspector.java:91) at org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.serialize(LazySimpleSerDe.java:559) at org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.serialize(LazySimpleSerDe.java:529) at org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.serializeField(LazySimpleSerDe.java:439) at org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.serialize(LazySimpleSerDe.java:423) at org.apache.hadoop.hive.ql.exec.FileSinkOperator.processOp(FileSinkOperator.java:618) at org.apache.hadoop.hive.ql.exec.Operator.process(Operator.java:504) at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:847) at org.apache.hadoop.hive.ql.exec.SelectOperator.processOp(SelectOperator.java:87) at org.apache.hadoop.hive.ql.exec.Operator.process(Operator.java:504) at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:847) at org.apache.hadoop.hive.ql.exec.CommonJoinOperator.genAllOneUniqueJoinObject(CommonJoinOperator.java:671) at org.apache.hadoop.hive.ql.exec.CommonJoinOperator.checkAndGenObject(CommonJoinOperator.java:754) at org.apache.hadoop.hive.ql.exec.MapJoinOperator.processOp(MapJoinOperator.java:222) at org.apache.hadoop.hive.ql.exec.Operator.process(Operator.java:504) at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:847) at org.apache.hadoop.hive.ql.exec.TableScanOperator.processOp(TableScanOperator.java:91) at org.apache.hadoop.hive.ql.exec.Operator.process(Operator.java:504) at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:847) at org.apache.hadoop.hive.ql.exec.MapOperator.process(MapOperator.java:519) at org.apache.hadoop.hive.ql.exec.mr.ExecMapper.map(ExecMapper.java:157) at org.apache.hadoop.mapred.MapRunner.run(MapRunner.java:50) at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:417) at org.apache.hadoop.mapred.MapTask.run(MapTask.java:332) at org.apache.hadoop.mapred.Child$4.run(Child.java:268) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:415) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1554) at org.apache.hadoop.mapred.Child.main(Child.java:262) org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while processing row {"hash":"nzM2VsRBJReFrv4fGeSX","custom_fields":{"listOfIntegers":{5:[{4:10},{4:28},{4:42},{4:51},{4:63},{4:92},{4:113},{4:126},{4:128},{4:130},{4:144},{4:145},{4:146},{4:179},{4:182},{4:198},{4:225},{4:260},{4:232},{4:233},{4:237},{4:238},{4:255}]}}} {code} was: Handling nullable map of union of 3 types or more is causing serialization issues. Given tables: ag_map_of_uniontype: # col_name data_type comment hash string None custom_fields map<string,uniontype<bigint,double,string,boolean,int,array<uniontype<bigint,double,string,boolean,int>>,map<string,uniontype<bigint,double,string,boolean,int>>>> None ag_simple_join_table: # col_name data_type comment hash string None size int None Query will throw java.lang.RuntimeException: {code} select i.hash, i.custom_fields, b.size from ag_map_of_uniontype i LEFT OUTER JOIN ag_simple_join_table b ON (i.hash = b.hash); {code} {code} org.apache.hadoop.hive.ql.exec.MapJoinOperator: Unxpected exception: java.util.ArrayList cannot be cast to org.apache.hadoop.hive.serde2.objectinspector.UnionObject java.lang.ClassCastException: java.util.ArrayList cannot be cast to org.apache.hadoop.hive.serde2.objectinspector.UnionObject at org.apache.hadoop.hive.serde2.objectinspector.StandardUnionObjectInspector.getTag(StandardUnionObjectInspector.java:91) at org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.serialize(LazySimpleSerDe.java:559) at org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.serialize(LazySimpleSerDe.java:529) at org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.serializeField(LazySimpleSerDe.java:439) at org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.serialize(LazySimpleSerDe.java:423) at org.apache.hadoop.hive.ql.exec.FileSinkOperator.processOp(FileSinkOperator.java:618) at org.apache.hadoop.hive.ql.exec.Operator.process(Operator.java:504) at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:847) at org.apache.hadoop.hive.ql.exec.SelectOperator.processOp(SelectOperator.java:87) at org.apache.hadoop.hive.ql.exec.Operator.process(Operator.java:504) at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:847) at org.apache.hadoop.hive.ql.exec.CommonJoinOperator.genAllOneUniqueJoinObject(CommonJoinOperator.java:671) at org.apache.hadoop.hive.ql.exec.CommonJoinOperator.checkAndGenObject(CommonJoinOperator.java:754) at org.apache.hadoop.hive.ql.exec.MapJoinOperator.processOp(MapJoinOperator.java:222) at org.apache.hadoop.hive.ql.exec.Operator.process(Operator.java:504) at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:847) at org.apache.hadoop.hive.ql.exec.TableScanOperator.processOp(TableScanOperator.java:91) at org.apache.hadoop.hive.ql.exec.Operator.process(Operator.java:504) at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:847) at org.apache.hadoop.hive.ql.exec.MapOperator.process(MapOperator.java:519) at org.apache.hadoop.hive.ql.exec.mr.ExecMapper.map(ExecMapper.java:157) at org.apache.hadoop.mapred.MapRunner.run(MapRunner.java:50) at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:417) at org.apache.hadoop.mapred.MapTask.run(MapTask.java:332) at org.apache.hadoop.mapred.Child$4.run(Child.java:268) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:415) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1554) at org.apache.hadoop.mapred.Child.main(Child.java:262) org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while processing row {"hash":"nzM2VsRBJReFrv4fGeSX","custom_fields":{"listOfIntegers":{5:[{4:10},{4:28},{4:42},{4:51},{4:63},{4:92},{4:113},{4:126},{4:128},{4:130},{4:144},{4:145},{4:146},{4:179},{4:182},{4:198},{4:225},{4:260},{4:232},{4:233},{4:237},{4:238},{4:255}]}}} {code} Summary: Map of union of 3 or more types cannot be joined (was: Nullable map of union of 3 or more types cannot be joined) > Map of union of 3 or more types cannot be joined > ------------------------------------------------ > > Key: HIVE-7657 > URL: https://issues.apache.org/jira/browse/HIVE-7657 > Project: Hive > Issue Type: Bug > Components: Serializers/Deserializers > Reporter: Arkadiusz Gasior > Assignee: Ashish Kumar Singh > Labels: serde > > Joining table which contains map of union of 3 types or more is causing > serialization issues. > Given tables: > ag_map_of_uniontype: > # col_name data_type comment > hash string None > custom_fields > map<string,uniontype<bigint,double,string,boolean,int,array<uniontype<bigint,double,string,boolean,int>>,map<string,uniontype<bigint,double,string,boolean,int>>>> > None > ag_simple_join_table: > # col_name data_type comment > hash string None > size int None > Query will throw java.lang.RuntimeException: > {code} > select > i.hash, > i.custom_fields, > b.size > from ag_map_of_uniontype i LEFT OUTER JOIN > ag_simple_join_table b ON (i.hash = b.hash); > {code} > {code} > org.apache.hadoop.hive.ql.exec.MapJoinOperator: Unxpected exception: > java.util.ArrayList cannot be cast to > org.apache.hadoop.hive.serde2.objectinspector.UnionObject > java.lang.ClassCastException: java.util.ArrayList cannot be cast to > org.apache.hadoop.hive.serde2.objectinspector.UnionObject > at > org.apache.hadoop.hive.serde2.objectinspector.StandardUnionObjectInspector.getTag(StandardUnionObjectInspector.java:91) > at > org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.serialize(LazySimpleSerDe.java:559) > at > org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.serialize(LazySimpleSerDe.java:529) > at > org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.serializeField(LazySimpleSerDe.java:439) > at > org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.serialize(LazySimpleSerDe.java:423) > at > org.apache.hadoop.hive.ql.exec.FileSinkOperator.processOp(FileSinkOperator.java:618) > at org.apache.hadoop.hive.ql.exec.Operator.process(Operator.java:504) > at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:847) > at > org.apache.hadoop.hive.ql.exec.SelectOperator.processOp(SelectOperator.java:87) > at org.apache.hadoop.hive.ql.exec.Operator.process(Operator.java:504) > at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:847) > at > org.apache.hadoop.hive.ql.exec.CommonJoinOperator.genAllOneUniqueJoinObject(CommonJoinOperator.java:671) > at > org.apache.hadoop.hive.ql.exec.CommonJoinOperator.checkAndGenObject(CommonJoinOperator.java:754) > at > org.apache.hadoop.hive.ql.exec.MapJoinOperator.processOp(MapJoinOperator.java:222) > at org.apache.hadoop.hive.ql.exec.Operator.process(Operator.java:504) > at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:847) > at > org.apache.hadoop.hive.ql.exec.TableScanOperator.processOp(TableScanOperator.java:91) > at org.apache.hadoop.hive.ql.exec.Operator.process(Operator.java:504) > at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:847) > at > org.apache.hadoop.hive.ql.exec.MapOperator.process(MapOperator.java:519) > at org.apache.hadoop.hive.ql.exec.mr.ExecMapper.map(ExecMapper.java:157) > at org.apache.hadoop.mapred.MapRunner.run(MapRunner.java:50) > at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:417) > at org.apache.hadoop.mapred.MapTask.run(MapTask.java:332) > at org.apache.hadoop.mapred.Child$4.run(Child.java:268) > at java.security.AccessController.doPrivileged(Native Method) > at javax.security.auth.Subject.doAs(Subject.java:415) > at > org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1554) > at org.apache.hadoop.mapred.Child.main(Child.java:262) > org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while > processing row > {"hash":"nzM2VsRBJReFrv4fGeSX","custom_fields":{"listOfIntegers":{5:[{4:10},{4:28},{4:42},{4:51},{4:63},{4:92},{4:113},{4:126},{4:128},{4:130},{4:144},{4:145},{4:146},{4:179},{4:182},{4:198},{4:225},{4:260},{4:232},{4:233},{4:237},{4:238},{4:255}]}}} > {code} -- This message was sent by Atlassian JIRA (v6.2#6252)