Repository: hive Updated Branches: refs/heads/master c30771639 -> 46d8a0746
HIVE-13065: Hive throws NPE when writing map type data to a HBase backed table (Yongzhi Chen, reviewed by Aihua Xu) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/46d8a074 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/46d8a074 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/46d8a074 Branch: refs/heads/master Commit: 46d8a07469d617463b5173c8983aeff5e564d554 Parents: c307716 Author: Yongzhi Chen <[email protected]> Authored: Tue Feb 16 16:12:28 2016 -0500 Committer: Yongzhi Chen <[email protected]> Committed: Thu Feb 18 13:37:15 2016 -0500 ---------------------------------------------------------------------- .../hadoop/hive/hbase/HBaseRowSerializer.java | 7 +- .../src/test/queries/positive/hbase_queries.q | 26 ++++ .../test/results/positive/hbase_queries.q.out | 154 +++++++++++++++++++ 3 files changed, 185 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/46d8a074/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseRowSerializer.java ---------------------------------------------------------------------- diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseRowSerializer.java b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseRowSerializer.java index 7c141fc..c6f3b0f 100644 --- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseRowSerializer.java +++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseRowSerializer.java @@ -271,8 +271,11 @@ public class HBaseRowSerializer { ss.write(sep); } serialize(entry.getKey(), koi, level+2, ss); - ss.write(keyValueSeparator); - serialize(entry.getValue(), voi, level+2, ss); + + if ( entry.getValue() != null) { + ss.write(keyValueSeparator); + serialize(entry.getValue(), voi, level+2, ss); + } } } return true; http://git-wip-us.apache.org/repos/asf/hive/blob/46d8a074/hbase-handler/src/test/queries/positive/hbase_queries.q ---------------------------------------------------------------------- diff --git a/hbase-handler/src/test/queries/positive/hbase_queries.q b/hbase-handler/src/test/queries/positive/hbase_queries.q index b445c4b..49fa829 100644 --- a/hbase-handler/src/test/queries/positive/hbase_queries.q +++ b/hbase-handler/src/test/queries/positive/hbase_queries.q @@ -164,6 +164,30 @@ SELECT COUNT(*) FROM hbase_table_1_like; SHOW CREATE TABLE hbase_table_1_like; +DROP TABLE IF EXISTS hbase_table_9; +CREATE TABLE hbase_table_9 (id bigint, data map<string, string>, str string) +stored by 'org.apache.hadoop.hive.hbase.HBaseStorageHandler' +with serdeproperties ("hbase.columns.mapping" = ":key,cf:map_col#s:s,cf:str_col"); + +insert overwrite table hbase_table_9 select 1 as id, map('abcd', null) as data , null as str from src limit 1; +insert into table hbase_table_9 select 2 as id, map('efgh', null) as data , '1234' as str from src limit 1; +insert into table hbase_table_9 select 3 as id, map('hij', '') as data , '1234' as str from src limit 1; +insert into table hbase_table_9 select 4 as id, map('klm', 'avalue') as data , '1234' as str from src limit 1; +insert into table hbase_table_9 select 5 as id, map('key1',null, 'key2', 'avalue') as data , '1234' as str from src limit 1; +select * from hbase_table_9; + +DROP TABLE IF EXISTS hbase_table_10; +CREATE TABLE hbase_table_10 (id bigint, data map<int, int>, str string) +stored by 'org.apache.hadoop.hive.hbase.HBaseStorageHandler' +with serdeproperties ("hbase.columns.mapping" = ":key,cf:map_col2,cf:str2_col"); +insert overwrite table hbase_table_10 select 1 as id, map(10, cast(null as int)) as data , null as str from src limit 1; +insert into table hbase_table_10 select 2 as id, map(20, cast(null as int)) as data , '1234' as str from src limit 1; +insert into table hbase_table_10 select 3 as id, map(30, 31) as data , '1234' as str from src limit 1; +insert into table hbase_table_10 select 4 as id, map(40, cast(null as int), 45, cast(null as int)) as data , '1234' as str from src limit 1; +insert into table hbase_table_10 select 5 as id, map(50,cast(null as int), 55, 58) as data , '1234' as str from src limit 1; +select * from hbase_table_10; + + DROP TABLE hbase_table_1; DROP TABLE hbase_table_1_like; DROP TABLE hbase_table_2; @@ -176,3 +200,5 @@ DROP TABLE hbase_table_7; DROP TABLE hbase_table_8; DROP TABLE empty_hbase_table; DROP TABLE empty_normal_table; +DROP TABLE hbase_table_9; +DROP TABLE hbase_table_10; http://git-wip-us.apache.org/repos/asf/hive/blob/46d8a074/hbase-handler/src/test/results/positive/hbase_queries.q.out ---------------------------------------------------------------------- diff --git a/hbase-handler/src/test/results/positive/hbase_queries.q.out b/hbase-handler/src/test/results/positive/hbase_queries.q.out index a18bb8a..a99f561 100644 --- a/hbase-handler/src/test/results/positive/hbase_queries.q.out +++ b/hbase-handler/src/test/results/positive/hbase_queries.q.out @@ -919,6 +919,144 @@ WITH SERDEPROPERTIES ( TBLPROPERTIES ( 'hbase.table.name'='hbase_table_0', #### A masked pattern was here #### +PREHOOK: query: DROP TABLE IF EXISTS hbase_table_9 +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS hbase_table_9 +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE hbase_table_9 (id bigint, data map<string, string>, str string) +stored by 'org.apache.hadoop.hive.hbase.HBaseStorageHandler' +with serdeproperties ("hbase.columns.mapping" = ":key,cf:map_col#s:s,cf:str_col") +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@hbase_table_9 +POSTHOOK: query: CREATE TABLE hbase_table_9 (id bigint, data map<string, string>, str string) +stored by 'org.apache.hadoop.hive.hbase.HBaseStorageHandler' +with serdeproperties ("hbase.columns.mapping" = ":key,cf:map_col#s:s,cf:str_col") +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@hbase_table_9 +PREHOOK: query: insert overwrite table hbase_table_9 select 1 as id, map('abcd', null) as data , null as str from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@hbase_table_9 +POSTHOOK: query: insert overwrite table hbase_table_9 select 1 as id, map('abcd', null) as data , null as str from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@hbase_table_9 +PREHOOK: query: insert into table hbase_table_9 select 2 as id, map('efgh', null) as data , '1234' as str from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@hbase_table_9 +POSTHOOK: query: insert into table hbase_table_9 select 2 as id, map('efgh', null) as data , '1234' as str from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@hbase_table_9 +PREHOOK: query: insert into table hbase_table_9 select 3 as id, map('hij', '') as data , '1234' as str from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@hbase_table_9 +POSTHOOK: query: insert into table hbase_table_9 select 3 as id, map('hij', '') as data , '1234' as str from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@hbase_table_9 +PREHOOK: query: insert into table hbase_table_9 select 4 as id, map('klm', 'avalue') as data , '1234' as str from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@hbase_table_9 +POSTHOOK: query: insert into table hbase_table_9 select 4 as id, map('klm', 'avalue') as data , '1234' as str from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@hbase_table_9 +PREHOOK: query: insert into table hbase_table_9 select 5 as id, map('key1',null, 'key2', 'avalue') as data , '1234' as str from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@hbase_table_9 +POSTHOOK: query: insert into table hbase_table_9 select 5 as id, map('key1',null, 'key2', 'avalue') as data , '1234' as str from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@hbase_table_9 +PREHOOK: query: select * from hbase_table_9 +PREHOOK: type: QUERY +PREHOOK: Input: default@hbase_table_9 +#### A masked pattern was here #### +POSTHOOK: query: select * from hbase_table_9 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@hbase_table_9 +#### A masked pattern was here #### +1 {"abcd":null} NULL +2 {"efgh":null} 1234 +3 {"hij":""} 1234 +4 {"klm":"avalue"} 1234 +5 {"key1":null,"key2":"avalue"} 1234 +PREHOOK: query: DROP TABLE IF EXISTS hbase_table_10 +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS hbase_table_10 +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE hbase_table_10 (id bigint, data map<int, int>, str string) +stored by 'org.apache.hadoop.hive.hbase.HBaseStorageHandler' +with serdeproperties ("hbase.columns.mapping" = ":key,cf:map_col2,cf:str2_col") +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@hbase_table_10 +POSTHOOK: query: CREATE TABLE hbase_table_10 (id bigint, data map<int, int>, str string) +stored by 'org.apache.hadoop.hive.hbase.HBaseStorageHandler' +with serdeproperties ("hbase.columns.mapping" = ":key,cf:map_col2,cf:str2_col") +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@hbase_table_10 +PREHOOK: query: insert overwrite table hbase_table_10 select 1 as id, map(10, cast(null as int)) as data , null as str from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@hbase_table_10 +POSTHOOK: query: insert overwrite table hbase_table_10 select 1 as id, map(10, cast(null as int)) as data , null as str from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@hbase_table_10 +PREHOOK: query: insert into table hbase_table_10 select 2 as id, map(20, cast(null as int)) as data , '1234' as str from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@hbase_table_10 +POSTHOOK: query: insert into table hbase_table_10 select 2 as id, map(20, cast(null as int)) as data , '1234' as str from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@hbase_table_10 +PREHOOK: query: insert into table hbase_table_10 select 3 as id, map(30, 31) as data , '1234' as str from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@hbase_table_10 +POSTHOOK: query: insert into table hbase_table_10 select 3 as id, map(30, 31) as data , '1234' as str from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@hbase_table_10 +PREHOOK: query: insert into table hbase_table_10 select 4 as id, map(40, cast(null as int), 45, cast(null as int)) as data , '1234' as str from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@hbase_table_10 +POSTHOOK: query: insert into table hbase_table_10 select 4 as id, map(40, cast(null as int), 45, cast(null as int)) as data , '1234' as str from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@hbase_table_10 +PREHOOK: query: insert into table hbase_table_10 select 5 as id, map(50,cast(null as int), 55, 58) as data , '1234' as str from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@hbase_table_10 +POSTHOOK: query: insert into table hbase_table_10 select 5 as id, map(50,cast(null as int), 55, 58) as data , '1234' as str from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@hbase_table_10 +PREHOOK: query: select * from hbase_table_10 +PREHOOK: type: QUERY +PREHOOK: Input: default@hbase_table_10 +#### A masked pattern was here #### +POSTHOOK: query: select * from hbase_table_10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@hbase_table_10 +#### A masked pattern was here #### +1 {10:null} NULL +2 {20:null} 1234 +3 {30:31} 1234 +4 {40:null,45:null} 1234 +5 {50:null,55:58} 1234 PREHOOK: query: DROP TABLE hbase_table_1 PREHOOK: type: DROPTABLE PREHOOK: Input: default@hbase_table_1 @@ -1015,3 +1153,19 @@ POSTHOOK: query: DROP TABLE empty_normal_table POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@empty_normal_table POSTHOOK: Output: default@empty_normal_table +PREHOOK: query: DROP TABLE hbase_table_9 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@hbase_table_9 +PREHOOK: Output: default@hbase_table_9 +POSTHOOK: query: DROP TABLE hbase_table_9 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@hbase_table_9 +POSTHOOK: Output: default@hbase_table_9 +PREHOOK: query: DROP TABLE hbase_table_10 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@hbase_table_10 +PREHOOK: Output: default@hbase_table_10 +POSTHOOK: query: DROP TABLE hbase_table_10 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@hbase_table_10 +POSTHOOK: Output: default@hbase_table_10
