Repository: hive Updated Branches: refs/heads/master dfe929c9b -> 34c83fb36
HIVE-12635: Hive should return the latest hbase cell timestamp as the row timestamp value Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/34c83fb3 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/34c83fb3 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/34c83fb3 Branch: refs/heads/master Commit: 34c83fb36f2baf116cbd128ba47773472a24a9c4 Parents: dfe929c Author: Aihua Xu <[email protected]> Authored: Wed Dec 9 17:16:04 2015 -0500 Committer: Aihua Xu <[email protected]> Committed: Mon Dec 21 16:44:38 2015 -0500 ---------------------------------------------------------------------- .../apache/hadoop/hive/hbase/LazyHBaseRow.java | 4 ++ .../src/test/queries/positive/hbase_timestamp.q | 9 +++ .../test/results/positive/hbase_timestamp.q.out | 72 ++++++++++++++++++++ 3 files changed, 85 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/34c83fb3/hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseRow.java ---------------------------------------------------------------------- diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseRow.java b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseRow.java index 003dd1c..ffb005e 100644 --- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseRow.java +++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseRow.java @@ -154,7 +154,11 @@ public class LazyHBaseRow extends LazyStruct { } if (colMap.hbaseTimestamp) { + // Get the latest timestamp of all the cells as the row timestamp long timestamp = result.rawCells()[0].getTimestamp(); // from hbase-0.96.0 + for (int i = 1; i < result.rawCells().length; i++) { + timestamp = Math.max(timestamp, result.rawCells()[i].getTimestamp()); + } LazyObjectBase lz = fields[fieldID]; if (lz instanceof LazyTimestamp) { ((LazyTimestamp) lz).getWritableObject().setTime(timestamp); http://git-wip-us.apache.org/repos/asf/hive/blob/34c83fb3/hbase-handler/src/test/queries/positive/hbase_timestamp.q ---------------------------------------------------------------------- diff --git a/hbase-handler/src/test/queries/positive/hbase_timestamp.q b/hbase-handler/src/test/queries/positive/hbase_timestamp.q index 6f1e205..0350afe 100644 --- a/hbase-handler/src/test/queries/positive/hbase_timestamp.q +++ b/hbase-handler/src/test/queries/positive/hbase_timestamp.q @@ -39,3 +39,12 @@ SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND explain SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time >= 200000000000; SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time >= 200000000000; + +DROP TABLE hbase_table; +CREATE TABLE hbase_table(key string, value map<string, string>, time timestamp) +STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler' +WITH SERDEPROPERTIES ("hbase.columns.mapping" = ":key,cf:,:timestamp"); +FROM src INSERT OVERWRITE TABLE hbase_table SELECT key, MAP("name", CONCAT(value, " Jr")), "2012-02-23 10:14:52" WHERE (key % 17) = 0; +FROM src INSERT INTO TABLE hbase_table SELECT key, MAP("age", '40'), "2015-12-12 12:12:12" WHERE (key % 17) = 0; +FROM src INSERT INTO TABLE hbase_table SELECT key, MAP("name", value), "2000-01-01 01:01:01" WHERE (key % 17) = 0; +SELECT * FROM hbase_table; http://git-wip-us.apache.org/repos/asf/hive/blob/34c83fb3/hbase-handler/src/test/results/positive/hbase_timestamp.q.out ---------------------------------------------------------------------- diff --git a/hbase-handler/src/test/results/positive/hbase_timestamp.q.out b/hbase-handler/src/test/results/positive/hbase_timestamp.q.out index 6c42fc3..54ddec7 100644 --- a/hbase-handler/src/test/results/positive/hbase_timestamp.q.out +++ b/hbase-handler/src/test/results/positive/hbase_timestamp.q.out @@ -355,3 +355,75 @@ POSTHOOK: Input: default@hbase_table 222 val_222 1976-05-03 12:33:20 296 val_296 1976-05-03 12:33:20 333 val_333 1976-05-03 12:33:20 +PREHOOK: query: DROP TABLE hbase_table +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@hbase_table +PREHOOK: Output: default@hbase_table +POSTHOOK: query: DROP TABLE hbase_table +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@hbase_table +POSTHOOK: Output: default@hbase_table +PREHOOK: query: CREATE TABLE hbase_table(key string, value map<string, string>, time timestamp) +STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler' +WITH SERDEPROPERTIES ("hbase.columns.mapping" = ":key,cf:,:timestamp") +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@hbase_table +POSTHOOK: query: CREATE TABLE hbase_table(key string, value map<string, string>, time timestamp) +STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler' +WITH SERDEPROPERTIES ("hbase.columns.mapping" = ":key,cf:,:timestamp") +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@hbase_table +PREHOOK: query: FROM src INSERT OVERWRITE TABLE hbase_table SELECT key, MAP("name", CONCAT(value, " Jr")), "2012-02-23 10:14:52" WHERE (key % 17) = 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@hbase_table +POSTHOOK: query: FROM src INSERT OVERWRITE TABLE hbase_table SELECT key, MAP("name", CONCAT(value, " Jr")), "2012-02-23 10:14:52" WHERE (key % 17) = 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@hbase_table +PREHOOK: query: FROM src INSERT INTO TABLE hbase_table SELECT key, MAP("age", '40'), "2015-12-12 12:12:12" WHERE (key % 17) = 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@hbase_table +POSTHOOK: query: FROM src INSERT INTO TABLE hbase_table SELECT key, MAP("age", '40'), "2015-12-12 12:12:12" WHERE (key % 17) = 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@hbase_table +PREHOOK: query: FROM src INSERT INTO TABLE hbase_table SELECT key, MAP("name", value), "2000-01-01 01:01:01" WHERE (key % 17) = 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@hbase_table +POSTHOOK: query: FROM src INSERT INTO TABLE hbase_table SELECT key, MAP("name", value), "2000-01-01 01:01:01" WHERE (key % 17) = 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@hbase_table +PREHOOK: query: SELECT * FROM hbase_table +PREHOOK: type: QUERY +PREHOOK: Input: default@hbase_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM hbase_table +POSTHOOK: type: QUERY +POSTHOOK: Input: default@hbase_table +#### A masked pattern was here #### +0 {"age":"40","name":"val_0 Jr"} 2015-12-12 12:12:12 +119 {"age":"40","name":"val_119 Jr"} 2015-12-12 12:12:12 +136 {"age":"40","name":"val_136 Jr"} 2015-12-12 12:12:12 +153 {"age":"40","name":"val_153 Jr"} 2015-12-12 12:12:12 +17 {"age":"40","name":"val_17 Jr"} 2015-12-12 12:12:12 +170 {"age":"40","name":"val_170 Jr"} 2015-12-12 12:12:12 +187 {"age":"40","name":"val_187 Jr"} 2015-12-12 12:12:12 +221 {"age":"40","name":"val_221 Jr"} 2015-12-12 12:12:12 +238 {"age":"40","name":"val_238 Jr"} 2015-12-12 12:12:12 +255 {"age":"40","name":"val_255 Jr"} 2015-12-12 12:12:12 +272 {"age":"40","name":"val_272 Jr"} 2015-12-12 12:12:12 +289 {"age":"40","name":"val_289 Jr"} 2015-12-12 12:12:12 +306 {"age":"40","name":"val_306 Jr"} 2015-12-12 12:12:12 +323 {"age":"40","name":"val_323 Jr"} 2015-12-12 12:12:12 +34 {"age":"40","name":"val_34 Jr"} 2015-12-12 12:12:12 +374 {"age":"40","name":"val_374 Jr"} 2015-12-12 12:12:12 +459 {"age":"40","name":"val_459 Jr"} 2015-12-12 12:12:12 +493 {"age":"40","name":"val_493 Jr"} 2015-12-12 12:12:12 +51 {"age":"40","name":"val_51 Jr"} 2015-12-12 12:12:12 +85 {"age":"40","name":"val_85 Jr"} 2015-12-12 12:12:12
