HIVE-11849: NPE in HiveHBaseTableShapshotInputFormat in query with just count(*) (Enis Soztutar via Jason Dere)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2a65989a Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2a65989a Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2a65989a Branch: refs/heads/beeline-cli Commit: 2a65989a48e043404b7060296be8da9d3494911e Parents: 4ff5b25 Author: Jason Dere <[email protected]> Authored: Mon Sep 21 10:59:21 2015 -0700 Committer: Jason Dere <[email protected]> Committed: Mon Sep 21 10:59:21 2015 -0700 ---------------------------------------------------------------------- .../HiveHBaseTableSnapshotInputFormat.java | 21 ++++++++++++------- .../queries/positive/hbase_handler_snapshot.q | 4 ++++ .../positive/hbase_handler_snapshot.q.out | 22 ++++++++++++++++++++ 3 files changed, 39 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/2a65989a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableSnapshotInputFormat.java ---------------------------------------------------------------------- diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableSnapshotInputFormat.java b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableSnapshotInputFormat.java index 45e4de9..aa3a02f 100644 --- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableSnapshotInputFormat.java +++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableSnapshotInputFormat.java @@ -24,6 +24,9 @@ import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.mapred.TableInputFormat; import org.apache.hadoop.hbase.mapred.TableSnapshotInputFormat; +import org.apache.hadoop.hbase.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.protobuf.generated.ClientProtos; +import org.apache.hadoop.hbase.util.Base64; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.mapred.FileInputFormat; import org.apache.hadoop.mapred.InputFormat; @@ -41,15 +44,17 @@ public class HiveHBaseTableSnapshotInputFormat TableSnapshotInputFormat delegate = new TableSnapshotInputFormat(); private static void setColumns(JobConf job) throws IOException { - // hbase mapred API doesn't support scan at the moment. Scan scan = HiveHBaseInputFormatUtil.getScan(job); - byte[][] families = scan.getFamilies(); - StringBuilder sb = new StringBuilder(); - for (int i = 0; i < families.length; i++) { - if (i > 0) sb.append(" "); - sb.append(Bytes.toString(families[i])); - } - job.set(TableInputFormat.COLUMN_LIST, sb.toString()); + job.set(org.apache.hadoop.hbase.mapreduce.TableInputFormat.SCAN, + convertScanToString(scan)); + } + + // TODO: Once HBASE-11163 is completed, use that API, or switch to + // using mapreduce version of the APIs. rather than mapred + // Copied from HBase's TableMapreduceUtil since it is not public API + static String convertScanToString(Scan scan) throws IOException { + ClientProtos.Scan proto = ProtobufUtil.toScan(scan); + return Base64.encodeBytes(proto.toByteArray()); } @Override http://git-wip-us.apache.org/repos/asf/hive/blob/2a65989a/hbase-handler/src/test/queries/positive/hbase_handler_snapshot.q ---------------------------------------------------------------------- diff --git a/hbase-handler/src/test/queries/positive/hbase_handler_snapshot.q b/hbase-handler/src/test/queries/positive/hbase_handler_snapshot.q index 11d52fd..ebdc63c 100644 --- a/hbase-handler/src/test/queries/positive/hbase_handler_snapshot.q +++ b/hbase-handler/src/test/queries/positive/hbase_handler_snapshot.q @@ -2,3 +2,7 @@ SET hive.hbase.snapshot.name=src_hbase_snapshot; SET hive.hbase.snapshot.restoredir=/tmp; SELECT * FROM src_hbase LIMIT 5; + +SELECT value FROM src_hbase LIMIT 5; + +select count(*) from src_hbase; http://git-wip-us.apache.org/repos/asf/hive/blob/2a65989a/hbase-handler/src/test/results/positive/hbase_handler_snapshot.q.out ---------------------------------------------------------------------- diff --git a/hbase-handler/src/test/results/positive/hbase_handler_snapshot.q.out b/hbase-handler/src/test/results/positive/hbase_handler_snapshot.q.out index 1cb18b2..731646c 100644 --- a/hbase-handler/src/test/results/positive/hbase_handler_snapshot.q.out +++ b/hbase-handler/src/test/results/positive/hbase_handler_snapshot.q.out @@ -11,3 +11,25 @@ POSTHOOK: Input: default@src_hbase 100 val_100 103 val_103 104 val_104 +PREHOOK: query: SELECT value FROM src_hbase LIMIT 5 +PREHOOK: type: QUERY +PREHOOK: Input: default@src_hbase +#### A masked pattern was here #### +POSTHOOK: query: SELECT value FROM src_hbase LIMIT 5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_hbase +#### A masked pattern was here #### +val_0 +val_10 +val_100 +val_103 +val_104 +PREHOOK: query: select count(*) from src_hbase +PREHOOK: type: QUERY +PREHOOK: Input: default@src_hbase +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from src_hbase +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_hbase +#### A masked pattern was here #### +309
