Repository: hbase Updated Branches: refs/heads/branch-1.2 bcc575c35 -> 55229d457
HBASE-15152 Automatically include prefix-tree module in MR jobs if present Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/55229d45 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/55229d45 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/55229d45 Branch: refs/heads/branch-1.2 Commit: 55229d45764215edc050938fbc63a203e86ac530 Parents: bcc575c Author: Jonathan M Hsieh <[email protected]> Authored: Thu Jan 21 07:25:00 2016 -0800 Committer: Jonathan M Hsieh <[email protected]> Committed: Thu Jan 21 19:02:21 2016 -0800 ---------------------------------------------------------------------- .../hbase/mapreduce/TableMapReduceUtil.java | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/55229d45/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java index 1614883..8cad7ab 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java @@ -754,7 +754,7 @@ public class TableMapReduceUtil { * Add HBase and its dependencies (only) to the job configuration. * <p> * This is intended as a low-level API, facilitating code reuse between this - * class and its mapred counterpart. It also of use to extenral tools that + * class and its mapred counterpart. It also of use to external tools that * need to build a MapReduce job that interacts with HBase but want * fine-grained control over the jars shipped to the cluster. * </p> @@ -763,6 +763,21 @@ public class TableMapReduceUtil { * @see <a href="https://issues.apache.org/jira/browse/PIG-3285";>PIG-3285</a> */ public static void addHBaseDependencyJars(Configuration conf) throws IOException { + + // PrefixTreeCodec is part of the hbase-prefix-tree module. If not included in MR jobs jar + // dependencies, MR jobs that write encoded hfiles will fail. + // We used reflection here so to prevent a circular module dependency. + // TODO - if we extract the MR into a module, make it depend on hbase-prefix-tree. + Class prefixTreeCodecClass = null; + try { + prefixTreeCodecClass = + Class.forName("org.apache.hadoop.hbase.code.prefixtree.PrefixTreeCodec"); + } catch (ClassNotFoundException e) { + // this will show up in unit tests but should not show in real deployments + LOG.warn("The hbase-prefix-tree module jar containing PrefixTreeCodec is not present." + + " Continuing without it."); + } + addDependencyJars(conf, // explicitly pull a class from each module org.apache.hadoop.hbase.HConstants.class, // hbase-common @@ -770,6 +785,7 @@ public class TableMapReduceUtil { org.apache.hadoop.hbase.client.Put.class, // hbase-client org.apache.hadoop.hbase.CompatibilityFactory.class, // hbase-hadoop-compat org.apache.hadoop.hbase.mapreduce.TableMapper.class, // hbase-server + prefixTreeCodecClass, // hbase-prefix-tree (if null will be skipped) // pull necessary dependencies org.apache.zookeeper.ZooKeeper.class, io.netty.channel.Channel.class,
