Repository: hbase Updated Branches: refs/heads/branch-1.1 7ecba62c4 -> 8c858c214
HBASE-15152 Automatically include prefix-tree module in MR jobs if present Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8c858c21 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8c858c21 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8c858c21 Branch: refs/heads/branch-1.1 Commit: 8c858c214a90f70df40d900d6f65c5bfe6dd7b65 Parents: 7ecba62 Author: Jonathan M Hsieh <[email protected]> Authored: Thu Jan 21 07:25:00 2016 -0800 Committer: Jonathan M Hsieh <[email protected]> Committed: Thu Jan 21 19:00:59 2016 -0800 ---------------------------------------------------------------------- .../hbase/mapreduce/TableMapReduceUtil.java | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/8c858c21/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java index 410e411..ce273f5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java @@ -705,7 +705,7 @@ public class TableMapReduceUtil { * Add HBase and its dependencies (only) to the job configuration. * <p> * This is intended as a low-level API, facilitating code reuse between this - * class and its mapred counterpart. It also of use to extenral tools that + * class and its mapred counterpart. It also of use to external tools that * need to build a MapReduce job that interacts with HBase but want * fine-grained control over the jars shipped to the cluster. * </p> @@ -714,6 +714,21 @@ public class TableMapReduceUtil { * @see <a href="https://issues.apache.org/jira/browse/PIG-3285";>PIG-3285</a> */ public static void addHBaseDependencyJars(Configuration conf) throws IOException { + + // PrefixTreeCodec is part of the hbase-prefix-tree module. If not included in MR jobs jar + // dependencies, MR jobs that write encoded hfiles will fail. + // We used reflection here so to prevent a circular module dependency. + // TODO - if we extract the MR into a module, make it depend on hbase-prefix-tree. + Class prefixTreeCodecClass = null; + try { + prefixTreeCodecClass = + Class.forName("org.apache.hadoop.hbase.code.prefixtree.PrefixTreeCodec"); + } catch (ClassNotFoundException e) { + // this will show up in unit tests but should not show in real deployments + LOG.warn("The hbase-prefix-tree module jar containing PrefixTreeCodec is not present." + + " Continuing without it."); + } + addDependencyJars(conf, // explicitly pull a class from each module org.apache.hadoop.hbase.HConstants.class, // hbase-common @@ -721,6 +736,7 @@ public class TableMapReduceUtil { org.apache.hadoop.hbase.client.Put.class, // hbase-client org.apache.hadoop.hbase.CompatibilityFactory.class, // hbase-hadoop-compat org.apache.hadoop.hbase.mapreduce.TableMapper.class, // hbase-server + prefixTreeCodecClass, // hbase-prefix-tree (if null will be skipped) // pull necessary dependencies org.apache.zookeeper.ZooKeeper.class, io.netty.channel.Channel.class,
