Repository: hbase Updated Branches: refs/heads/master 110274b33 -> b6f091e0f
HBASE-15152 Automatically include prefix-tree module in MR jobs if present Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b6f091e0 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b6f091e0 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b6f091e0 Branch: refs/heads/master Commit: b6f091e0fd02111714ae8820c8a4d3b9b4828343 Parents: 110274b Author: Jonathan M Hsieh <[email protected]> Authored: Thu Jan 21 07:25:00 2016 -0800 Committer: Jonathan M Hsieh <[email protected]> Committed: Thu Jan 21 18:55:19 2016 -0800 ---------------------------------------------------------------------- .../hbase/mapreduce/TableMapReduceUtil.java | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/b6f091e0/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java index cf5f7ac..d43c4d9 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java @@ -763,7 +763,7 @@ public class TableMapReduceUtil { * Add HBase and its dependencies (only) to the job configuration. * <p> * This is intended as a low-level API, facilitating code reuse between this - * class and its mapred counterpart. It also of use to extenral tools that + * class and its mapred counterpart. It also of use to external tools that * need to build a MapReduce job that interacts with HBase but want * fine-grained control over the jars shipped to the cluster. * </p> @@ -772,6 +772,21 @@ public class TableMapReduceUtil { * @see <a href="https://issues.apache.org/jira/browse/PIG-3285";>PIG-3285</a> */ public static void addHBaseDependencyJars(Configuration conf) throws IOException { + + // PrefixTreeCodec is part of the hbase-prefix-tree module. If not included in MR jobs jar + // dependencies, MR jobs that write encoded hfiles will fail. + // We used reflection here so to prevent a circular module dependency. + // TODO - if we extract the MR into a module, make it depend on hbase-prefix-tree. + Class prefixTreeCodecClass = null; + try { + prefixTreeCodecClass = + Class.forName("org.apache.hadoop.hbase.code.prefixtree.PrefixTreeCodec"); + } catch (ClassNotFoundException e) { + // this will show up in unit tests but should not show in real deployments + LOG.warn("The hbase-prefix-tree module jar containing PrefixTreeCodec is not present." + + " Continuing without it."); + } + addDependencyJars(conf, // explicitly pull a class from each module org.apache.hadoop.hbase.HConstants.class, // hbase-common @@ -779,6 +794,7 @@ public class TableMapReduceUtil { org.apache.hadoop.hbase.client.Put.class, // hbase-client org.apache.hadoop.hbase.CompatibilityFactory.class, // hbase-hadoop-compat org.apache.hadoop.hbase.mapreduce.TableMapper.class, // hbase-server + prefixTreeCodecClass, // hbase-prefix-tree (if null will be skipped) // pull necessary dependencies org.apache.zookeeper.ZooKeeper.class, io.netty.channel.Channel.class,
