PHOENIX-3490 PhoenixStorageHandler doesn't handler hbase configuration properly and don't work in secured env
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/cd931c0b Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/cd931c0b Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/cd931c0b Branch: refs/heads/4.x-HBase-1.1 Commit: cd931c0b304a54019c2c7ea66186005f4f5b5e3e Parents: 5048af4 Author: Sergey Soldatov <s...@apache.org> Authored: Wed Nov 16 17:07:08 2016 -0800 Committer: Sergey Soldatov <s...@apache.org> Committed: Thu Nov 17 12:07:38 2016 -0800 ---------------------------------------------------------------------- .../phoenix/hive/PhoenixStorageHandler.java | 55 ++++++++++++++++++++ 1 file changed, 55 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/cd931c0b/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java ---------------------------------------------------------------------- diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java index bda2282..a425b7c 100644 --- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java +++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java @@ -19,7 +19,10 @@ package org.apache.phoenix.hive; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.mapred.TableMapReduceUtil; import org.apache.hadoop.hive.common.JavaUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaHook; @@ -34,6 +37,7 @@ import org.apache.hadoop.hive.ql.plan.TableDesc; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.serde2.Deserializer; import org.apache.hadoop.hive.serde2.SerDe; +import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.mapred.InputFormat; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.OutputFormat; @@ -43,6 +47,7 @@ import org.apache.phoenix.hive.mapreduce.PhoenixOutputFormat; import org.apache.phoenix.hive.ppd.PhoenixPredicateDecomposer; import org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil; +import java.io.IOException; import java.util.List; import java.util.Map; import java.util.Properties; @@ -54,6 +59,22 @@ import java.util.Properties; public class PhoenixStorageHandler extends DefaultStorageHandler implements HiveStoragePredicateHandler, InputEstimator { + + private Configuration jobConf; + private Configuration hbaseConf; + + + @Override + public void setConf(Configuration conf) { + jobConf = conf; + hbaseConf = HBaseConfiguration.create(conf); + } + + @Override + public Configuration getConf() { + return hbaseConf; + } + private static final Log LOG = LogFactory.getLog(PhoenixStorageHandler.class); public PhoenixStorageHandler() { @@ -67,6 +88,22 @@ public class PhoenixStorageHandler extends DefaultStorageHandler implements return new PhoenixMetaHook(); } + @Override + public void configureJobConf(TableDesc tableDesc, JobConf jobConf) { + try { + TableMapReduceUtil.addDependencyJars(jobConf); + org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.addDependencyJars(jobConf, + PhoenixStorageHandler.class); + JobConf hbaseJobConf = new JobConf(getConf()); + org.apache.hadoop.hbase.mapred.TableMapReduceUtil.initCredentials(hbaseJobConf); + ShimLoader.getHadoopShims().mergeCredentials(jobConf, hbaseJobConf); + } catch (IOException e) { + throw new RuntimeException(e); + } + + + } + @SuppressWarnings("rawtypes") @Override public Class<? extends OutputFormat> getOutputFormatClass() { @@ -167,6 +204,24 @@ public class PhoenixStorageHandler extends DefaultStorageHandler implements (PhoenixStorageHandlerConstants.ZOOKEEPER_PORT)); jobProperties.put(HConstants.ZOOKEEPER_ZNODE_PARENT, jobProperties.get (PhoenixStorageHandlerConstants.ZOOKEEPER_PARENT)); + addHBaseResources(jobConf, jobProperties); + } + + /** + * Utility method to add hbase-default.xml and hbase-site.xml properties to a new map + * if they are not already present in the jobConf. + * @param jobConf Job configuration + * @param newJobProperties Map to which new properties should be added + */ + private void addHBaseResources(Configuration jobConf, + Map<String, String> newJobProperties) { + Configuration conf = new Configuration(false); + HBaseConfiguration.addHbaseResources(conf); + for (Map.Entry<String, String> entry : conf) { + if (jobConf.get(entry.getKey()) == null) { + newJobProperties.put(entry.getKey(), entry.getValue()); + } + } } @Override