Repository: phoenix Updated Branches: refs/heads/4.0 18b0d6904 -> b9d1722c6
PHOENIX-1078 Unable to run pig script with Phoenix in a secure HBase cluster Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/b9d1722c Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/b9d1722c Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/b9d1722c Branch: refs/heads/4.0 Commit: b9d1722c6085e534846a5b10205ef1b85dc7297c Parents: 18b0d69 Author: Jeffrey Zhong <jeffr...@apache.org> Authored: Thu Feb 12 19:22:17 2015 -0800 Committer: Jeffrey Zhong <jeffr...@apache.org> Committed: Wed Feb 25 14:58:00 2015 -0800 ---------------------------------------------------------------------- .../phoenix/mapreduce/util/ConnectionUtil.java | 9 +++++++++ .../mapreduce/util/PhoenixConfigurationUtil.java | 17 +++++++++++++++++ .../org/apache/phoenix/pig/PhoenixHBaseLoader.java | 9 ++++++--- .../apache/phoenix/pig/PhoenixHBaseStorage.java | 2 ++ 4 files changed, 34 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/b9d1722c/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/ConnectionUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/ConnectionUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/ConnectionUtil.java index 0864cba..364baf7 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/ConnectionUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/ConnectionUtil.java @@ -20,6 +20,8 @@ package org.apache.phoenix.mapreduce.util; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; +import java.util.Iterator; +import java.util.Map; import java.util.Properties; import org.apache.hadoop.conf.Configuration; @@ -42,6 +44,13 @@ public class ConnectionUtil { public static Connection getConnection(final Configuration configuration) throws SQLException { Preconditions.checkNotNull(configuration); final Properties props = new Properties(); + Iterator<Map.Entry<String, String>> iterator = configuration.iterator(); + if(iterator != null) { + while (iterator.hasNext()) { + Map.Entry<String, String> entry = iterator.next(); + props.setProperty(entry.getKey(), entry.getValue()); + } + } final Connection conn = DriverManager.getConnection(QueryUtil.getUrl(configuration.get(HConstants.ZOOKEEPER_QUORUM)), props); return conn; } http://git-wip-us.apache.org/repos/asf/phoenix/blob/b9d1722c/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java index 83a606b..4d025ee 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java @@ -19,13 +19,18 @@ package org.apache.phoenix.mapreduce.util; import static org.apache.commons.lang.StringUtils.isNotEmpty; +import java.io.IOException; import java.sql.Connection; import java.sql.SQLException; import java.util.List; +import java.util.Map; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; +import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.lib.db.DBInputFormat.NullDBWritable; import org.apache.hadoop.mapreduce.lib.db.DBWritable; import org.apache.phoenix.jdbc.PhoenixConnection; @@ -296,4 +301,16 @@ public final class PhoenixConfigurationUtil { Preconditions.checkNotNull(configuration); return configuration.get(OUTPUT_TABLE_NAME); } + + public static void loadHBaseConfiguration(Job job) throws IOException { + // load hbase-site.xml + Configuration hbaseConf = HBaseConfiguration.create(); + for (Map.Entry<String, String> entry : hbaseConf) { + if (job.getConfiguration().get(entry.getKey()) == null) { + job.getConfiguration().set(entry.getKey(), entry.getValue()); + } + } + //In order to have phoenix working on a secured cluster + TableMapReduceUtil.initCredentials(job); + } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/b9d1722c/phoenix-pig/src/main/java/org/apache/phoenix/pig/PhoenixHBaseLoader.java ---------------------------------------------------------------------- diff --git a/phoenix-pig/src/main/java/org/apache/phoenix/pig/PhoenixHBaseLoader.java b/phoenix-pig/src/main/java/org/apache/phoenix/pig/PhoenixHBaseLoader.java index 1218e82..18e362a 100644 --- a/phoenix-pig/src/main/java/org/apache/phoenix/pig/PhoenixHBaseLoader.java +++ b/phoenix-pig/src/main/java/org/apache/phoenix/pig/PhoenixHBaseLoader.java @@ -106,12 +106,13 @@ public final class PhoenixHBaseLoader extends LoadFunc implements LoadMetadata { } @Override - public void setLocation(String location, Job job) throws IOException { + public void setLocation(String location, Job job) throws IOException { + PhoenixConfigurationUtil.loadHBaseConfiguration(job); + final Configuration configuration = job.getConfiguration(); //explicitly turning off combining splits. configuration.setBoolean("pig.noSplitCombination", true); - //to have phoenix working on a secured cluster - TableMapReduceUtil.initCredentials(job); + this.initializePhoenixPigConfiguration(location, configuration); } @@ -222,6 +223,8 @@ public final class PhoenixHBaseLoader extends LoadFunc implements LoadMetadata { if(schema != null) { return schema; } + + PhoenixConfigurationUtil.loadHBaseConfiguration(job); final Configuration configuration = job.getConfiguration(); this.initializePhoenixPigConfiguration(location, configuration); this.schema = PhoenixPigSchemaUtil.getResourceSchema(this.config); http://git-wip-us.apache.org/repos/asf/phoenix/blob/b9d1722c/phoenix-pig/src/main/java/org/apache/phoenix/pig/PhoenixHBaseStorage.java ---------------------------------------------------------------------- diff --git a/phoenix-pig/src/main/java/org/apache/phoenix/pig/PhoenixHBaseStorage.java b/phoenix-pig/src/main/java/org/apache/phoenix/pig/PhoenixHBaseStorage.java index eb2c124..72d958b 100644 --- a/phoenix-pig/src/main/java/org/apache/phoenix/pig/PhoenixHBaseStorage.java +++ b/phoenix-pig/src/main/java/org/apache/phoenix/pig/PhoenixHBaseStorage.java @@ -140,6 +140,8 @@ public class PhoenixHBaseStorage implements StoreFuncInterface { if (!"hbase".equals(locationURI.getScheme())) { throw new IOException(String.format("Location must use the hbase protocol, hbase://tableName[/columnList]. Supplied location=%s",location)); } + + PhoenixConfigurationUtil.loadHBaseConfiguration(job); config = job.getConfiguration(); config.set(HConstants.ZOOKEEPER_QUORUM, server); String tableName = locationURI.getAuthority();
