RANGER-237: Ranger to work with HA enabled WebHDFS with automatic failover
Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/c4d066b6 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/c4d066b6 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/c4d066b6 Branch: refs/heads/master Commit: c4d066b6be26be7b1cc4c9a67b6dd7fae93f1fc2 Parents: a6f8050 Author: Dilli Dorai Arumugam <[email protected]> Authored: Tue Feb 10 09:48:29 2015 -0800 Committer: Dilli Dorai Arumugam <[email protected]> Committed: Tue Feb 10 20:00:27 2015 -0800 ---------------------------------------------------------------------- .../apache/ranger/plugin/client/BaseClient.java | 25 +++++++++++------ .../plugin/client/HadoopConfigHolder.java | 29 ++++++++++++++------ .../ranger/services/hdfs/client/HdfsClient.java | 4 +-- .../main/resources/resourcenamemap.properties | 3 +- 4 files changed, 41 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c4d066b6/agents-common/src/main/java/org/apache/ranger/plugin/client/BaseClient.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/client/BaseClient.java b/agents-common/src/main/java/org/apache/ranger/plugin/client/BaseClient.java index 53f5859..cdd813e 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/client/BaseClient.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/client/BaseClient.java @@ -34,22 +34,29 @@ import org.apache.hadoop.security.SecureClientLogin; public abstract class BaseClient { private static final Log LOG = LogFactory.getLog(BaseClient.class) ; - - private String serviceName ; + + + private String serviceName ; + private String defaultConfigFile ; private Subject loginSubject ; private HadoopConfigHolder configHolder; protected Map<String,String> connectionProperties ; - public BaseClient(String serivceName) { - this.serviceName = serivceName ; - init() ; - login() ; + public BaseClient(String serviceName) { + this.serviceName = serviceName ; + init() ; + login() ; } - - public BaseClient(String serivceName, Map<String,String> connectionProperties) { + + public BaseClient(String svcName, Map<String,String> connectionProperties) { + this(svcName, connectionProperties, null); + } + + public BaseClient(String serivceName, Map<String,String> connectionProperties, String defaultConfigFile) { this.serviceName = serivceName ; this.connectionProperties = connectionProperties ; + this.defaultConfigFile = defaultConfigFile ; init() ; login() ; } @@ -60,7 +67,7 @@ public abstract class BaseClient { configHolder = HadoopConfigHolder.getInstance(serviceName) ; } else { - configHolder = HadoopConfigHolder.getInstance(serviceName,connectionProperties) ; + configHolder = HadoopConfigHolder.getInstance(serviceName,connectionProperties, defaultConfigFile) ; } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c4d066b6/agents-common/src/main/java/org/apache/ranger/plugin/client/HadoopConfigHolder.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/client/HadoopConfigHolder.java b/agents-common/src/main/java/org/apache/ranger/plugin/client/HadoopConfigHolder.java index b80780b..c0fdbd8 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/client/HadoopConfigHolder.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/client/HadoopConfigHolder.java @@ -51,6 +51,7 @@ public class HadoopConfigHolder { private String datasourceName ; + private String defaultConfigFile ; private String userName ; private String keyTabFile ; private String password ; @@ -72,14 +73,19 @@ public class HadoopConfigHolder { } return ret ; } - - public static HadoopConfigHolder getInstance(String aDatasourceName, Map<String,String> connectionProperties) { + + public static HadoopConfigHolder getInstance(String aDatasourceName, Map<String,String> connectionProperties) { + return getInstance(aDatasourceName, connectionProperties, null); + } + + public static HadoopConfigHolder getInstance(String aDatasourceName, Map<String,String> connectionProperties, + String defaultConfigFile) { HadoopConfigHolder ret = dataSource2HadoopConfigHolder.get(aDatasourceName) ; if (ret == null) { synchronized(HadoopConfigHolder.class) { HadoopConfigHolder temp = ret ; if (temp == null) { - ret = new HadoopConfigHolder(aDatasourceName,connectionProperties) ; + ret = new HadoopConfigHolder(aDatasourceName,connectionProperties, defaultConfigFile) ; dataSource2HadoopConfigHolder.put(aDatasourceName, ret) ; } } @@ -104,10 +110,16 @@ public class HadoopConfigHolder { initLoginInfo(); initClassLoader() ; } - - private HadoopConfigHolder(String aDatasourceName, Map<String,String> connectionProperties) { + + private HadoopConfigHolder(String aDatasourceName, Map<String,String> connectionProperties) { + this(aDatasourceName, connectionProperties, null); + } + + private HadoopConfigHolder(String aDatasourceName, Map<String,String> connectionProperties, + String defaultConfigFile) { datasourceName = aDatasourceName; this.connectionProperties = connectionProperties ; + this.defaultConfigFile = defaultConfigFile; initConnectionProp() ; initLoginInfo(); initClassLoader() ; @@ -131,12 +143,13 @@ public class HadoopConfigHolder { if (resourcemapProperties == null) { initResourceMap(); } - + if (resourcemapProperties != null) { - return resourcemapProperties.getProperty(key); + String rn = resourcemapProperties.getProperty(key); + return ( rn != null) ? rn : defaultConfigFile; } else { - return null; + return defaultConfigFile; } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c4d066b6/hdfs-agent/src/main/java/org/apache/ranger/services/hdfs/client/HdfsClient.java ---------------------------------------------------------------------- diff --git a/hdfs-agent/src/main/java/org/apache/ranger/services/hdfs/client/HdfsClient.java b/hdfs-agent/src/main/java/org/apache/ranger/services/hdfs/client/HdfsClient.java index ff34f4f..a08bf89 100644 --- a/hdfs-agent/src/main/java/org/apache/ranger/services/hdfs/client/HdfsClient.java +++ b/hdfs-agent/src/main/java/org/apache/ranger/services/hdfs/client/HdfsClient.java @@ -45,12 +45,12 @@ public class HdfsClient extends BaseClient { private static final Log LOG = LogFactory.getLog(HdfsClient.class) ; - public HdfsClient(String serviceName) { + public HdfsClient(String serviceName) { super(serviceName) ; } public HdfsClient(String serviceName, Map<String,String> connectionProperties) { - super(serviceName,connectionProperties) ; + super(serviceName,connectionProperties, "hdfs-client") ; } private List<String> listFilesInternal(String baseDir, String fileMatching, final List<String> pathList) { http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c4d066b6/security-admin/src/main/resources/resourcenamemap.properties ---------------------------------------------------------------------- diff --git a/security-admin/src/main/resources/resourcenamemap.properties b/security-admin/src/main/resources/resourcenamemap.properties index d9b4d71..7a9e89c 100644 --- a/security-admin/src/main/resources/resourcenamemap.properties +++ b/security-admin/src/main/resources/resourcenamemap.properties @@ -18,6 +18,7 @@ hadoop.security.authentication=core-site.xml hadoop.security.authorization=core-site.xml hadoop.security.auth_to_local=core-site.xml hadoop.rpc.protection=core-site.xml +dfs.nameservices=hdfs-site.xml dfs.datanode.kerberos.principal=hdfs-site.xml dfs.namenode.kerberos.principal=hdfs-site.xml dfs.secondary.namenode.kerberos.principal=hdfs-site.xml @@ -30,4 +31,4 @@ hbase.rpc.protection=hbase-site.xml hbase.security.authentication=hbase-site.xml hbase.zookeeper.property.clientPort=hbase-site.xml hbase.zookeeper.quorum=hbase-site.xml -zookeeper.znode.parent=hbase-site.xml \ No newline at end of file +zookeeper.znode.parent=hbase-site.xml
