Repository: incubator-ranger Updated Branches: refs/heads/master ccf8d6264 -> a200d82d8
RANGER-314: Remove custom class loader used by ranger admin for resource lookup Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/a200d82d Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/a200d82d Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/a200d82d Branch: refs/heads/master Commit: a200d82d8dd08d7b562019d57ba9aa681abd185c Parents: ccf8d62 Author: Dilli Dorai Arumugam <[email protected]> Authored: Thu Mar 12 16:27:23 2015 -0700 Committer: Dilli Dorai Arumugam <[email protected]> Committed: Mon Mar 16 15:10:33 2015 -0700 ---------------------------------------------------------------------- .../apache/ranger/plugin/client/BaseClient.java | 8 +- .../ranger/plugin/client/HadoopClassLoader.java | 104 ------------------ .../plugin/client/HadoopConfigHolder.java | 35 +++---- .../service-defs/ranger-servicedef-hdfs.json | 2 +- .../services/hbase/client/HBaseClient.java | 70 ++++++------- .../hbase/client/HBaseConnectionMgr.java | 2 +- .../ranger/services/hdfs/client/HdfsClient.java | 55 +++++----- .../services/hdfs/client/HdfsConnectionMgr.java | 2 +- .../services/hdfs/client/HdfsClientTest.java | 25 ++++- .../ranger/services/hive/client/HiveClient.java | 2 +- .../hadoop/client/config/HadoopClassLoader.java | 105 ------------------- .../client/config/HadoopConfigHolder.java | 19 +--- .../main/resources/resourcenamemap.properties | 16 --- 13 files changed, 103 insertions(+), 342 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a200d82d/agents-common/src/main/java/org/apache/ranger/plugin/client/BaseClient.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/client/BaseClient.java b/agents-common/src/main/java/org/apache/ranger/plugin/client/BaseClient.java index cdd813e..4ef3b48 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/client/BaseClient.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/client/BaseClient.java @@ -43,12 +43,6 @@ public abstract class BaseClient { protected Map<String,String> connectionProperties ; - public BaseClient(String serviceName) { - this.serviceName = serviceName ; - init() ; - login() ; - } - public BaseClient(String svcName, Map<String,String> connectionProperties) { this(svcName, connectionProperties, null); } @@ -78,7 +72,7 @@ public abstract class BaseClient { + "policies, but you would not be able to use autocomplete for " + "resource names. Check xa_portal.log for more info."; try { - Thread.currentThread().setContextClassLoader(configHolder.getClassLoader()); + //Thread.currentThread().setContextClassLoader(configHolder.getClassLoader()); String userName = configHolder.getUserName() ; if (userName == null) { String msgDesc = "Unable to find login username for hadoop environment, [" http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a200d82d/agents-common/src/main/java/org/apache/ranger/plugin/client/HadoopClassLoader.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/client/HadoopClassLoader.java b/agents-common/src/main/java/org/apache/ranger/plugin/client/HadoopClassLoader.java deleted file mode 100644 index b90f4f6..0000000 --- a/agents-common/src/main/java/org/apache/ranger/plugin/client/HadoopClassLoader.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - package org.apache.ranger.plugin.client; - -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.io.PrintWriter; -import java.net.URL; -import java.util.Properties; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -public class HadoopClassLoader extends ClassLoader { - - private static final Log LOG = LogFactory.getLog(HadoopClassLoader.class) ; - - private HadoopConfigHolder confHolder ; - - public HadoopClassLoader(HadoopConfigHolder confHolder) { - super(Thread.currentThread().getContextClassLoader()) ; - this.confHolder = confHolder; - } - - - @Override - protected URL findResource(String resourceName) { - LOG.debug("findResource(" + resourceName + ") is called.") ; - URL ret = null; - - if (confHolder.hasResourceExists(resourceName)) { - ret = buildResourceFile(resourceName) ; - } - else { - ret = super.findResource(resourceName); - } - LOG.debug("findResource(" + resourceName + ") is returning [" + ret + "]") ; - return ret ; - } - - - @SuppressWarnings("deprecation") - private URL buildResourceFile(String aResourceName) { - URL ret = null ; - String prefix = aResourceName ; - String suffix = ".txt" ; - - Properties prop = confHolder.getProperties(aResourceName) ; - LOG.debug("Building XML for: " + prop.toString()); - if (prop != null && prop.size() > 0) { - if (aResourceName.contains(".")) { - int lastDotFound = aResourceName.indexOf(".") ; - prefix = aResourceName.substring(0,lastDotFound) + "-" ; - suffix = aResourceName.substring(lastDotFound) ; - } - - try { - File tempFile = File.createTempFile(prefix, suffix) ; - tempFile.deleteOnExit(); - PrintWriter out = new PrintWriter(new FileWriter(tempFile)) ; - out.println("<?xml version=\"1.0\"?>") ; - out.println("<?xml-stylesheet type=\"text/xsl\" href=\"configuration.xsl\"?>") ; - out.println("<configuration xmlns:xi=\"http://www.w3.org/2001/XInclude\";>") ; - for(Object keyobj : prop.keySet()) { - String key = (String)keyobj; - String val = prop.getProperty(key) ; - if (HadoopConfigHolder.HADOOP_RPC_PROTECTION.equals(key) && (val == null || val.trim().isEmpty())) { - continue; - } - out.println("<property><name>" + key.trim() + "</name><value>" + val + "</value></property>") ; - } - out.println("</configuration>") ; - out.close() ; - ret = tempFile.toURL() ; - } catch (IOException e) { - throw new HadoopException("Unable to load create hadoop configuration file [" + aResourceName + "]", e) ; - } - - } - - return ret ; - - } - - -} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a200d82d/agents-common/src/main/java/org/apache/ranger/plugin/client/HadoopConfigHolder.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/client/HadoopConfigHolder.java b/agents-common/src/main/java/org/apache/ranger/plugin/client/HadoopConfigHolder.java index c0fdbd8..a341a44 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/client/HadoopConfigHolder.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/client/HadoopConfigHolder.java @@ -21,9 +21,7 @@ import java.io.IOException; import java.io.InputStream; -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; +import java.util.*; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -57,8 +55,9 @@ public class HadoopConfigHolder { private String password ; private boolean isKerberosAuth ; - private HadoopClassLoader classLoader ; - private Map<String,String> connectionProperties; + private Map<String,String> connectionProperties; + + private static Set<String> rangerInternalPropertyKeys = new HashSet<String>(); public static HadoopConfigHolder getInstance(String aDatasourceName) { HadoopConfigHolder ret = dataSource2HadoopConfigHolder.get(aDatasourceName) ; @@ -108,7 +107,6 @@ public class HadoopConfigHolder { init() ; } initLoginInfo(); - initClassLoader() ; } private HadoopConfigHolder(String aDatasourceName, Map<String,String> connectionProperties) { @@ -122,7 +120,6 @@ public class HadoopConfigHolder { this.defaultConfigFile = defaultConfigFile; initConnectionProp() ; initLoginInfo(); - initClassLoader() ; } private void initConnectionProp() { @@ -160,6 +157,13 @@ public class HadoopConfigHolder { if (in != null) { try { resourcemapProperties.load(in); + for (Map.Entry<Object, Object> entry : resourcemapProperties.entrySet() ) { + String key = (String)entry.getKey(); + String value = (String)entry.getValue(); + if (RANGER_SECTION_NAME.equals(value)) { + rangerInternalPropertyKeys.add(key); + } + } } catch (IOException e) { throw new HadoopException("Unable to load resource map properties from [" + RESOURCEMAP_PROP_FILE + "]", e); } @@ -261,11 +265,7 @@ public class HadoopConfigHolder { } } - - private void initClassLoader() { - classLoader = new HadoopClassLoader(this) ; - } - + public Properties getRangerSection() { Properties prop = this.getProperties(RANGER_SECTION_NAME) ; @@ -317,7 +317,7 @@ public class HadoopConfigHolder { return datasourceName ; } - public boolean hasResourceExists(String aResourceName) { + public boolean hasResourceExists(String aResourceName) { // dilli HashMap<String,Properties> resourceName2PropertiesMap = dataSource2ResourceListMap.get(datasourceName) ; return (resourceName2PropertiesMap != null && resourceName2PropertiesMap.containsKey(aResourceName)) ; } @@ -359,15 +359,14 @@ public class HadoopConfigHolder { return password; } - public HadoopClassLoader getClassLoader() { - return classLoader; - } - public boolean isKerberosAuthentication() { return isKerberosAuth; } - + public Set<String> getRangerInternalPropertyKeys() { + return rangerInternalPropertyKeys; + + } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a200d82d/agents-common/src/main/resources/service-defs/ranger-servicedef-hdfs.json ---------------------------------------------------------------------- diff --git a/agents-common/src/main/resources/service-defs/ranger-servicedef-hdfs.json b/agents-common/src/main/resources/service-defs/ranger-servicedef-hdfs.json index bb9d428..925cc77 100755 --- a/agents-common/src/main/resources/service-defs/ranger-servicedef-hdfs.json +++ b/agents-common/src/main/resources/service-defs/ranger-servicedef-hdfs.json @@ -75,7 +75,7 @@ "name": "fs.default.name", "type": "string", "subType": "", - "mandatory": false, + "mandatory": true, "validationRegEx":"", "validationMessage": "", "uiHint":"", http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a200d82d/hbase-agent/src/main/java/org/apache/ranger/services/hbase/client/HBaseClient.java ---------------------------------------------------------------------- diff --git a/hbase-agent/src/main/java/org/apache/ranger/services/hbase/client/HBaseClient.java b/hbase-agent/src/main/java/org/apache/ranger/services/hbase/client/HBaseClient.java index 320e084..e051bb9 100644 --- a/hbase-agent/src/main/java/org/apache/ranger/services/hbase/client/HBaseClient.java +++ b/hbase-agent/src/main/java/org/apache/ranger/services/hbase/client/HBaseClient.java @@ -21,11 +21,7 @@ import java.io.IOException; import java.security.PrivilegedAction; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.Map.Entry; import javax.security.auth.Subject; @@ -50,12 +46,27 @@ public class HBaseClient extends BaseClient { private static Subject subj = null; - public HBaseClient(String serivceName) { - super(serivceName) ; - } + private Configuration conf; + + private static List<String> rangerInternalPropertyKeys = Arrays.asList("username", + "password", "keytabfile"); public HBaseClient(String serivceName,Map<String,String> connectionProp) { - super(serivceName, addDefaultHBaseProp(connectionProp)) ; + + super(serivceName, addDefaultHBaseProp(connectionProp)) ; + conf = HBaseConfiguration.create() ; + + Set<String> rangerInternalPropertyKeys = getConfigHolder().getRangerInternalPropertyKeys(); + for (Map.Entry<String, String> entry: connectionProperties.entrySet()) { + String key = entry.getKey(); + String value = entry.getValue(); + if (rangerInternalPropertyKeys.contains(key)) { + // skip + } else { + conf.set(key, value); + } + } + } //TODO: temporary solution - to be added to the UI for HBase @@ -113,17 +124,15 @@ public class HBaseClient extends BaseClient { + "policies, but you would not be able to use autocomplete for " + "resource names. Check xa_portal.log for more info."; if (subj != null) { - ClassLoader prevCl = Thread.currentThread().getContextClassLoader() ; try { - Thread.currentThread().setContextClassLoader(getConfigHolder().getClassLoader()); - + hbaseStatus = Subject.doAs(subj, new PrivilegedAction<Boolean>() { @Override public Boolean run() { Boolean hbaseStatus1 = false; try { LOG.info("getHBaseStatus: creating default Hbase configuration"); - Configuration conf = HBaseConfiguration.create() ; + LOG.info("getHBaseStatus: setting config values from client"); setClientConfigValues(conf); LOG.info("getHBaseStatus: checking HbaseAvailability with the new config"); @@ -184,15 +193,12 @@ public class HBaseClient extends BaseClient { } }) ; } catch (SecurityException se) { - String msgDesc = "getHBaseStatus: Unable to connect to HBase Server instance, " - + "current thread might not be able set the context ClassLoader."; + String msgDesc = "getHBaseStatus: Unable to connect to HBase Server instance "; HadoopException hdpException = new HadoopException(msgDesc, se); hdpException.generateResponseDataMap(false, getMessage(se), msgDesc + errMsg, null, null); LOG.error(msgDesc + se) ; throw hdpException; - } finally { - Thread.currentThread().setContextClassLoader(prevCl); } } else { LOG.error("getHBaseStatus: secure login not done, subject is null"); @@ -222,10 +228,6 @@ public class HBaseClient extends BaseClient { subj = getLoginSubject(); if (subj != null) { - ClassLoader prevCl = Thread.currentThread().getContextClassLoader() ; - try { - Thread.currentThread().setContextClassLoader(getConfigHolder().getClassLoader()); - ret = Subject.doAs(subj, new PrivilegedAction<List<String>>() { @Override @@ -235,7 +237,7 @@ public class HBaseClient extends BaseClient { HBaseAdmin admin = null ; try { LOG.info("getTableList: creating default Hbase configuration"); - Configuration conf = HBaseConfiguration.create() ; + Configuration conf = HBaseConfiguration.create() ; // dilli LOG.info("getTableList: setting config values from client"); setClientConfigValues(conf); LOG.info("getTableList: checking HbaseAvailability with the new config"); @@ -303,10 +305,6 @@ public class HBaseClient extends BaseClient { } }) ; - } - finally { - Thread.currentThread().setContextClassLoader(prevCl); - } } return ret ; } @@ -320,10 +318,8 @@ public class HBaseClient extends BaseClient { subj = getLoginSubject(); if (subj != null) { - ClassLoader prevCl = Thread.currentThread().getContextClassLoader() ; try { - Thread.currentThread().setContextClassLoader(getConfigHolder().getClassLoader()); - + ret = Subject.doAs(subj, new PrivilegedAction<List<String>>() { String tblName = null; @Override @@ -332,7 +328,7 @@ public class HBaseClient extends BaseClient { HBaseAdmin admin = null ; try { LOG.info("getColumnFamilyList: creating default Hbase configuration"); - Configuration conf = HBaseConfiguration.create() ; + Configuration conf = HBaseConfiguration.create() ; // dilli LOG.info("getColumnFamilyList: setting config values from client"); setClientConfigValues(conf); LOG.info("getColumnFamilyList: checking HbaseAvailability with the new config"); @@ -382,8 +378,7 @@ public class HBaseClient extends BaseClient { } catch(IOException io) { String msgDesc = "getColumnFamilyList: Unable to get HBase ColumnFamilyList for " + "[repository:" +getConfigHolder().getDatasourceName() + ",table:" + tblName - + ", table-match:" + columnFamilyMatching + "], " - + "current thread might not be able set the context ClassLoader."; + + ", table-match:" + columnFamilyMatching + "] "; HadoopException hdpException = new HadoopException(msgDesc, io); hdpException.generateResponseDataMap(false, getMessage(io), msgDesc + errMsg, null, null); @@ -392,8 +387,7 @@ public class HBaseClient extends BaseClient { } catch (SecurityException se) { String msgDesc = "getColumnFamilyList: Unable to get HBase ColumnFamilyList for " + "[repository:" +getConfigHolder().getDatasourceName() + ",table:" + tblName - + ", table-match:" + columnFamilyMatching + "], " - + "current thread might not be able set the context ClassLoader."; + + ", table-match:" + columnFamilyMatching + "] "; HadoopException hdpException = new HadoopException(msgDesc, se); hdpException.generateResponseDataMap(false, getMessage(se), msgDesc + errMsg, null, null); @@ -403,8 +397,7 @@ public class HBaseClient extends BaseClient { } catch (Throwable e) { String msgDesc = "getColumnFamilyList: Unable to get HBase ColumnFamilyList for " + "[repository:" +getConfigHolder().getDatasourceName() + ",table:" + tblName - + ", table-match:" + columnFamilyMatching + "], " - + "current thread might not be able set the context ClassLoader."; + + ", table-match:" + columnFamilyMatching + "] "; LOG.error(msgDesc); HadoopException hdpException = new HadoopException(msgDesc, e); hdpException.generateResponseDataMap(false, getMessage(e), @@ -426,15 +419,12 @@ public class HBaseClient extends BaseClient { }) ; } catch (SecurityException se) { - String msgDesc = "getColumnFamilyList: Unable to connect to HBase Server instance, " - + "current thread might not be able set the context ClassLoader."; + String msgDesc = "getColumnFamilyList: Unable to connect to HBase Server instance "; HadoopException hdpException = new HadoopException(msgDesc, se); hdpException.generateResponseDataMap(false, getMessage(se), msgDesc + errMsg, null, null); LOG.error(msgDesc + se) ; throw hdpException; - } finally { - Thread.currentThread().setContextClassLoader(prevCl); } } return ret ; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a200d82d/hbase-agent/src/main/java/org/apache/ranger/services/hbase/client/HBaseConnectionMgr.java ---------------------------------------------------------------------- diff --git a/hbase-agent/src/main/java/org/apache/ranger/services/hbase/client/HBaseConnectionMgr.java b/hbase-agent/src/main/java/org/apache/ranger/services/hbase/client/HBaseConnectionMgr.java index ef0b9ae..5c1c73b 100644 --- a/hbase-agent/src/main/java/org/apache/ranger/services/hbase/client/HBaseConnectionMgr.java +++ b/hbase-agent/src/main/java/org/apache/ranger/services/hbase/client/HBaseConnectionMgr.java @@ -58,7 +58,7 @@ public class HBaseConnectionMgr { HBaseClient hBaseClient=null; if(serviceName!=null){ try{ - hBaseClient=new HBaseClient(serviceName); + hBaseClient=new HBaseClient(serviceName, configs); }catch(Exception ex){ LOG.error("Error connecting HBase repository : ", ex); } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a200d82d/hdfs-agent/src/main/java/org/apache/ranger/services/hdfs/client/HdfsClient.java ---------------------------------------------------------------------- diff --git a/hdfs-agent/src/main/java/org/apache/ranger/services/hdfs/client/HdfsClient.java b/hdfs-agent/src/main/java/org/apache/ranger/services/hdfs/client/HdfsClient.java index 1d1eef9..779133f 100644 --- a/hdfs-agent/src/main/java/org/apache/ranger/services/hdfs/client/HdfsClient.java +++ b/hdfs-agent/src/main/java/org/apache/ranger/services/hdfs/client/HdfsClient.java @@ -23,10 +23,7 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.net.UnknownHostException; import java.security.PrivilegedAction; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import javax.security.auth.Subject; @@ -44,30 +41,39 @@ import org.apache.ranger.plugin.client.HadoopException; public class HdfsClient extends BaseClient { private static final Log LOG = LogFactory.getLog(HdfsClient.class) ; + private Configuration conf; + private static List<String> rangerInternalPropertyKeys = Arrays.asList("username", + "password", "keytabfile"); - public HdfsClient(String serviceName) { - super(serviceName) ; - } - public HdfsClient(String serviceName, Map<String,String> connectionProperties) { super(serviceName,connectionProperties, "hdfs-client") ; + conf = new Configuration() ; + Set<String> rangerInternalPropertyKeys = getConfigHolder().getRangerInternalPropertyKeys(); + for (Map.Entry<String, String> entry: connectionProperties.entrySet()) { + String key = entry.getKey(); + String value = entry.getValue(); + if (rangerInternalPropertyKeys.contains(key)) { + // skip + } else { + conf.set(key, value); + } + } + } private List<String> listFilesInternal(String baseDir, String fileMatching, final List<String> pathList) { List<String> fileList = new ArrayList<String>() ; - ClassLoader prevCl = Thread.currentThread().getContextClassLoader() ; String errMsg = " You can still save the repository and start creating " + "policies, but you would not be able to use autocomplete for " + "resource names. Check xa_portal.log for more info."; try { - Thread.currentThread().setContextClassLoader(getConfigHolder().getClassLoader()); String dirPrefix = (baseDir.endsWith("/") ? baseDir : (baseDir + "/")) ; String filterRegEx = null; if (fileMatching != null && fileMatching.trim().length() > 0) { filterRegEx = fileMatching.trim() ; } - Configuration conf = new Configuration() ; + UserGroupInformation.setConfiguration(conf); FileSystem fs = null ; @@ -147,9 +153,6 @@ public class HdfsClient extends BaseClient { } throw hdpException; } - finally { - Thread.currentThread().setContextClassLoader(prevCl); - } return fileList ; } @@ -177,7 +180,7 @@ public class HdfsClient extends BaseClient { String baseDir = args[1] ; String fileNameToMatch = (args.length == 2 ? null : args[2]) ; - HdfsClient fs = new HdfsClient(repositoryName) ; + HdfsClient fs = new HdfsClient(repositoryName, null) ; List<String> fsList = fs.listFiles(baseDir, fileNameToMatch,null) ; if (fsList != null && fsList.size() > 0) { for(String s : fsList) { @@ -252,25 +255,25 @@ public class HdfsClient extends BaseClient { String fsDefaultName = configs.get("fs.default.name") ; fsDefaultName = (fsDefaultName == null) ? "" : fsDefaultName.trim(); - String dfsNameservices = configs.get("dfs.nameservices"); - dfsNameservices = (dfsNameservices == null) ? "" : dfsNameservices.trim(); - - if (fsDefaultName.isEmpty() && dfsNameservices.isEmpty()) { - throw new IllegalArgumentException("Value for neither fs.default.name nor dfs.nameservices is specified"); - } - - if (!fsDefaultName.isEmpty() && !dfsNameservices.isEmpty()) { - throw new IllegalArgumentException("Value for both fs.default.name and dfs.nameservices are specified. They are mutually exclusive"); + if (fsDefaultName.isEmpty()) { + throw new IllegalArgumentException("Value for neither fs.default.name is specified"); } + String dfsNameservices = configs.get("dfs.nameservices"); + dfsNameservices = (dfsNameservices == null) ? "" : dfsNameservices.trim(); if (!dfsNameservices.isEmpty()) { + String proxyProvider = configs.get("dfs.client.failover.proxy.provider." + dfsNameservices); + proxyProvider = (proxyProvider == null) ? "" : proxyProvider.trim(); + if (proxyProvider.isEmpty()) { + throw new IllegalArgumentException("Value for " + "dfs.client.failover.proxy.provider." + dfsNameservices + " not specified"); + } + String dfsNameNodes = configs.get("dfs.ha.namenodes." + dfsNameservices); dfsNameNodes = (dfsNameNodes == null) ? "" : dfsNameNodes.trim(); if (dfsNameNodes.isEmpty()) { - throw new IllegalArgumentException("Value for " + "dfs.ha.namenodes." + dfsNameservices + " not specified"); + throw new IllegalArgumentException("Value for " + "dfs.ha.namenodes." + proxyProvider + " not specified"); } String[] dfsNameNodeElements = dfsNameNodes.split(","); - System.out.println("elements: " + dfsNameNodeElements); for (String dfsNameNodeElement : dfsNameNodeElements) { String nameNodeUrlKey = "dfs.namenode.rpc-address." + dfsNameservices + "." + dfsNameNodeElement.trim(); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a200d82d/hdfs-agent/src/main/java/org/apache/ranger/services/hdfs/client/HdfsConnectionMgr.java ---------------------------------------------------------------------- diff --git a/hdfs-agent/src/main/java/org/apache/ranger/services/hdfs/client/HdfsConnectionMgr.java b/hdfs-agent/src/main/java/org/apache/ranger/services/hdfs/client/HdfsConnectionMgr.java index c329a94..d62bb9c 100644 --- a/hdfs-agent/src/main/java/org/apache/ranger/services/hdfs/client/HdfsConnectionMgr.java +++ b/hdfs-agent/src/main/java/org/apache/ranger/services/hdfs/client/HdfsConnectionMgr.java @@ -54,7 +54,7 @@ public class HdfsConnectionMgr { final Callable<HdfsClient> connectHDFS = new Callable<HdfsClient>() { @Override public HdfsClient call() throws Exception { - return new HdfsClient(serviceName); + return new HdfsClient(serviceName, configs); } }; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a200d82d/hdfs-agent/src/test/java/org/apache/ranger/services/hdfs/client/HdfsClientTest.java ---------------------------------------------------------------------- diff --git a/hdfs-agent/src/test/java/org/apache/ranger/services/hdfs/client/HdfsClientTest.java b/hdfs-agent/src/test/java/org/apache/ranger/services/hdfs/client/HdfsClientTest.java index b5ccbf0..5e4fa57 100644 --- a/hdfs-agent/src/test/java/org/apache/ranger/services/hdfs/client/HdfsClientTest.java +++ b/hdfs-agent/src/test/java/org/apache/ranger/services/hdfs/client/HdfsClientTest.java @@ -48,7 +48,7 @@ public class HdfsClientTest { } @Test(expected = IllegalArgumentException.class) - public void testNeietherNnNorHaSpecified() throws IllegalArgumentException { + public void testFsDefaultNameNotSpecified() throws IllegalArgumentException { Map configs = new HashMap<String, String>(); configs.put("username", "hdfsuser"); configs.put("password", "hdfsuser"); @@ -57,15 +57,15 @@ public class HdfsClientTest { } @Test(expected = IllegalArgumentException.class) - public void testBothNnAndHaSpecified() throws IllegalArgumentException { + public void testProxyProviderNotSpecified() throws IllegalArgumentException { Map configs = new HashMap<String, String>(); configs.put("username", "hdfsuser"); configs.put("password", "hdfsuser"); configs.put("hadoop.security.authentication", "simple"); - configs.put("fs.default.name", "hdfs://node-2.example.com:8020"); + configs.put("fs.default.name", "hdfs://hwqe-1425428405"); configs.put("dfs.nameservices", "hwqe-1425428405"); HdfsClient.validateConnectionConfigs(configs); - } + } @Test(expected = IllegalArgumentException.class) public void testNnElementsNotSpecified() throws IllegalArgumentException { @@ -73,7 +73,10 @@ public class HdfsClientTest { configs.put("username", "hdfsuser"); configs.put("password", "hdfsuser"); configs.put("hadoop.security.authentication", "simple"); + configs.put("fs.default.name", "hdfs://hwqe-1425428405"); configs.put("dfs.nameservices", "hwqe-1425428405"); + configs.put("dfs.client.failover.proxy.provider.hwqe-1425428405", + "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"); HdfsClient.validateConnectionConfigs(configs); } @@ -83,7 +86,10 @@ public class HdfsClientTest { configs.put("username", "hdfsuser"); configs.put("password", "hdfsuser"); configs.put("hadoop.security.authentication", "simple"); + configs.put("fs.default.name", "hdfs://hwqe-1425428405"); configs.put("dfs.nameservices", "hwqe-1425428405"); + configs.put("dfs.client.failover.proxy.provider.hwqe-1425428405", + "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"); configs.put("dfs.ha.namenodes.hwqe-1425428405", "nn1,nn2"); HdfsClient.validateConnectionConfigs(configs); } @@ -94,7 +100,10 @@ public class HdfsClientTest { configs.put("username", "hdfsuser"); configs.put("password", "hdfsuser"); configs.put("hadoop.security.authentication", "simple"); + configs.put("fs.default.name", "hdfs://hwqe-1425428405"); configs.put("dfs.nameservices", "hwqe-1425428405"); + configs.put("dfs.client.failover.proxy.provider.hwqe-1425428405", + "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"); configs.put("dfs.ha.namenodes.hwqe-1425428405", "nn1,nn2"); configs.put("dfs.namenode.rpc-address.hwqe-1425428405.nn2", "node-2.example.com:8020"); HdfsClient.validateConnectionConfigs(configs); @@ -106,7 +115,10 @@ public class HdfsClientTest { configs.put("username", "hdfsuser"); configs.put("password", "hdfsuser"); configs.put("hadoop.security.authentication", "simple"); + configs.put("fs.default.name", "hdfs://hwqe-1425428405"); configs.put("dfs.nameservices", "hwqe-1425428405"); + configs.put("dfs.client.failover.proxy.provider.hwqe-1425428405", + "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"); configs.put("dfs.ha.namenodes.hwqe-1425428405", "nn1,nn2"); configs.put("dfs.namenode.rpc-address.hwqe-1425428405.nn1", "node-1.example.com:8020"); HdfsClient.validateConnectionConfigs(configs); @@ -144,8 +156,11 @@ public class HdfsClientTest { configs.put("username", "hdfsuser"); configs.put("password", "hdfsuser"); configs.put("hadoop.security.authentication", "simple"); - // configs.put("fs.default.name", "hdfs://node-2.example.com:8020"); + configs.put("fs.default.name", "hdfs://node-2.example.com:8020"); + configs.put("fs.default.name", "hdfs://hwqe-1425428405"); configs.put("dfs.nameservices", "hwqe-1425428405"); + configs.put("dfs.client.failover.proxy.provider.hwqe-1425428405", + "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"); configs.put("dfs.ha.namenodes.hwqe-1425428405", "nn1,nn2"); configs.put("dfs.namenode.rpc-address.hwqe-1425428405.nn1", "node-1.example.com:8020"); configs.put("dfs.namenode.rpc-address.hwqe-1425428405.nn2", "node-2.example.com:8020"); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a200d82d/hive-agent/src/main/java/org/apache/ranger/services/hive/client/HiveClient.java ---------------------------------------------------------------------- diff --git a/hive-agent/src/main/java/org/apache/ranger/services/hive/client/HiveClient.java b/hive-agent/src/main/java/org/apache/ranger/services/hive/client/HiveClient.java index da08240..40efe2e 100644 --- a/hive-agent/src/main/java/org/apache/ranger/services/hive/client/HiveClient.java +++ b/hive-agent/src/main/java/org/apache/ranger/services/hive/client/HiveClient.java @@ -51,7 +51,7 @@ public class HiveClient extends BaseClient implements Closeable { public HiveClient(String serviceName) { - super(serviceName) ; + super(serviceName, null) ; initHive() ; } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a200d82d/lookup-client/src/main/java/org/apache/ranger/hadoop/client/config/HadoopClassLoader.java ---------------------------------------------------------------------- diff --git a/lookup-client/src/main/java/org/apache/ranger/hadoop/client/config/HadoopClassLoader.java b/lookup-client/src/main/java/org/apache/ranger/hadoop/client/config/HadoopClassLoader.java deleted file mode 100644 index bb13538..0000000 --- a/lookup-client/src/main/java/org/apache/ranger/hadoop/client/config/HadoopClassLoader.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - package org.apache.ranger.hadoop.client.config; - -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.io.PrintWriter; -import java.net.URL; -import java.util.Properties; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.ranger.hadoop.client.exceptions.HadoopException; - -public class HadoopClassLoader extends ClassLoader { - - private static final Log LOG = LogFactory.getLog(HadoopClassLoader.class) ; - - private HadoopConfigHolder confHolder ; - - public HadoopClassLoader(HadoopConfigHolder confHolder) { - super(Thread.currentThread().getContextClassLoader()) ; - this.confHolder = confHolder; - } - - - @Override - protected URL findResource(String resourceName) { - LOG.debug("findResource(" + resourceName + ") is called.") ; - URL ret = null; - - if (confHolder.hasResourceExists(resourceName)) { - ret = buildResourceFile(resourceName) ; - } - else { - ret = super.findResource(resourceName); - } - LOG.debug("findResource(" + resourceName + ") is returning [" + ret + "]") ; - return ret ; - } - - - @SuppressWarnings("deprecation") - private URL buildResourceFile(String aResourceName) { - URL ret = null ; - String prefix = aResourceName ; - String suffix = ".txt" ; - - Properties prop = confHolder.getProperties(aResourceName) ; - LOG.debug("Building XML for: " + prop.toString()); - if (prop != null && prop.size() > 0) { - if (aResourceName.contains(".")) { - int lastDotFound = aResourceName.indexOf(".") ; - prefix = aResourceName.substring(0,lastDotFound) + "-" ; - suffix = aResourceName.substring(lastDotFound) ; - } - - try { - File tempFile = File.createTempFile(prefix, suffix) ; - tempFile.deleteOnExit(); - PrintWriter out = new PrintWriter(new FileWriter(tempFile)) ; - out.println("<?xml version=\"1.0\"?>") ; - out.println("<?xml-stylesheet type=\"text/xsl\" href=\"configuration.xsl\"?>") ; - out.println("<configuration xmlns:xi=\"http://www.w3.org/2001/XInclude\";>") ; - for(Object keyobj : prop.keySet()) { - String key = (String)keyobj; - String val = prop.getProperty(key) ; - if (HadoopConfigHolder.HADOOP_RPC_PROTECTION.equals(key) && (val == null || val.trim().isEmpty())) { - continue; - } - out.println("<property><name>" + key.trim() + "</name><value>" + val + "</value></property>") ; - } - out.println("</configuration>") ; - out.close() ; - ret = tempFile.toURL() ; - } catch (IOException e) { - throw new HadoopException("Unable to load create hadoop configuration file [" + aResourceName + "]", e) ; - } - - } - - return ret ; - - } - - -} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a200d82d/lookup-client/src/main/java/org/apache/ranger/hadoop/client/config/HadoopConfigHolder.java ---------------------------------------------------------------------- diff --git a/lookup-client/src/main/java/org/apache/ranger/hadoop/client/config/HadoopConfigHolder.java b/lookup-client/src/main/java/org/apache/ranger/hadoop/client/config/HadoopConfigHolder.java index 7651bb5..f9b3eee 100644 --- a/lookup-client/src/main/java/org/apache/ranger/hadoop/client/config/HadoopConfigHolder.java +++ b/lookup-client/src/main/java/org/apache/ranger/hadoop/client/config/HadoopConfigHolder.java @@ -56,8 +56,7 @@ public class HadoopConfigHolder { private String password ; private boolean isKerberosAuth ; - private HadoopClassLoader classLoader ; - private HashMap<String,String> connectionProperties; + private HashMap<String,String> connectionProperties; public static HadoopConfigHolder getInstance(String aDatasourceName) { HadoopConfigHolder ret = dataSource2HadoopConfigHolder.get(aDatasourceName) ; @@ -102,7 +101,6 @@ public class HadoopConfigHolder { init() ; } initLoginInfo(); - initClassLoader() ; } private HadoopConfigHolder(String aDatasourceName, HashMap<String,String> connectionProperties) { @@ -110,7 +108,6 @@ public class HadoopConfigHolder { this.connectionProperties = connectionProperties ; initConnectionProp() ; initLoginInfo(); - initClassLoader() ; } private void initConnectionProp() { @@ -248,12 +245,7 @@ public class HadoopConfigHolder { } } - - private void initClassLoader() { - classLoader = new HadoopClassLoader(this) ; - } - - + public Properties getRangerSection() { Properties prop = this.getProperties(RANGER_SECTION_NAME) ; if (prop == null) { @@ -346,15 +338,8 @@ public class HadoopConfigHolder { return password; } - public HadoopClassLoader getClassLoader() { - return classLoader; - } - public boolean isKerberosAuthentication() { return isKerberosAuth; } - - - } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a200d82d/security-admin/src/main/resources/resourcenamemap.properties ---------------------------------------------------------------------- diff --git a/security-admin/src/main/resources/resourcenamemap.properties b/security-admin/src/main/resources/resourcenamemap.properties index 7a9e89c..201c0fa 100644 --- a/security-admin/src/main/resources/resourcenamemap.properties +++ b/security-admin/src/main/resources/resourcenamemap.properties @@ -13,22 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -fs.default.name=core-site.xml -hadoop.security.authentication=core-site.xml -hadoop.security.authorization=core-site.xml -hadoop.security.auth_to_local=core-site.xml -hadoop.rpc.protection=core-site.xml -dfs.nameservices=hdfs-site.xml -dfs.datanode.kerberos.principal=hdfs-site.xml -dfs.namenode.kerberos.principal=hdfs-site.xml -dfs.secondary.namenode.kerberos.principal=hdfs-site.xml username=xalogin.xml keytabfile=xalogin.xml password=xalogin.xml -hbase.master.kerberos.principal=hbase-site.xml -hbase.rpc.engine=hbase-site.xml -hbase.rpc.protection=hbase-site.xml -hbase.security.authentication=hbase-site.xml -hbase.zookeeper.property.clientPort=hbase-site.xml -hbase.zookeeper.quorum=hbase-site.xml -zookeeper.znode.parent=hbase-site.xml
