Repository: incubator-ranger Updated Branches: refs/heads/master a733b7c33 -> af8377f27
RANGER-754:Ranger YARN Plugin lookup and test connection should support SPENGO enabled HTTP Authentication Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/90b7f0ba Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/90b7f0ba Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/90b7f0ba Branch: refs/heads/master Commit: 90b7f0ba3f4bfa16060709e643b48a017ff43863 Parents: 0dadcd1 Author: rmani <[email protected]> Authored: Wed Dec 2 15:31:37 2015 -0800 Committer: rmani <[email protected]> Committed: Wed Dec 2 15:31:37 2015 -0800 ---------------------------------------------------------------------- .../plugin/client/HadoopConfigHolder.java | 62 +++-- .../service-defs/ranger-servicedef-yarn.json | 35 ++- .../ranger/services/yarn/client/YarnClient.java | 228 ++++++++++--------- .../services/yarn/client/YarnConnectionMgr.java | 18 +- .../services/yarn/client/YarnResourceMgr.java | 14 +- 5 files changed, 217 insertions(+), 140 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/90b7f0ba/agents-common/src/main/java/org/apache/ranger/plugin/client/HadoopConfigHolder.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/client/HadoopConfigHolder.java b/agents-common/src/main/java/org/apache/ranger/plugin/client/HadoopConfigHolder.java index f95e10e..9d14ae6 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/client/HadoopConfigHolder.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/client/HadoopConfigHolder.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.io.InputStream; import java.util.*; +import java.util.Map.Entry; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -96,6 +97,7 @@ public class HadoopConfigHolder { dataSource2HadoopConfigHolder.put(aDatasourceName, ret) ; } } + return ret ; } @@ -265,14 +267,15 @@ public class HadoopConfigHolder { userName = prop.getProperty(RANGER_LOGIN_USER_NAME_PROP) ; keyTabFile = prop.getProperty(RANGER_LOGIN_KEYTAB_FILE_PROP) ; password = prop.getProperty(RANGER_LOGIN_PASSWORD) ; - - if ( getHadoopSecurityAuthentication() != null) { - isKerberosAuth = ( getHadoopSecurityAuthentication().equalsIgnoreCase(HADOOP_SECURITY_AUTHENTICATION_METHOD)); + + String hadoopSecurityAuthenticationn = getHadoopSecurityAuthentication(); + + if ( hadoopSecurityAuthenticationn != null) { + isKerberosAuth = ( hadoopSecurityAuthenticationn.equalsIgnoreCase(HADOOP_SECURITY_AUTHENTICATION_METHOD)); } else { isKerberosAuth = (userName != null) && (userName.indexOf("@") > -1) ; } - } } @@ -342,21 +345,26 @@ public class HadoopConfigHolder { } public String getHadoopSecurityAuthentication() { - Properties repoParam = null ; String ret = null; - - HashMap<String,Properties> resourceName2PropertiesMap = dataSource2ResourceListMap.get(this.getDatasourceName()) ; - - if ( resourceName2PropertiesMap != null) { - repoParam=resourceName2PropertiesMap.get(DEFAULT_RESOURCE_NAME); + String sectionName = RANGER_SECTION_NAME; + + if ( defaultConfigFile != null) { + sectionName = defaultConfigFile; + } + + if ( LOG.isDebugEnabled() ) { + LOG.debug("==> HadoopConfigHolder.getHadoopSecurityAuthentication( " + " DataSource : " + sectionName + " Property : " + HADOOP_SECURITY_AUTHENTICATION + ")" ); } + + ret = getProperties(sectionName,HADOOP_SECURITY_AUTHENTICATION); - if ( repoParam != null ) { - ret = (String)repoParam.get(HADOOP_SECURITY_AUTHENTICATION); + if ( LOG.isDebugEnabled() ) { + LOG.debug("<== HadoopConfigHolder.getHadoopSecurityAuthentication(" + " DataSource : " + sectionName + " Property : " + HADOOP_SECURITY_AUTHENTICATION + " Value : " + ret + ")" ); } + return ret; - } - + } + public String getUserName() { return userName; } @@ -377,6 +385,32 @@ public class HadoopConfigHolder { return rangerInternalPropertyKeys; } + + private String getProperties(String sectionName, String property) { + + if ( LOG.isDebugEnabled() ) { + LOG.debug("==> HadoopConfigHolder.getProperties( " + " DataSource : " + sectionName + " Property : " + property + ")" ); + } + + Properties repoParam = null ; + String ret = null; + + HashMap<String,Properties> resourceName2PropertiesMap = dataSource2ResourceListMap.get(this.getDatasourceName()) ; + + if ( resourceName2PropertiesMap != null) { + repoParam=resourceName2PropertiesMap.get(sectionName); + } + + if ( repoParam != null ) { + ret = (String)repoParam.get(property); + } + + if ( LOG.isDebugEnabled() ) { + LOG.debug("<== HadoopConfigHolder.getProperties( " + " DataSource : " + sectionName + " Property : " + property + " Value : " + ret); + } + + return ret; + } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/90b7f0ba/agents-common/src/main/resources/service-defs/ranger-servicedef-yarn.json ---------------------------------------------------------------------- diff --git a/agents-common/src/main/resources/service-defs/ranger-servicedef-yarn.json b/agents-common/src/main/resources/service-defs/ranger-servicedef-yarn.json index ff1f39f..ff93dfe 100644 --- a/agents-common/src/main/resources/service-defs/ranger-servicedef-yarn.json +++ b/agents-common/src/main/resources/service-defs/ranger-servicedef-yarn.json @@ -82,6 +82,19 @@ { "itemId": 4, + "name": "hadoop.security.authentication", + "type": "enum", + "subType": "authnType", + "mandatory": true, + "validationRegEx":"", + "validationMessage": "", + "uiHint":"", + "label": "Authentication Type", + "defaultValue": "simple" + }, + + { + "itemId": 5, "name": "commonNameForCertificate", "type": "string", "mandatory": false, @@ -90,11 +103,31 @@ "uiHint":"", "label": "Common Name for Certificate" } + ], "enums": [ - + { + "itemId": 1, + "name": "authnType", + "elements": + [ + { + "itemId": 1, + "name": "simple", + "label": "Simple" + }, + + { + "itemId": 2, + "name": "kerberos", + "label": "Kerberos" + } + ], + + "defaultIndex": 0 + } ], "contextEnrichers": http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/90b7f0ba/plugin-yarn/src/main/java/org/apache/ranger/services/yarn/client/YarnClient.java ---------------------------------------------------------------------- diff --git a/plugin-yarn/src/main/java/org/apache/ranger/services/yarn/client/YarnClient.java b/plugin-yarn/src/main/java/org/apache/ranger/services/yarn/client/YarnClient.java index fc07760..0f3ed89 100644 --- a/plugin-yarn/src/main/java/org/apache/ranger/services/yarn/client/YarnClient.java +++ b/plugin-yarn/src/main/java/org/apache/ranger/services/yarn/client/YarnClient.java @@ -19,6 +19,9 @@ package org.apache.ranger.services.yarn.client; +import java.net.Authenticator; +import java.net.PasswordAuthentication; +import java.security.PrivilegedAction; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -26,6 +29,7 @@ import java.util.Map; import java.util.concurrent.Callable; import java.util.concurrent.TimeUnit; +import javax.security.auth.Subject; import org.apache.commons.io.FilenameUtils; import org.apache.log4j.Logger; @@ -40,7 +44,7 @@ import com.sun.jersey.api.client.Client; import com.sun.jersey.api.client.ClientResponse; import com.sun.jersey.api.client.WebResource; -public class YarnClient { +public class YarnClient extends BaseClient { public static final Logger LOG = Logger.getLogger(YarnClient.class) ; @@ -57,16 +61,27 @@ public class YarnClient { String userName; String password; - public YarnClient(String yarnQueueUrl, String yarnUserName, String yarnPassWord) { - - this.yarnQUrl = yarnQueueUrl; - this.userName = yarnUserName ; - this.password = yarnPassWord; + public YarnClient(String serviceName, Map<String, String> configs) { + + super(serviceName,configs,"yarn-client") ; + + this.yarnQUrl = configs.get("yarn.url"); + this.userName = configs.get("username"); + this.password = configs.get("password"); + if (this.yarnQUrl == null || this.yarnQUrl.isEmpty()) { + LOG.error("No value found for configuration 'yarn.url'. YARN resource lookup will fail"); + } + if (this.userName == null || this.userName.isEmpty()) { + LOG.error("No value found for configuration 'usename'. YARN resource lookup will fail"); + } + if (this.password == null || this.password.isEmpty()) { + LOG.error("No value found for configuration 'password'. YARN resource lookup will fail"); + } + if (LOG.isDebugEnabled()) { - LOG.debug("Yarn Client is build with url [" + yarnQueueUrl + "] user: [" + yarnPassWord + "], password: [" + "" + "]"); + LOG.debug("Yarn Client is build with url [" + this.yarnQUrl + "] user: [" + this.userName + "], password: [" + "*********" + "]"); } - } public List<String> getQueueList(final String queueNameMatching, final List<String> existingQueueList) { @@ -74,107 +89,122 @@ public class YarnClient { if (LOG.isDebugEnabled()) { LOG.debug("Getting Yarn queue list for queueNameMatching : " + queueNameMatching); } - final String errMsg = errMessage; + final String errMsg = errMessage; List<String> ret = null; - - Callable<List<String>> yarnQueueListGetter = new Callable<List<String>>() { + + Callable<List<String>> callableYarnQListGetter = new Callable<List<String>>() { + @Override public List<String> call() { - - List<String> lret = new ArrayList<String>(); - - String url = yarnQUrl + YARN_LIST_API_ENDPOINT ; - - Client client = null ; - ClientResponse response = null ; - - try { - client = Client.create() ; - - WebResource webResource = client.resource(url); - - response = webResource.accept(EXPECTED_MIME_TYPE) - .get(ClientResponse.class); - - if (LOG.isDebugEnabled()) { - LOG.debug("getQueueList():calling " + url); - } - - if (response != null) { - if (LOG.isDebugEnabled()) { - LOG.debug("getQueueList():response.getStatus()= " + response.getStatus()); - } - if (response.getStatus() == 200) { - String jsonString = response.getEntity(String.class); - Gson gson = new GsonBuilder().setPrettyPrinting().create(); - YarnSchedulerResponse yarnQResponse = gson.fromJson(jsonString, YarnSchedulerResponse.class); - if (yarnQResponse != null) { - List<String> yarnQueueList = yarnQResponse.getQueueNames(); - if (yarnQueueList != null) { - for ( String yarnQueueName : yarnQueueList) { - if ( existingQueueList != null && existingQueueList.contains(yarnQueueName)) { - continue; - } - if (queueNameMatching == null || queueNameMatching.isEmpty() - || yarnQueueName.startsWith(queueNameMatching)) { - if (LOG.isDebugEnabled()) { - LOG.debug("getQueueList():Adding yarnQueue " + yarnQueueName); + List<String> yarnQueueListGetter = null; + + Subject subj = getLoginSubject(); + + if (subj != null) { + yarnQueueListGetter = Subject.doAs(subj, new PrivilegedAction<List<String>>() { + + @Override + public List<String> run() { + + List<String> lret = new ArrayList<String>(); + + String url = yarnQUrl + YARN_LIST_API_ENDPOINT ; + + Client client = null ; + + ClientResponse response = null ; + + try { + client = Client.create() ; + + WebResource webResource = client.resource(url); + + response = webResource.accept(EXPECTED_MIME_TYPE) + .get(ClientResponse.class); + + if (LOG.isDebugEnabled()) { + LOG.debug("getQueueList():calling " + url); + } + + if (response != null) { + if (LOG.isDebugEnabled()) { + LOG.debug("getQueueList():response.getStatus()= " + response.getStatus()); + } + if (response.getStatus() == 200) { + String jsonString = response.getEntity(String.class); + Gson gson = new GsonBuilder().setPrettyPrinting().create(); + YarnSchedulerResponse yarnQResponse = gson.fromJson(jsonString, YarnSchedulerResponse.class); + if (yarnQResponse != null) { + List<String> yarnQueueList = yarnQResponse.getQueueNames(); + if (yarnQueueList != null) { + for ( String yarnQueueName : yarnQueueList) { + if ( existingQueueList != null && existingQueueList.contains(yarnQueueName)) { + continue; + } + if (queueNameMatching == null || queueNameMatching.isEmpty() + || yarnQueueName.startsWith(queueNameMatching)) { + if (LOG.isDebugEnabled()) { + LOG.debug("getQueueList():Adding yarnQueue " + yarnQueueName); + } + lret.add(yarnQueueName) ; + } } - lret.add(yarnQueueName) ; } } - } + } else{ + LOG.info("getQueueList():response.getStatus()= " + response.getStatus() + " for URL " + url + ", so returning null list"); + String jsonString = response.getEntity(String.class); + LOG.info(jsonString); + lret = null; } - } else{ - LOG.info("getQueueList():response.getStatus()= " + response.getStatus() + " for URL " + url + ", so returning null list"); - String jsonString = response.getEntity(String.class); - LOG.info(jsonString); + } else { + lret = null; + String msgDesc = "Unable to get a valid response for " + + "expected mime type : [" + EXPECTED_MIME_TYPE + + "] URL : " + url + " - got null response."; + LOG.error(msgDesc); + HadoopException hdpException = new HadoopException(msgDesc); + hdpException.generateResponseDataMap(false, msgDesc, + msgDesc + errMsg, null, null); + throw hdpException; + } + } catch (HadoopException he) { lret = null; + throw he; + } catch (Throwable t) { + lret = null; + String msgDesc = "Exception while getting Yarn Queue List." + + " URL : " + url; + HadoopException hdpException = new HadoopException(msgDesc, + t); + + LOG.error(msgDesc, t); + + hdpException.generateResponseDataMap(false, + BaseClient.getMessage(t), msgDesc + errMsg, null, + null); + throw hdpException; + + } finally { + if (response != null) { + response.close(); + } + + if (client != null) { + client.destroy(); + } } - } else { - lret = null; - String msgDesc = "Unable to get a valid response for " - + "expected mime type : [" + EXPECTED_MIME_TYPE - + "] URL : " + url + " - got null response."; - LOG.error(msgDesc); - HadoopException hdpException = new HadoopException(msgDesc); - hdpException.generateResponseDataMap(false, msgDesc, - msgDesc + errMsg, null, null); - throw hdpException; - } - } catch (HadoopException he) { - lret = null; - throw he; - } catch (Throwable t) { - lret = null; - String msgDesc = "Exception while getting Yarn Queue List." - + " URL : " + url; - HadoopException hdpException = new HadoopException(msgDesc, - t); - - LOG.error(msgDesc, t); - - hdpException.generateResponseDataMap(false, - BaseClient.getMessage(t), msgDesc + errMsg, null, - null); - throw hdpException; - - } finally { - if (response != null) { - response.close(); - } - - if (client != null) { - client.destroy(); + return lret ; } + } ); } - return lret ; - } - } ; + return yarnQueueListGetter; + } + }; try { - ret = timedTask(yarnQueueListGetter, 5, TimeUnit.SECONDS); + ret = timedTask(callableYarnQListGetter, 5, TimeUnit.SECONDS); } catch ( Throwable t) { LOG.error("Unable to get Yarn Queue list from [" + yarnQUrl + "]", t) ; String msgDesc = "Unable to get a valid response for " @@ -243,12 +273,7 @@ public class YarnClient { + errMsg, null, null); throw hdpException; } else { - String yarnUrl = configs.get("yarn.url"); - String yarnUserName = configs.get("username"); - String yarnPassWord = configs.get("password"); - yarnClient = new YarnClient (yarnUrl, yarnUserName, - yarnPassWord); - + yarnClient = new YarnClient (serviceName, configs); } return yarnClient; } @@ -299,5 +324,4 @@ public class YarnClient { TimeUnit timeUnit) throws Exception { return callableObj.call(); } - } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/90b7f0ba/plugin-yarn/src/main/java/org/apache/ranger/services/yarn/client/YarnConnectionMgr.java ---------------------------------------------------------------------- diff --git a/plugin-yarn/src/main/java/org/apache/ranger/services/yarn/client/YarnConnectionMgr.java b/plugin-yarn/src/main/java/org/apache/ranger/services/yarn/client/YarnConnectionMgr.java index e2cc2ef..1d39998 100644 --- a/plugin-yarn/src/main/java/org/apache/ranger/services/yarn/client/YarnConnectionMgr.java +++ b/plugin-yarn/src/main/java/org/apache/ranger/services/yarn/client/YarnConnectionMgr.java @@ -19,25 +19,17 @@ package org.apache.ranger.services.yarn.client; +import java.util.Map; + import org.apache.log4j.Logger; public class YarnConnectionMgr { public static final Logger LOG = Logger.getLogger(YarnConnectionMgr.class); - - public static YarnClient getYarnClient(final String yarnURL, String userName, String password) { - YarnClient yarnClient = null; - if (yarnURL == null || yarnURL.isEmpty()) { - LOG.error("Can not create YarnClient: yarnURL is empty"); - } else if (userName == null || userName.isEmpty()) { - LOG.error("Can not create YarnClient: YarnuserName is empty"); - } else if (password == null || password.isEmpty()) { - LOG.error("Can not create YarnClient: YarnPassWord is empty"); - } else { - yarnClient = new YarnClient(yarnURL, userName, password); - } - return yarnClient; + + public static YarnClient getYarnClient(String serviceName, Map<String, String> configs) { + return new YarnClient(serviceName, configs); } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/90b7f0ba/plugin-yarn/src/main/java/org/apache/ranger/services/yarn/client/YarnResourceMgr.java ---------------------------------------------------------------------- diff --git a/plugin-yarn/src/main/java/org/apache/ranger/services/yarn/client/YarnResourceMgr.java b/plugin-yarn/src/main/java/org/apache/ranger/services/yarn/client/YarnResourceMgr.java index 95d29c0..97fdf19 100644 --- a/plugin-yarn/src/main/java/org/apache/ranger/services/yarn/client/YarnResourceMgr.java +++ b/plugin-yarn/src/main/java/org/apache/ranger/services/yarn/client/YarnResourceMgr.java @@ -65,23 +65,17 @@ public class YarnResourceMgr { } else { yarnQueueName = userInput; } - - + if (configs == null || configs.isEmpty()) { LOG.error("Connection Config is empty"); - } else { - - String url = configs.get("yarn.url"); - String username = configs.get("username"); - String password = configs.get("password"); - resultList = getYarnResource(url, username, password,yarnQueueName,yarnQueueList) ; + resultList = getYarnResource(serviceName, configs, yarnQueueName,yarnQueueList) ; } return resultList ; } - public static List<String> getYarnResource(String url, String username, String password,String yarnQueueName, List<String> yarnQueueList) { - final YarnClient yarnClient = YarnConnectionMgr.getYarnClient(url, username, password); + public static List<String> getYarnResource(String serviceName, Map<String, String> configs, String yarnQueueName, List<String> yarnQueueList) { + final YarnClient yarnClient = YarnConnectionMgr.getYarnClient(serviceName, configs); List<String> topologyList = null; if (yarnClient != null) { synchronized(yarnClient) {
