RANGER-313: Ranger Admin to load plugin classes in a child class-loader to avoid potential library conflicts
Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/d017b31f Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/d017b31f Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/d017b31f Branch: refs/heads/master Commit: d017b31f913087613623a1cf711f3455153845ad Parents: 7bb6868 Author: Madhan Neethiraj <[email protected]> Authored: Sat Mar 14 02:24:06 2015 -0700 Committer: Madhan Neethiraj <[email protected]> Committed: Mon Mar 16 00:54:02 2015 -0700 ---------------------------------------------------------------------- pom.xml | 1 - security-admin/pom.xml | 43 -- .../apache/ranger/biz/AssetConnectionMgr.java | 424 ----------------- .../java/org/apache/ranger/biz/AssetMgr.java | 454 ------------------- .../java/org/apache/ranger/biz/HadoopFSMgr.java | 83 ---- .../java/org/apache/ranger/biz/HiveFSMgr.java | 81 ---- .../java/org/apache/ranger/biz/ServiceMgr.java | 223 +++++++-- .../org/apache/ranger/rest/ServiceREST.java | 4 +- .../org/apache/ranger/biz/TestAssetMgr.java | 3 - src/main/assembly/admin-web.xml | 129 ++++++ 10 files changed, 317 insertions(+), 1128 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/d017b31f/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index fe340c4..ef39d37 100644 --- a/pom.xml +++ b/pom.xml @@ -88,7 +88,6 @@ <module>knox-agent</module> <module>storm-agent</module> <module>plugin-yarn</module> - <module>lookup-client</module> <module>security-admin</module> <module>ugsync</module> <module>unixauthclient</module> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/d017b31f/security-admin/pom.xml ---------------------------------------------------------------------- diff --git a/security-admin/pom.xml b/security-admin/pom.xml index 7ba6a7f..1c17f81 100644 --- a/security-admin/pom.xml +++ b/security-admin/pom.xml @@ -323,49 +323,6 @@ <artifactId>jackson-mapper-asl</artifactId> <version>${codehaus.jackson.version}</version> </dependency> - <dependency> - <groupId>org.apache.ranger</groupId> - <artifactId>lookup-client</artifactId> - <version>${project.version}</version> - <exclusions> - <exclusion> - <groupId>javax.servlet</groupId> - <artifactId>*</artifactId> - </exclusion> - <exclusion> - <groupId>org.mortbay.jetty</groupId> - <artifactId>*</artifactId> - </exclusion> - <exclusion> - <groupId>tomcat</groupId> - <artifactId>*</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.geronimo.specs</groupId> - <artifactId>*</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>security_plugins.ranger-hdfs-plugin</groupId> - <artifactId>ranger-hdfs-plugin</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>security_plugins.ranger-hbase-plugin</groupId> - <artifactId>ranger-hbase-plugin</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>security_plugins.ranger-hive-plugin</groupId> - <artifactId>ranger-hive-plugin</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>security_plugins.ranger-yarn-plugin</groupId> - <artifactId>ranger-yarn-plugin</artifactId> - <version>${project.version}</version> - </dependency> <!-- <dependency> <groupId>org.apache.ranger</groupId> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/d017b31f/security-admin/src/main/java/org/apache/ranger/biz/AssetConnectionMgr.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/biz/AssetConnectionMgr.java b/security-admin/src/main/java/org/apache/ranger/biz/AssetConnectionMgr.java deleted file mode 100644 index da86e1a..0000000 --- a/security-admin/src/main/java/org/apache/ranger/biz/AssetConnectionMgr.java +++ /dev/null @@ -1,424 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - package org.apache.ranger.biz; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.Callable; -import java.util.concurrent.TimeUnit; - -import org.apache.log4j.Logger; -import org.apache.ranger.common.AppConstants; -import org.apache.ranger.common.JSONUtil; -import org.apache.ranger.common.StringUtil; -import org.apache.ranger.common.TimedEventUtil; -import org.apache.ranger.db.RangerDaoManager; -import org.apache.ranger.entity.XXAsset; -import org.apache.ranger.hadoop.client.HadoopFS; -import org.apache.ranger.hbase.client.HBaseClient; -import org.apache.ranger.hive.client.HiveClient; -import org.apache.ranger.knox.client.KnoxClient; -import org.apache.ranger.service.XAssetService; -import org.apache.ranger.storm.client.StormClient; -import org.apache.ranger.view.VXAsset; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Scope; -import org.springframework.stereotype.Component; - -@Component -@Scope("singleton") -public class AssetConnectionMgr { - - private static Logger logger = Logger.getLogger(AssetConnectionMgr.class); - - protected HashMap<String, HadoopFS> hadoopConnectionCache; - protected HashMap<String, HiveClient> hiveConnectionCache; - protected HashMap<String, HBaseClient> hbaseConnectionCache; - - protected HashMap<String, Boolean> repoConnectStatusMap; - - @Autowired - protected JSONUtil jsonUtil; - - @Autowired - protected StringUtil stringUtil; - - @Autowired - protected RangerDaoManager rangerDaoManager; - - @Autowired - XAssetService xAssetService; - - public AssetConnectionMgr(){ - hadoopConnectionCache = new HashMap<String, HadoopFS>(); - hiveConnectionCache = new HashMap<String, HiveClient>(); - hbaseConnectionCache = new HashMap<String, HBaseClient>(); - repoConnectStatusMap = new HashMap<String, Boolean>(); - } - - public HadoopFS getHadoopConnection(final String dataSourceName) { - HadoopFS hadoopFS = null; - XXAsset asset = rangerDaoManager.getXXAsset().findByAssetName(dataSourceName); - if (asset != null) { - // get it from the cache - synchronized (hadoopConnectionCache) { - hadoopFS = hadoopConnectionCache.get(asset.getName()); - if (hadoopFS == null) { - // if it doesn't exist in cache then create the connection - String config = asset.getConfig(); - if(!stringUtil.isEmpty(config)){ - config=xAssetService.getConfigWithDecryptedPassword(config); - } - // FIXME remove this once we start using putting config for - // default asset "hadoopdev" (should come from properties) - if (stringUtil.isEmpty(config) - && asset.getName().equals("hadoopdev")) { - - final Callable<HadoopFS> connectHDFS = new Callable<HadoopFS>() { - @Override - public HadoopFS call() throws Exception { - return new HadoopFS(dataSourceName); - } - }; - - try { - hadoopFS = TimedEventUtil.timedTask(connectHDFS, 10, TimeUnit.SECONDS); - } catch(Exception e){ - logger.error("Error establishing connection for HDFS repository : " - + dataSourceName, e); - } - - } else if (!stringUtil.isEmpty(config)) { - final HashMap<String, String> configMap = (HashMap<String, String>) jsonUtil - .jsonToMap(config); - final String assetName = asset.getName(); - - final Callable<HadoopFS> connectHDFS = new Callable<HadoopFS>() { - @Override - public HadoopFS call() throws Exception { - return new HadoopFS(assetName, configMap); - } - }; - - try { - hadoopFS = TimedEventUtil.timedTask(connectHDFS, 5, TimeUnit.SECONDS); - } catch(Exception e){ - logger.error("Error establishing connection for HDFS repository : " - + dataSourceName + " using configuration : " +config, e); - } - // put it into the cache - } else { - logger.error("Connection Config not defined for asset :" - + asset.getName(), new Throwable()); - } - hadoopConnectionCache.put(asset.getName(), hadoopFS); - repoConnectStatusMap.put(asset.getName(), true); - } else { - List<String> testConnect = hadoopFS.listFiles("/", "*"); - if(testConnect == null){ - hadoopConnectionCache.remove(dataSourceName); - hadoopFS = getHadoopConnection(dataSourceName); - } - } - } - } else { - logger.error("Asset not found with name "+dataSourceName, new Throwable()); - } - - return hadoopFS; - } - - public HiveClient getHiveConnection(final String dataSourceName) { - HiveClient hiveClient = null; - XXAsset asset = rangerDaoManager.getXXAsset().findByAssetName(dataSourceName); - if (asset != null) { - // get it from the cache - synchronized (hiveConnectionCache) { - hiveClient = hiveConnectionCache.get(asset.getName()); - if (hiveClient == null) { - String config = asset.getConfig(); - if (!stringUtil.isEmpty(config)) { - config=xAssetService.getConfigWithDecryptedPassword(config); - final HashMap<String, String> configMap = (HashMap<String, String>) jsonUtil - .jsonToMap(config); - - final Callable<HiveClient> connectHive = new Callable<HiveClient>() { - @Override - public HiveClient call() throws Exception { - return new HiveClient(dataSourceName, configMap); - } - }; - try { - hiveClient = TimedEventUtil.timedTask(connectHive, 5, TimeUnit.SECONDS); - } catch(Exception e){ - logger.error("Error connecting hive repository : "+ - dataSourceName +" using config : "+ config, e); - } - hiveConnectionCache.put(asset.getName(), hiveClient); - repoConnectStatusMap.put(asset.getName(), true); - } else { - logger.error("Connection Config not defined for asset :" - + asset.getName(), new Throwable()); - } - } else { - try { - List<String> testConnect = hiveClient.getDatabaseList("*"); - } catch(Exception e) { - hiveConnectionCache.remove(dataSourceName); - hiveClient = getHiveConnection(dataSourceName); - } - } - } - } else { - logger.error("Asset not found with name "+dataSourceName, new Throwable()); - } - return hiveClient; - } - - public KnoxClient getKnoxClient(String dataSourceName) { - KnoxClient knoxClient = null; - logger.debug("Getting knoxClient for datasource: " + dataSourceName); - XXAsset asset = rangerDaoManager.getXXAsset().findByAssetName(dataSourceName); - if (asset == null) { - logger.error("Asset not found with name " + dataSourceName, new Throwable()); - } else { - knoxClient = getKnoxClient(asset); - } - return knoxClient; - } - - public KnoxClient getKnoxClient(XXAsset asset) { - KnoxClient knoxClient = null; - if (asset == null) { - logger.error("Asset is null", new Throwable()); - } else { - String config = asset.getConfig(); - if(!stringUtil.isEmpty(config)){ - config=xAssetService.getConfigWithDecryptedPassword(config); - } - knoxClient = getKnoxClientByConfig(config); - } - return knoxClient; - } - - public KnoxClient getKnoxClientByConfig(String config) { - KnoxClient knoxClient = null; - if (config == null || config.trim().isEmpty()) { - logger.error("Connection Config is empty"); - - } else { - final HashMap<String, String> configMap = (HashMap<String, String>) jsonUtil - .jsonToMap(config); - String knoxUrl = configMap.get("knox.url"); - String knoxAdminUser = configMap.get("username"); - String knoxAdminPassword = configMap.get("password"); - knoxClient = new KnoxClient(knoxUrl, knoxAdminUser, knoxAdminPassword); - } - return knoxClient; - } - - public KnoxClient getKnoxClient(String dataSourceName, - Map<String, String> configMap) { - KnoxClient knoxClient = null; - logger.debug("Getting knoxClient for datasource: " + dataSourceName + - "configMap: " + configMap); - if (configMap == null || configMap.isEmpty()) { - logger.error("Connection ConfigMap is empty"); - } else { - String knoxUrl = configMap.get("knox.url"); - String knoxAdminUser = configMap.get("username"); - String knoxAdminPassword = configMap.get("password"); - knoxClient = new KnoxClient(knoxUrl, knoxAdminUser, knoxAdminPassword); - } - return knoxClient; - } - - - public static KnoxClient getKnoxClient(final String knoxUrl, String knoxAdminUser, String knoxAdminPassword) { - KnoxClient knoxClient = null; - if (knoxUrl == null || knoxUrl.isEmpty()) { - logger.error("Can not create KnoxClient: knoxUrl is empty"); - } else if (knoxAdminUser == null || knoxAdminUser.isEmpty()) { - logger.error("Can not create KnoxClient: knoxAdminUser is empty"); - } else if (knoxAdminPassword == null || knoxAdminPassword.isEmpty()) { - logger.error("Can not create KnoxClient: knoxAdminPassword is empty"); - } else { - knoxClient = new KnoxClient(knoxUrl, knoxAdminUser, knoxAdminPassword); - } - return knoxClient; - } - - public HBaseClient getHBaseConnection(final String dataSourceName) { - HBaseClient client = null; - XXAsset asset = rangerDaoManager.getXXAsset().findByAssetName( - dataSourceName); - if (asset != null) { - // get it from the cache - synchronized (hbaseConnectionCache) { - client = hbaseConnectionCache.get(asset.getName()); - if (client == null) { - // if it doesn't exist in cache then create the connection - String config = asset.getConfig(); - if(!stringUtil.isEmpty(config)){ - config=xAssetService.getConfigWithDecryptedPassword(config); - } - // FIXME remove this once we start using putting config for - // default asset "dev-hive" (should come from properties) - if (stringUtil.isEmpty(config) - && asset.getName().equals("hbase")) { - - final Callable<HBaseClient> connectHBase = new Callable<HBaseClient>() { - @Override - - public HBaseClient call() throws Exception { - HBaseClient hBaseClient=null; - if(dataSourceName!=null){ - try{ - hBaseClient=new HBaseClient(dataSourceName); - }catch(Exception ex){ - logger.error("Error connecting HBase repository : ", ex); - } - } - return hBaseClient; - } - - }; - - try { - if(connectHBase!=null){ - client = TimedEventUtil.timedTask(connectHBase, 5, TimeUnit.SECONDS); - } - } catch(Exception e){ - logger.error("Error connecting HBase repository : " + dataSourceName); - } - } else if (!stringUtil.isEmpty(config)) { - final HashMap<String, String> configMap = (HashMap<String, String>) jsonUtil - .jsonToMap(config); - - final Callable<HBaseClient> connectHBase = new Callable<HBaseClient>() { - @Override - public HBaseClient call() throws Exception { - HBaseClient hBaseClient=null; - if(dataSourceName!=null && configMap!=null){ - try{ - hBaseClient=new HBaseClient(dataSourceName,configMap); - }catch(Exception ex){ - logger.error("Error connecting HBase repository : ", ex); - } - } - return hBaseClient; - } - }; - - try { - if(connectHBase!=null){ - client = TimedEventUtil.timedTask(connectHBase, 5, TimeUnit.SECONDS); - } - } catch(Exception e){ - logger.error("Error connecting HBase repository : "+ - dataSourceName +" using config : "+ config); - } - - } else { - logger.error( - "Connection Config not defined for asset :" - + asset.getName(), new Throwable()); - } - if(client!=null){ - hbaseConnectionCache.put(asset.getName(), client); - } - } else { - List<String> testConnect = client.getTableList(".\\*"); - if(testConnect == null){ - hbaseConnectionCache.remove(dataSourceName); - client = getHBaseConnection(dataSourceName); - } - } - repoConnectStatusMap.put(asset.getName(), true); - } - } else { - logger.error("Asset not found with name " + dataSourceName, - new Throwable()); - } - - return client; - } - - public boolean destroyConnection(VXAsset asset) { - boolean result = false; - if (asset != null) { - if(asset.getAssetType() == AppConstants.ASSET_HDFS) { - synchronized (hadoopConnectionCache) { - - @SuppressWarnings("unused") - HadoopFS hadoopFS = hadoopConnectionCache.get(asset.getName()); - // TODO need a way to close the connection - hadoopConnectionCache.remove(asset.getName()); - repoConnectStatusMap.remove(asset.getName()); - - } - } else if(asset.getAssetType() == AppConstants.ASSET_HIVE) { - synchronized (hadoopConnectionCache) { - - HiveClient hiveClient = hiveConnectionCache.get(asset.getName()); - if(hiveClient != null) { - hiveClient.close(); - } - hadoopConnectionCache.remove(asset.getName()); - repoConnectStatusMap.remove(asset.getName()); - - } - } else if (asset.getAssetType() == AppConstants.ASSET_HBASE) { - synchronized (hbaseConnectionCache) { - @SuppressWarnings("unused") - HBaseClient hBaseClient = hbaseConnectionCache.get(asset - .getName()); - // TODO need a way to close the connection - hbaseConnectionCache.remove(asset.getName()); - repoConnectStatusMap.remove(asset.getName()); - - } - } - result = true; - } - return result; - } - - public HadoopFS resetHadoopConnection(final String dataSourceName){ - hadoopConnectionCache.remove(dataSourceName); - return getHadoopConnection(dataSourceName); - } - - public static StormClient getStormClient(final String stormUIURL, String userName, String password) { - StormClient stormClient = null; - if (stormUIURL == null || stormUIURL.isEmpty()) { - logger.error("Can not create KnoxClient: stormUIURL is empty"); - } else if (userName == null || userName.isEmpty()) { - logger.error("Can not create KnoxClient: knoxAdminUser is empty"); - } else if (password == null || password.isEmpty()) { - logger.error("Can not create KnoxClient: knoxAdminPassword is empty"); - } else { - stormClient = new StormClient(stormUIURL, userName, password); - } - return stormClient; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/d017b31f/security-admin/src/main/java/org/apache/ranger/biz/AssetMgr.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/biz/AssetMgr.java b/security-admin/src/main/java/org/apache/ranger/biz/AssetMgr.java index e13b632..3c3bd77 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/AssetMgr.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/AssetMgr.java @@ -30,8 +30,6 @@ import java.util.List; import java.util.Map; import java.util.Random; import java.util.Set; -import java.util.concurrent.Callable; -import java.util.concurrent.TimeUnit; import javax.naming.InvalidNameException; import javax.naming.ldap.LdapName; @@ -50,7 +48,6 @@ import org.apache.ranger.common.RangerCommonEnums; import org.apache.ranger.common.RangerConstants; import org.apache.ranger.common.SearchCriteria; import org.apache.ranger.common.StringUtil; -import org.apache.ranger.common.TimedEventUtil; import org.apache.ranger.common.UserSessionBase; import org.apache.ranger.db.RangerDaoManager; import org.apache.ranger.entity.XXAsset; @@ -62,11 +59,6 @@ import org.apache.ranger.entity.XXPortalUserRole; import org.apache.ranger.entity.XXResource; import org.apache.ranger.entity.XXTrxLog; import org.apache.ranger.entity.XXUser; -import org.apache.ranger.hadoop.client.HadoopFS; -import org.apache.ranger.hadoop.client.exceptions.HadoopException; -import org.apache.ranger.hbase.client.HBaseClient; -import org.apache.ranger.hive.client.HiveClient; -import org.apache.ranger.knox.client.KnoxClient; import org.apache.ranger.service.XAccessAuditService; import org.apache.ranger.service.XAuditMapService; import org.apache.ranger.service.XGroupService; @@ -74,14 +66,12 @@ import org.apache.ranger.service.XPermMapService; import org.apache.ranger.service.XPolicyService; import org.apache.ranger.service.XTrxLogService; import org.apache.ranger.service.XUserService; -import org.apache.ranger.storm.client.StormClient; import org.apache.ranger.util.RestUtil; import org.apache.ranger.view.VXAccessAuditList; import org.apache.ranger.view.VXAsset; import org.apache.ranger.view.VXAuditMap; import org.apache.ranger.view.VXAuditMapList; import org.apache.ranger.view.VXLong; -import org.apache.ranger.view.VXMessage; import org.apache.ranger.view.VXPermMap; import org.apache.ranger.view.VXPermMapList; import org.apache.ranger.view.VXPolicy; @@ -118,9 +108,6 @@ public class AssetMgr extends AssetMgrBase { JSONUtil jsonUtil; @Autowired - AssetConnectionMgr assetConnectionMgr; - - @Autowired RangerBizUtil msBizUtil; @Autowired @@ -725,62 +712,6 @@ public class AssetMgr extends AssetMgrBase { return file; } - public VXStringList getHdfsResources(final String dataSourceName, - String baseDir) { - if (dataSourceName != null && baseDir != null) { - List<String> strList = new ArrayList<String>(); - try { - String wildCardToMatch; - final HadoopFS hdfsClient = assetConnectionMgr - .getHadoopConnection(dataSourceName); - - if (hdfsClient != null) { - Integer lastIndex = baseDir.lastIndexOf("/"); - if (lastIndex < 0) { - wildCardToMatch = baseDir + "*"; - baseDir = "/"; - } else if (lastIndex == 0 && baseDir.length() == 1) { - wildCardToMatch = null; - baseDir = "/"; - } else if ((lastIndex + 1) == baseDir.length()) { - wildCardToMatch = null; - baseDir = baseDir.substring(0, lastIndex + 1); - } else { - wildCardToMatch = baseDir.substring(lastIndex + 1) - + "*"; - baseDir = baseDir.substring(0, lastIndex + 1); - } - - final String finalBaseDir = baseDir; - final String finalWildCardToMatch = wildCardToMatch; - final Callable<List<String>> callableObj = new Callable<List<String>>() { - - @Override - public List<String> call() throws Exception { - return hdfsClient.listFiles(finalBaseDir, - finalWildCardToMatch); - } - - }; - - strList = TimedEventUtil.timedTask(callableObj, 5, - TimeUnit.SECONDS); // If - // strList = hdfsClient.listFiles(finalBaseDir, - // finalWildCardToMatch); - logger.debug("Resource dir : " + baseDir - + " wild card to match : " + wildCardToMatch - + "\n Matching resources : " + strList); - } - } catch (Exception e) { - logger.error("Unable to get hdfs resources.", e); - } - - return msBizUtil.mapStringListToVStringList(strList); - } else { - return new VXStringList(); - } - } - public String getLatestRepoPolicy(VXAsset xAsset, List<VXResource> xResourceList, Long updatedTime, X509Certificate[] certchain, boolean httpEnabled, String epoch, String ipAddress, boolean isSecure, String count, String agentId) { @@ -1090,77 +1021,6 @@ public class AssetMgr extends AssetMgrBase { return updatedPolicyStr; } - public VXStringList getHiveResources(final String dataSourceName, - String databaseName, String tableName, String columnName) { - - List<String> resultList = new ArrayList<String>(); - if (dataSourceName != null) { - final HiveClient hiveClient = assetConnectionMgr - .getHiveConnection(dataSourceName); - - try { - final Callable<List<String>> callableObj; - final String finalDbName; - final String finalColName; - final String finalTableName; - - if (hiveClient != null && databaseName != null - && !databaseName.isEmpty()) { - if (tableName != null && !tableName.isEmpty()) { - if (columnName != null && !columnName.isEmpty()) { - columnName += "*"; - finalColName = columnName; - finalDbName = databaseName; - finalTableName = tableName; - - callableObj = new Callable<List<String>>() { - @Override - public List<String> call() { - return hiveClient.getColumnList( - finalDbName, finalTableName, - finalColName); - } - }; - } else { - tableName += "*"; - finalTableName = tableName; - finalDbName = databaseName; - callableObj = new Callable<List<String>>() { - - @Override - public List<String> call() { - return hiveClient.getTableList(finalDbName, - finalTableName); - } - - }; - } - } else { - databaseName += "*"; - finalDbName = databaseName; - callableObj = new Callable<List<String>>() { - @Override - public List<String> call() { - return hiveClient.getDatabaseList(finalDbName); - } - }; - - } - - synchronized (hiveClient) { - resultList = TimedEventUtil.timedTask(callableObj, 5, - TimeUnit.SECONDS); - } - - } - } catch (Exception e) { - logger.error("Unable to get hive resources.", e); - } - } - - return msBizUtil.mapStringListToVStringList(resultList); - } - @Override public VXAsset createXAsset(VXAsset vXAsset) { UserSessionBase usb = ContextUtil.getCurrentUserSession(); @@ -1331,8 +1191,6 @@ public class AssetMgr extends AssetMgrBase { vXAsset = (VXAsset) xAssetService.updateResource(vXAsset); // update default policy permission and user updateDefaultPolicy(vXAsset, vXAsset.getConfig()); - // TODO this should reset the connection - assetConnectionMgr.destroyConnection(vXAsset); // TODO : Log in transaction log table xaBizUtil.createTrxLog(trxLogList); return vXAsset; @@ -1377,292 +1235,6 @@ public class AssetMgr extends AssetMgrBase { } } - public VXStringList getHBaseResources(final String dataSourceName, - String tableName, String columnFamilies) { - - List<String> resultList = new ArrayList<String>(); - if (dataSourceName != null) { - final HBaseClient hBaseClient = assetConnectionMgr - .getHBaseConnection(dataSourceName); - - try { - final Callable<List<String>> callableObj; - if (hBaseClient != null && tableName != null - && !tableName.isEmpty()) { - final String finalColFamilies; - final String finalTableName; - - if (columnFamilies != null && !columnFamilies.isEmpty()) { - if (!columnFamilies.endsWith("*")) { - columnFamilies += "*"; - } - - columnFamilies = columnFamilies.replaceAll("\\*", - ".\\*"); - finalColFamilies = columnFamilies; - finalTableName = tableName; - - callableObj = new Callable<List<String>>() { - @Override - public List<String> call() { - return hBaseClient.getColumnFamilyList( - finalTableName, finalColFamilies); - } - }; - - } else { - if (!tableName.endsWith("*")) { - tableName += "*"; - } - - tableName = tableName.replaceAll("\\*", ".\\*"); - finalTableName = tableName; - - callableObj = new Callable<List<String>>() { - @Override - public List<String> call() { - return hBaseClient.getTableList(finalTableName); - } - }; - - } - resultList = TimedEventUtil.timedTask(callableObj, 5, - TimeUnit.SECONDS); - } - - } catch (Exception e) { - logger.error("Unable to get hbase resources.", e); - } - } - - return msBizUtil.mapStringListToVStringList(resultList); - } - - public VXStringList getKnoxResources( - String knoxUrl,String knoxAdminUser, String knoxAdminPassword, - String topologyName, String serviceName) { - List<String> resultList = new ArrayList<String>(); - if (knoxUrl == null || knoxUrl.isEmpty()) { - logger.error("Unable to get knox resources: knoxUrl is empty"); - return msBizUtil.mapStringListToVStringList(resultList); - } else if (knoxAdminUser == null || knoxAdminUser.isEmpty()) { - logger.error("Unable to get knox resources: knoxAdminUser is empty"); - return msBizUtil.mapStringListToVStringList(resultList); - } else if (knoxAdminPassword == null || knoxAdminPassword.isEmpty()) { - logger.error("Unable to get knox resources: knoxAdminPassword is empty"); - return msBizUtil.mapStringListToVStringList(resultList); - } - - final KnoxClient knoxClient = assetConnectionMgr - .getKnoxClient(knoxUrl, knoxAdminUser, knoxAdminPassword); - VXStringList knoxResources = getKnoxResources(knoxClient, topologyName, serviceName); - return knoxResources; - } - - public VXStringList getKnoxResources(final String dataSourceName, - String topologyName, String serviceName) { - List<String> resultList = new ArrayList<String>(); - if (dataSourceName == null || dataSourceName.isEmpty()) { - logger.error("Unable to get knox resources: dataSourceName is null"); - return msBizUtil.mapStringListToVStringList(resultList); - } - - final KnoxClient knoxClient = assetConnectionMgr - .getKnoxClient(dataSourceName); - VXStringList knoxResources = getKnoxResources(knoxClient, topologyName, serviceName); - return knoxResources; - } - - - public VXStringList getKnoxResources(final KnoxClient knoxClient, - String topologyName, String serviceName) { - - List<String> resultList = new ArrayList<String>(); - - try { - if (knoxClient == null) { - logger.error("Unable to get knox resources: knoxClient is null"); - return msBizUtil.mapStringListToVStringList(resultList); - } - - final Callable<List<String>> callableObj; - if (serviceName != null) { - final String finalServiceNameMatching = serviceName.trim(); - final String finalTopologyName = topologyName; - callableObj = new Callable<List<String>>() { - @Override - public List<String> call() { - return knoxClient.getServiceList(finalTopologyName, - finalServiceNameMatching); - } - }; - - } else { - final String finalTopologyNameMatching = (topologyName == null) ? - "" : topologyName.trim(); - callableObj = new Callable<List<String>>() { - @Override - public List<String> call() { - return knoxClient.getTopologyList(finalTopologyNameMatching); - } - }; - } - resultList = TimedEventUtil.timedTask(callableObj, 5, - TimeUnit.SECONDS); - - } catch (Exception e) { - logger.error("Unable to get knox resources.", e); - } - - return msBizUtil.mapStringListToVStringList(resultList); - } - - public VXResponse testConfig(VXAsset vXAsset) { - - if (vXAsset.getActiveStatus() == RangerCommonEnums.STATUS_DELETED) { - logger.error("Trying to test Asset which is soft deleted"); - throw restErrorUtil.createRESTException( - "Repository not found, Repository Name : " + vXAsset.getName(), - MessageEnums.DATA_NOT_FOUND, vXAsset.getId(), null, - "Repository not exist for this Id : " + vXAsset.getId()); - } - - int assetType = vXAsset.getAssetType(); - - VXResponse testResponse = new VXResponse(); - HashMap<String, Object> responseData = new HashMap<String, Object>(); - - HashMap<String, String> configMap = (HashMap<String, String>) jsonUtil - .jsonToMap(vXAsset.getConfig()); - String password = configMap.get("password"); - String hiddenPasswordString = PropertiesUtil.getProperty( - "xa.password.hidden", "*****"); - - if (password != null && password.equals(hiddenPasswordString)) { - String assetName = vXAsset.getName(); - if (assetName != null) { - XXAsset existingVXAsset = rangerDaoManager.getXXAsset() - .findByAssetName(assetName); - if (existingVXAsset != null - && existingVXAsset.getConfig() != null) { - String existingConfig = existingVXAsset.getConfig(); - existingConfig=xAssetService.getConfigWithDecryptedPassword(existingConfig); - HashMap<String, String> existingConfigMap = (HashMap<String, String>) jsonUtil - .jsonToMap(existingConfig); - String existingPassword = existingConfigMap.get("password"); - if (existingPassword != null) { - configMap.put("password", existingPassword); - } - } - } - } - - try { - String dataSource = vXAsset.getName(); - if (assetType == AppConstants.ASSET_HDFS) { - // HadoopFS connectionObj = new HadoopFS(vXAsset.getName(), - // configMap); - // if (connectionObj != null) { - // List<String> testResult = connectionObj - // .listFiles("/", null); - // if (testResult != null && testResult.size() != 0) { - // connectivityStatus = true; - // } - // } - responseData = HadoopFS.testConnection(dataSource, configMap); - } else if (assetType == AppConstants.ASSET_HIVE) { - // HiveClient connectionObj = new HiveClient(vXAsset.getName(), - // configMap); - // if (connectionObj != null) { - // List<String> testResult = connectionObj - // .getDatabaseList("*"); - // if (testResult != null && testResult.size() != 0) { - // connectivityStatus = true; - // } - // } - // connectionObj.close(); - responseData = HiveClient.testConnection(dataSource, configMap); - } else if (assetType == AppConstants.ASSET_HBASE) { - // HBaseClient connectionObj = new - // HBaseClient(vXAsset.getName(), - // configMap); - // if (connectionObj != null) { - // connectivityStatus = connectionObj.getHBaseStatus(); - // } else { - // Log.error("testConfig: Not able to create HBaseClient"); - // } - responseData = HBaseClient - .testConnection(dataSource, configMap); - } else if (assetType == AppConstants.ASSET_KNOX) { - // KnoxClient knoxClient = assetConnectionMgr.getKnoxClient( - // vXAsset.getName(), configMap); - // VXStringList vxStringList = getKnoxResources(knoxClient, "", - // null); - // if (vxStringList != null && (vxStringList.getListSize() != - // 0)) { - // connectivityStatus = true; - // } - responseData = KnoxClient.testConnection(dataSource, configMap); - } else if (assetType == AppConstants.ASSET_STORM) { - responseData = StormClient.testConnection(dataSource, configMap); - } else { - throw restErrorUtil.createRESTException( - "Invalid repository type.", - MessageEnums.INVALID_INPUT_DATA); - } - testResponse = generateResponseForTestConn(responseData, ""); - - } catch (Exception e) { - - String msg = "Unable to connect repository with given config for " - + vXAsset.getName(); - HashMap<String, Object> respData = new HashMap<String, Object>(); - String message = ""; - if (e instanceof HadoopException) { - respData = ((HadoopException) e).responseData; - message = (respData != null && respData.get("message") != null) ? respData.get( - "message").toString() : msg; - } - if(respData == null) { - respData = new HashMap<String, Object>(); - } - testResponse = generateResponseForTestConn(respData, message); - logger.error(msg, e); - } - return testResponse; - } - - private VXResponse generateResponseForTestConn( - HashMap<String, Object> responseData, String msg) { - VXResponse vXResponse = new VXResponse(); - - Long objId = (responseData.get("objectId") != null) ? Long - .parseLong(responseData.get("objectId").toString()) : null; - boolean connectivityStatus = (responseData.get("connectivityStatus") != null) ? Boolean - .parseBoolean(responseData.get("connectivityStatus").toString()) - : false; - int statusCode = (connectivityStatus) ? VXResponse.STATUS_SUCCESS - : VXResponse.STATUS_ERROR; - String message = (responseData.get("message") != null) ? responseData - .get("message").toString() : msg; - String description = (responseData.get("description") != null) ? responseData - .get("description").toString() : msg; - String fieldName = (responseData.get("fieldName") != null) ? responseData - .get("fieldName").toString() : null; - - VXMessage vXMsg = new VXMessage(); - List<VXMessage> vXMsgList = new ArrayList<VXMessage>(); - vXMsg.setFieldName(fieldName); - vXMsg.setMessage(message); - vXMsg.setObjectId(objId); - vXMsgList.add(vXMsg); - - vXResponse.setMessageList(vXMsgList); - vXResponse.setMsgDesc(description); - vXResponse.setStatusCode(statusCode); - return vXResponse; - } - private void createResourcePathForHive(VXResource vXResource) { String[] databases = (vXResource.getDatabases() == null || vXResource @@ -3117,32 +2689,6 @@ public class AssetMgr extends AssetMgrBase { return vXResource; } - - public VXStringList getStormResources(final String dataSourceName,String topologyName) { - VXStringList ret = null ; - XXAsset asset = rangerDaoManager.getXXAsset().findByAssetName(dataSourceName); - String config = asset.getConfig() ; - if(!stringUtil.isEmpty(config)){ - config=xAssetService.getConfigWithDecryptedPassword(config); - } - if (config == null || config.trim().isEmpty()) { - logger.error("Connection Config is empty"); - - } else { - final HashMap<String, String> configMap = (HashMap<String, String>) jsonUtil.jsonToMap(config); - String url = configMap.get("nimbus.url"); - String username = configMap.get("username"); - String password = configMap.get("password"); - ret = getStormResources(url, username, password,topologyName) ; - } - return ret ; - } - - public VXStringList getStormResources(String url, String username, String password,String topologyName) { - final StormClient stormClient = AssetConnectionMgr.getStormClient(url, username, password); - List<String> toplogyList = stormClient.getTopologyList(topologyName) ; - return msBizUtil.mapStringListToVStringList(toplogyList) ; - } @Override public VXLong getXResourceSearchCount(SearchCriteria searchCriteria) { http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/d017b31f/security-admin/src/main/java/org/apache/ranger/biz/HadoopFSMgr.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/biz/HadoopFSMgr.java b/security-admin/src/main/java/org/apache/ranger/biz/HadoopFSMgr.java deleted file mode 100644 index 86e2311..0000000 --- a/security-admin/src/main/java/org/apache/ranger/biz/HadoopFSMgr.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - /** - * - */ -package org.apache.ranger.biz; - -import org.apache.log4j.Logger; -import org.apache.ranger.hadoop.client.HadoopFS; -import org.springframework.context.annotation.Scope; -import org.springframework.stereotype.Component; - -/** - * - * - */ - -@Component -@Scope("singleton") -public class HadoopFSMgr { - - private HadoopFS fs; - private String dataSource; - private static Logger logger = Logger.getLogger(HadoopFSMgr.class); - - public HadoopFSMgr() { - init(); - } - - public HadoopFSMgr(String dataSource) { - this.dataSource = dataSource; - init(); - } - - private void init() { - try { -// if (dataSource != null) { -// fs = new HadoopFS(dataSource); -// } else { -// fs = new HadoopFS("hadoopdev"); -// } - } catch (Exception e) { - logger.error("Error connecting hive client", e); - } - } - - protected HadoopFS getInstance(String dataSourceName) { - if (dataSourceName == null) { - logger.info("Hadoop client name not provided."); - return fs; - } else { - if (fs.getDataSource() != null) { - if (fs.getDataSource().equalsIgnoreCase(dataSourceName)) { - return fs; - } else { - fs = new HadoopFS(dataSourceName); - return fs; - } - } else { - fs = new HadoopFS(dataSourceName); - return fs; - } - } - } - -} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/d017b31f/security-admin/src/main/java/org/apache/ranger/biz/HiveFSMgr.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/biz/HiveFSMgr.java b/security-admin/src/main/java/org/apache/ranger/biz/HiveFSMgr.java deleted file mode 100644 index 8d84db7..0000000 --- a/security-admin/src/main/java/org/apache/ranger/biz/HiveFSMgr.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - /** - * - */ -package org.apache.ranger.biz; - -import org.apache.log4j.Logger; -import org.apache.ranger.hive.client.HiveClient; - -/** - * - */ - -// @Component -// @Scope("singleton") - -public class HiveFSMgr { - - private HiveClient fs; - private String dataSource; - private static Logger logger = Logger.getLogger(HiveFSMgr.class); - - public HiveFSMgr() { - init(); - } - - public HiveFSMgr(String dataSource) { - this.dataSource = dataSource; - init(); - } - - private void init() { - try { - if (dataSource != null) { - fs = new HiveClient(dataSource); - } else { - fs = new HiveClient("dev-hive"); - } - } catch (Exception e) { - logger.error("Error connecting hive client", e); - } - } - - protected HiveClient getInstance(String dataSourceName) { - if (dataSourceName == null) { - logger.info("Hive client name not provided."); - return fs; - } else { - if (fs.getDataSource() != null) { - if (fs.getDataSource().equalsIgnoreCase(dataSourceName)) { - return fs; - } else { - fs = new HiveClient(dataSourceName); - return fs; - } - } else { - fs = new HiveClient(dataSourceName); - return fs; - } - } - } - -} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/d017b31f/security-admin/src/main/java/org/apache/ranger/biz/ServiceMgr.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/biz/ServiceMgr.java b/security-admin/src/main/java/org/apache/ranger/biz/ServiceMgr.java index b79cb9b..8e6aa3f 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/ServiceMgr.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/ServiceMgr.java @@ -19,10 +19,15 @@ package org.apache.ranger.biz; +import java.io.File; +import java.net.URL; +import java.net.URLClassLoader; import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import java.util.Map; +import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.ranger.plugin.model.RangerService; @@ -34,6 +39,7 @@ import org.apache.ranger.view.VXMessage; import org.apache.ranger.view.VXResponse; import org.springframework.stereotype.Component; + @Component public class ServiceMgr { @@ -42,89 +48,232 @@ public class ServiceMgr { public List<String> lookupResource(String serviceName, ResourceLookupContext context, ServiceStore svcStore) throws Exception { List<String> ret = null; - RangerBaseService svc = getRangerServiceByName(serviceName, svcStore); if(LOG.isDebugEnabled()) { LOG.debug("==> ServiceMgr.lookupResource for Service: (" + svc + "Context: " + context + ")"); } - if ( svc != null) { + if(svc != null) { + ClassLoader clsLoader = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(svc.getClass().getClassLoader()); + ret = svc.lookupResource(context); - } catch ( Exception e) { + } catch (Exception e) { LOG.error("==> ServiceMgr.lookupResource Error:" + e); throw e; + } finally { + Thread.currentThread().setContextClassLoader(clsLoader); } } - + if(LOG.isDebugEnabled()) { LOG.debug("==> ServiceMgr.lookupResource for Response: (" + ret + ")"); } - + return ret; } public VXResponse validateConfig(RangerService service, ServiceStore svcStore) throws Exception { - - VXResponse ret = new VXResponse(); - RangerBaseService svc = getRangerServiceByService(service, svcStore); - + VXResponse ret = new VXResponse(); + RangerBaseService svc = getRangerServiceByService(service, svcStore); + if(LOG.isDebugEnabled()) { LOG.debug("==> ServiceMgr.validateConfig for Service: (" + svc + ")"); } - - if (svc != null) { + + if(svc != null) { + ClassLoader clsLoader = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(svc.getClass().getClassLoader()); + HashMap<String, Object> responseData = svc.validateConfig(); + ret = generateResponseForTestConn(responseData, ""); } catch (Exception e) { LOG.error("==> ServiceMgr.validateConfig Error:" + e); throw e; + } finally { + Thread.currentThread().setContextClassLoader(clsLoader); } } - + if(LOG.isDebugEnabled()) { LOG.debug("==> ServiceMgr.validateConfig for Response: (" + ret + ")"); } - + return ret; } - - public RangerBaseService getRangerServiceByName(String serviceName, ServiceStore svcStore) throws Exception{ - RangerBaseService svc = null; - RangerService service = svcStore.getServiceByName(serviceName); - - if ( service != null) { - svc = getRangerServiceByService(service, svcStore); - } - return svc; + public RangerBaseService getRangerServiceByName(String serviceName, ServiceStore svcStore) throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> ServiceMgr.getRangerServiceByName(" + serviceName + ")"); + } + + RangerBaseService ret = null; + RangerService service = svcStore == null ? null : svcStore.getServiceByName(serviceName); + + if(service != null) { + ret = getRangerServiceByService(service, svcStore); + } else { + LOG.warn("ServiceMgr.getRangerServiceByName(" + serviceName + "): could not find the service"); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== ServiceMgr.getRangerServiceByName(" + serviceName + "): " + ret); + } + + return ret; } - + public RangerBaseService getRangerServiceByService(RangerService service, ServiceStore svcStore) throws Exception{ - - RangerServiceDef serviceDef = null; - RangerBaseService ret = null; - - String serviceType = service.getType(); - - if (serviceType != null) { - serviceDef = svcStore.getServiceDefByName(serviceType); - if ( serviceDef != null) { - ret = (RangerBaseService) Class.forName(serviceDef.getImplClass()).newInstance(); + if(LOG.isDebugEnabled()) { + LOG.debug("==> ServiceMgr.getRangerServiceByService(" + service + ")"); + } + + RangerBaseService ret = null; + String serviceType = service == null ? null : service.getType(); + + if(! StringUtils.isEmpty(serviceType)) { + RangerServiceDef serviceDef = svcStore == null ? null : svcStore.getServiceDefByName(serviceType); + + if(serviceDef != null) { + Class<RangerBaseService> cls = getClassForServiceType(serviceDef); + + if(cls != null) { + ret = cls.newInstance(); + + ret.init(serviceDef, service); + } else { + LOG.warn("ServiceMgr.getRangerServiceByService(" + service + "): could not find service class '" + serviceDef.getImplClass() + "'"); + } + } else { + LOG.warn("ServiceMgr.getRangerServiceByService(" + service + "): could not find the service-type '" + serviceType + "'"); } - - ret.init(serviceDef, service); + } else { + LOG.warn("ServiceMgr.getRangerServiceByService(" + service + "): could not find the service-type"); } if(LOG.isDebugEnabled()) { - LOG.debug("==> ServiceMgr.getRangerServiceByService ServiceType: " + serviceType + "ServiceDef: " + serviceDef + "Service Class: " + serviceDef.getImplClass()); + LOG.debug("<== ServiceMgr.getRangerServiceByService(" + service + "): " + ret); } - + return ret; } + + private static Map<String, Class<RangerBaseService>> serviceTypeClassMap = new HashMap<String, Class<RangerBaseService>>(); + + @SuppressWarnings("unchecked") + private Class<RangerBaseService> getClassForServiceType(RangerServiceDef serviceDef) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> ServiceMgr.getClassForServiceType(" + serviceDef + ")"); + } + + Class<RangerBaseService> ret = null; + + if(serviceDef != null) { + String serviceType = serviceDef.getName(); + + ret = serviceTypeClassMap.get(serviceType); + + if(ret == null) { + synchronized(serviceTypeClassMap) { + ret = serviceTypeClassMap.get(serviceType); + + if(ret == null) { + String clsName = serviceDef.getImplClass(); + + if(LOG.isDebugEnabled()) { + LOG.debug("ServiceMgr.getClassForServiceType(" + serviceType + "): service-class " + clsName + " not found in cache"); + } + + URL[] pluginFiles = getPluginFilesForServiceType(serviceType); + URLClassLoader clsLoader = new URLClassLoader(pluginFiles, Thread.currentThread().getContextClassLoader()); + + try { + Class<?> cls = Class.forName(clsName, true, clsLoader); + + ret = (Class<RangerBaseService>)cls; + + serviceTypeClassMap.put(serviceType, ret); + } catch (Exception excp) { + LOG.warn("ServiceMgr.getClassForServiceType(" + serviceType + "): failed to find service-class '" + clsName + "'. Resource lookup will not be available", excp); + } + } else { + if(LOG.isDebugEnabled()) { + LOG.debug("ServiceMgr.getClassForServiceType(" + serviceType + "): service-class " + ret.getCanonicalName() + " found in cache"); + } + } + } + } else { + if(LOG.isDebugEnabled()) { + LOG.debug("ServiceMgr.getClassForServiceType(" + serviceType + "): service-class " + ret.getCanonicalName() + " found in cache"); + } + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== ServiceMgr.getClassForServiceType(" + serviceDef + "): " + ret); + } + + return ret; + } + + private URL[] getPluginFilesForServiceType(String serviceType) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> ServiceMgr.getPluginFilesForServiceType(" + serviceType + ")"); + } + + List<URL> ret = new ArrayList<URL>(); + + getFilesInDirectory("ranger-plugins/" + serviceType, ret); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== ServiceMgr.getPluginFilesForServiceType(" + serviceType + "): " + ret.size() + " files"); + } + + return ret.toArray(new URL[] { }); + } + + private void getFilesInDirectory(String dirPath, List<URL> files) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> ServiceMgr.getFilesInDirectory(" + dirPath + ")"); + } + + URL pluginJarPath = getClass().getClassLoader().getResource(dirPath); + + if(pluginJarPath != null && pluginJarPath.getProtocol().equals("file")) { + try { + File[] dirFiles = new File(pluginJarPath.toURI()).listFiles(); + + if(dirFiles != null) { + for(File dirFile : dirFiles) { + try { + URL jarPath = dirFile.toURI().toURL(); + + LOG.warn("getFilesInDirectory('" + dirPath + "'): adding " + dirFile.getAbsolutePath()); + files.add(jarPath); + } catch(Exception excp) { + LOG.warn("getFilesInDirectory('" + dirPath + "'): failed to get URI for file " + dirFile.getAbsolutePath(), excp); + } + } + } + } catch(Exception excp) { + LOG.warn("getFilesInDirectory('" + dirPath + "'): error", excp); + } + } else { + LOG.warn("getFilesInDirectory('" + dirPath + "'): could not find directory in CLASSPATH"); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== ServiceMgr.getFilesInDirectory(" + dirPath + ")"); + } + } + private VXResponse generateResponseForTestConn( HashMap<String, Object> responseData, String msg) { VXResponse vXResponse = new VXResponse(); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/d017b31f/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java index 470ca50..e3492ee 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java @@ -41,8 +41,8 @@ import javax.ws.rs.core.Context; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; -import org.apache.commons.lang3.ArrayUtils; -import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang.ArrayUtils; +import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.ranger.admin.client.datatype.RESTResponse; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/d017b31f/security-admin/src/test/java/org/apache/ranger/biz/TestAssetMgr.java ---------------------------------------------------------------------- diff --git a/security-admin/src/test/java/org/apache/ranger/biz/TestAssetMgr.java b/security-admin/src/test/java/org/apache/ranger/biz/TestAssetMgr.java index f6bee3b..ec91f70 100644 --- a/security-admin/src/test/java/org/apache/ranger/biz/TestAssetMgr.java +++ b/security-admin/src/test/java/org/apache/ranger/biz/TestAssetMgr.java @@ -111,9 +111,6 @@ public class TestAssetMgr { @Mock RangerBizUtil xaBizUtil; - @Mock - AssetConnectionMgr assetConnectionMgr; - @Mock XPolicyService xPolicyService; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/d017b31f/src/main/assembly/admin-web.xml ---------------------------------------------------------------------- diff --git a/src/main/assembly/admin-web.xml b/src/main/assembly/admin-web.xml index 6988492..d4a8a91 100644 --- a/src/main/assembly/admin-web.xml +++ b/src/main/assembly/admin-web.xml @@ -49,6 +49,135 @@ <moduleSet> <binaries> + <includeDependencies>true</includeDependencies> + <outputDirectory>/ews/webapp/WEB-INF/classes/ranger-plugins/hdfs</outputDirectory> + <unpack>false</unpack> + <directoryMode>755</directoryMode> + <fileMode>644</fileMode> + <dependencySets> + <dependencySet> + <includes> + <include>org.apache.hadoop:hadoop-hdfs:jar:${hadoop-common.version}</include> + <include>org.apache.hadoop:hadoop-auth:jar:${hadoop-common.version}</include> + <include>org.apache.hadoop:hadoop-common:jar:${hadoop-common.version}</include> + </includes> + </dependencySet> + </dependencySets> + </binaries> + <includes> + <include>security_plugins.ranger-hdfs-plugin:ranger-hdfs-plugin</include> + </includes> + </moduleSet> + + <moduleSet> + <binaries> + <includeDependencies>true</includeDependencies> + <outputDirectory>/ews/webapp/WEB-INF/classes/ranger-plugins/hive</outputDirectory> + <unpack>false</unpack> + <directoryMode>755</directoryMode> + <fileMode>644</fileMode> + <dependencySets> + <dependencySet> + <includes> + <include>org.apache.hive:hive-jdbc:jar:${hive.version}</include> + </includes> + </dependencySet> + </dependencySets> + </binaries> + <includes> + <include>security_plugins.ranger-hive-plugin:ranger-hive-plugin</include> + </includes> + </moduleSet> + + <moduleSet> + <binaries> + <includeDependencies>true</includeDependencies> + <outputDirectory>/ews/webapp/WEB-INF/classes/ranger-plugins/hbase</outputDirectory> + <unpack>false</unpack> + <directoryMode>755</directoryMode> + <fileMode>644</fileMode> + <dependencySets> + <dependencySet> + <includes> + <include>org.apache.hbase:hbase-server:jar:${hbase.version}</include> + <include>org.apache.hadoop:hadoop-hdfs:jar:${hadoop-common.version}</include> + <include>org.apache.hadoop:hadoop-auth:jar:${hadoop-common.version}</include> + <include>org.apache.hadoop:hadoop-common:jar:${hadoop-common.version}</include> + </includes> + </dependencySet> + </dependencySets> + </binaries> + <includes> + <include>security_plugins.ranger-hbase-plugin:ranger-hbase-plugin</include> + </includes> + </moduleSet> + + <moduleSet> + <binaries> + <includeDependencies>true</includeDependencies> + <outputDirectory>/ews/webapp/WEB-INF/classes/ranger-plugins/knox</outputDirectory> + <unpack>false</unpack> + <directoryMode>755</directoryMode> + <fileMode>644</fileMode> + <dependencySets> + <dependencySet> + <includes> + <include>commons-lang:commons-lang:jar:${commons.lang.version}</include> + <include>commons-collections:commons-collections:jar:${commons.collections.version}</include> + <include>org.codehaus.jackson:jackson-core-asl:jar:${codehaus.jackson.version}</include> + <include>org.codehaus.jackson:jackson-mapper-asl:jar:${codehaus.jackson.version}</include> + </includes> + </dependencySet> + </dependencySets> + </binaries> + <includes> + <include>security_plugins.ranger-knox-plugin:ranger-knox-plugin</include> + </includes> + </moduleSet> + + <moduleSet> + <binaries> + <includeDependencies>true</includeDependencies> + <outputDirectory>/ews/webapp/WEB-INF/classes/ranger-plugins/storm</outputDirectory> + <unpack>false</unpack> + <directoryMode>755</directoryMode> + <fileMode>644</fileMode> + <dependencySets> + <dependencySet> + <includes> + <include>org.apache.storm:storm-core:jar:0.9.2-incubating</include> + </includes> + </dependencySet> + </dependencySets> + </binaries> + <includes> + <include>security_plugins.ranger-storm-plugin:ranger-storm-plugin</include> + </includes> + </moduleSet> + + <moduleSet> + <binaries> + <includeDependencies>true</includeDependencies> + <outputDirectory>/ews/webapp/WEB-INF/classes/ranger-plugins/yarn</outputDirectory> + <unpack>false</unpack> + <directoryMode>755</directoryMode> + <fileMode>644</fileMode> + <dependencySets> + <dependencySet> + <includes> + <include>org.apache.hadoop:hadoop-yarn-api:jar:${hadoop.version}</include> + <include>org.apache.hadoop:hadoop-yarn-common:jar:${hadoop.version}</include> + </includes> + </dependencySet> + </dependencySets> + </binaries> + <includes> + <include>security_plugins.ranger-yarn-plugin:ranger-yarn-plugin</include> + </includes> + </moduleSet> + + <moduleSet> + <binaries> <includeDependencies>false</includeDependencies> <outputDirectory>/ews/lib</outputDirectory> <unpack>false</unpack>
