Repository: incubator-ranger Updated Branches: refs/heads/stack 1201f2e3f -> ff4a817e5
RANGER-203: updated ServiceStore with SearchFilter; updated plugins to use download policies from REST interface (instead of file store) by default; Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/ff4a817e Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/ff4a817e Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/ff4a817e Branch: refs/heads/stack Commit: ff4a817e592f423d0b248bacf4823eafdddff766 Parents: 1201f2e Author: Madhan Neethiraj <[email protected]> Authored: Thu Jan 29 18:25:26 2015 -0800 Committer: Madhan Neethiraj <[email protected]> Committed: Thu Jan 29 18:25:26 2015 -0800 ---------------------------------------------------------------------- .../policyengine/RangerPolicyEngineImpl.java | 2 - .../ranger/plugin/service/RangerBasePlugin.java | 28 +- .../ranger/plugin/store/ServiceStore.java | 13 +- .../plugin/store/ServiceStoreFactory.java | 64 +- .../ranger/plugin/store/file/BaseFileStore.java | 4 +- .../plugin/store/file/ServiceFileStore.java | 671 ++++++++++++++----- .../plugin/store/rest/ServiceRESTStore.java | 120 ++-- .../ranger/plugin/util/PolicyRefresher.java | 42 +- .../ranger/plugin/store/TestServiceStore.java | 50 +- .../ranger/plugin/util/TestPolicyRefresher.java | 3 +- .../org/apache/ranger/rest/ServiceREST.java | 32 +- 11 files changed, 756 insertions(+), 273 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ff4a817e/plugin-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java ---------------------------------------------------------------------- diff --git a/plugin-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java b/plugin-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java index 14d0c92..d2053f5 100644 --- a/plugin-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java +++ b/plugin-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java @@ -23,8 +23,6 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.ranger.plugin.audit.RangerAuditHandler; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ff4a817e/plugin-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java ---------------------------------------------------------------------- diff --git a/plugin-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java b/plugin-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java index 8b312af..8f1fa5f 100644 --- a/plugin-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java +++ b/plugin-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java @@ -65,24 +65,28 @@ public class RangerBasePlugin { public synchronized void init(RangerPolicyEngine policyEngine) { cleanup(); - // get the serviceName from download URL: http://ranger-admin-host:port/service/assets/policyList/serviceName - String policyDownloadUrl = RangerConfiguration.getInstance().get("xasecure." + serviceType + ".policymgr.url"); - if(! StringUtils.isEmpty(policyDownloadUrl)) { - int idx = policyDownloadUrl.lastIndexOf('/'); - - if(idx != -1) { - serviceName = policyDownloadUrl.substring(idx + 1); - } - } + String serviceName = RangerConfiguration.getInstance().get("ranger.plugin." + serviceType + ".service.name"); + String serviceStoreClass = RangerConfiguration.getInstance().get("ranger.plugin." + serviceType + ".service.store.class", "org.apache.ranger.plugin.store.rest.ServiceRESTStore"); + String cacheDir = RangerConfiguration.getInstance().get("ranger.plugin." + serviceType + ".service.store.cache.dir", "/tmp"); + long pollingIntervalMs = RangerConfiguration.getInstance().getLong("ranger.plugin." + serviceType + ".service.store.pollIntervalMs", 30 * 1000); if(StringUtils.isEmpty(serviceName)) { - serviceName = RangerConfiguration.getInstance().get("ranger.plugin." + serviceType + ".service.name"); + // get the serviceName from download URL: http://ranger-admin-host:port/service/assets/policyList/serviceName + String policyDownloadUrl = RangerConfiguration.getInstance().get("xasecure." + serviceType + ".policymgr.url"); + + if(! StringUtils.isEmpty(policyDownloadUrl)) { + int idx = policyDownloadUrl.lastIndexOf('/'); + + if(idx != -1) { + serviceName = policyDownloadUrl.substring(idx + 1); + } + } } - ServiceStore serviceStore = ServiceStoreFactory.instance().getServiceStore(); + ServiceStore serviceStore = ServiceStoreFactory.instance().getServiceStore(serviceStoreClass); - refresher = new PolicyRefresher(policyEngine, serviceName, serviceStore); + refresher = new PolicyRefresher(policyEngine, serviceType, serviceName, serviceStore, pollingIntervalMs, cacheDir); refresher.startRefresher(); this.policyEngine = policyEngine; } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ff4a817e/plugin-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java ---------------------------------------------------------------------- diff --git a/plugin-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java b/plugin-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java index 8d48305..e8d970c 100644 --- a/plugin-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java +++ b/plugin-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java @@ -24,9 +24,12 @@ import java.util.List; import org.apache.ranger.plugin.model.RangerPolicy; import org.apache.ranger.plugin.model.RangerService; import org.apache.ranger.plugin.model.RangerServiceDef; +import org.apache.ranger.plugin.util.SearchFilter; import org.apache.ranger.plugin.util.ServicePolicies; public interface ServiceStore { + void init() throws Exception; + RangerServiceDef createServiceDef(RangerServiceDef serviceDef) throws Exception; RangerServiceDef updateServiceDef(RangerServiceDef serviceDef) throws Exception; @@ -37,7 +40,7 @@ public interface ServiceStore { RangerServiceDef getServiceDefByName(String name) throws Exception; - List<RangerServiceDef> getAllServiceDefs() throws Exception; + List<RangerServiceDef> getServiceDefs(SearchFilter filter) throws Exception; RangerService createService(RangerService service) throws Exception; @@ -50,7 +53,7 @@ public interface ServiceStore { RangerService getServiceByName(String name) throws Exception; - List<RangerService> getAllServices() throws Exception; + List<RangerService> getServices(SearchFilter filter) throws Exception; RangerPolicy createPolicy(RangerPolicy policy) throws Exception; @@ -61,11 +64,11 @@ public interface ServiceStore { RangerPolicy getPolicy(Long id) throws Exception; - List<RangerPolicy> getAllPolicies() throws Exception; + List<RangerPolicy> getPolicies(SearchFilter filter) throws Exception; - List<RangerPolicy> getServicePolicies(Long serviceId) throws Exception; + List<RangerPolicy> getServicePolicies(Long serviceId, SearchFilter filter) throws Exception; - List<RangerPolicy> getServicePolicies(String serviceName) throws Exception; + List<RangerPolicy> getServicePolicies(String serviceName, SearchFilter filter) throws Exception; ServicePolicies getServicePoliciesIfUpdated(String serviceName, Long lastKnownVersion) throws Exception; } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ff4a817e/plugin-common/src/main/java/org/apache/ranger/plugin/store/ServiceStoreFactory.java ---------------------------------------------------------------------- diff --git a/plugin-common/src/main/java/org/apache/ranger/plugin/store/ServiceStoreFactory.java b/plugin-common/src/main/java/org/apache/ranger/plugin/store/ServiceStoreFactory.java index 949792b..3d45f89 100644 --- a/plugin-common/src/main/java/org/apache/ranger/plugin/store/ServiceStoreFactory.java +++ b/plugin-common/src/main/java/org/apache/ranger/plugin/store/ServiceStoreFactory.java @@ -19,11 +19,13 @@ package org.apache.ranger.plugin.store; +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.collections.MapUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.ranger.plugin.store.file.ServiceFileStore; -import org.apache.ranger.plugin.store.rest.ServiceRESTStore; -import org.apache.ranger.plugin.util.RangerRESTClient; +import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; public class ServiceStoreFactory { @@ -31,7 +33,8 @@ public class ServiceStoreFactory { private static ServiceStoreFactory sInstance = null; - private ServiceStore serviceStore = null; + private Map<String, ServiceStore> serviceStores = null; + private ServiceStore defaultServiceStore = null; public static ServiceStoreFactory instance() { @@ -43,7 +46,46 @@ public class ServiceStoreFactory { } public ServiceStore getServiceStore() { - return serviceStore; + ServiceStore ret = defaultServiceStore; + + if(ret == null) { // if no service store has been created yet, create the default store. TODO: review the impact and update, if necessary + String defaultServiceStoreClass = RangerConfiguration.getInstance().get("ranger.default.service.store.class", "org.apache.ranger.plugin.store.file.ServiceFileStore"); + + ret = getServiceStore(defaultServiceStoreClass); + } + + return ret; + } + + public ServiceStore getServiceStore(String storeClassname) { + ServiceStore ret = serviceStores.get(storeClassname); + + if(ret == null) { + synchronized(this) { + ret = serviceStores.get(storeClassname); + + if(ret == null) { + try { + @SuppressWarnings("unchecked") + Class<ServiceStore> storeClass = (Class<ServiceStore>)Class.forName(storeClassname); + + ret = storeClass.newInstance(); + + ret.init(); + + serviceStores.put(storeClassname, ret); + + if(defaultServiceStore == null) { + defaultServiceStore = ret; + } + } catch(Exception excp) { + LOG.error("failed to instantiate service store of type " + storeClassname, excp); + } + } + } + } + + return ret; } private ServiceStoreFactory() { @@ -62,18 +104,8 @@ public class ServiceStoreFactory { if(LOG.isDebugEnabled()) { LOG.debug("==> ServiceStoreFactory.init()"); } - - boolean useFileStore = true; - - if(useFileStore) { - serviceStore = new ServiceFileStore(); // TODO: configurable store implementation - } else { - RangerRESTClient restClient = new RangerRESTClient("http://localhost:6080";, ""); - restClient.setBasicAuthInfo("admin", "admin"); - - serviceStore = new ServiceRESTStore(restClient); - } + serviceStores = new HashMap<String, ServiceStore>(); if(LOG.isDebugEnabled()) { LOG.debug("<== ServiceStoreFactory.init()"); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ff4a817e/plugin-common/src/main/java/org/apache/ranger/plugin/store/file/BaseFileStore.java ---------------------------------------------------------------------- diff --git a/plugin-common/src/main/java/org/apache/ranger/plugin/store/file/BaseFileStore.java b/plugin-common/src/main/java/org/apache/ranger/plugin/store/file/BaseFileStore.java index 9493e16..17b46f9 100644 --- a/plugin-common/src/main/java/org/apache/ranger/plugin/store/file/BaseFileStore.java +++ b/plugin-common/src/main/java/org/apache/ranger/plugin/store/file/BaseFileStore.java @@ -58,8 +58,8 @@ public class BaseFileStore { protected static String FILE_SUFFIX_JSON = ".json"; - protected void init() { - dataDir = RangerConfiguration.getInstance().get("ranger.policystore.file.dir", "file:///etc/ranger/data"); + protected void initStore() { + dataDir = RangerConfiguration.getInstance().get("ranger.service.store.file.dir", "file:///etc/ranger/data"); try { gsonBuilder = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z").setPrettyPrinting().create(); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ff4a817e/plugin-common/src/main/java/org/apache/ranger/plugin/store/file/ServiceFileStore.java ---------------------------------------------------------------------- diff --git a/plugin-common/src/main/java/org/apache/ranger/plugin/store/file/ServiceFileStore.java b/plugin-common/src/main/java/org/apache/ranger/plugin/store/file/ServiceFileStore.java index 276c87e..b7471f3 100644 --- a/plugin-common/src/main/java/org/apache/ranger/plugin/store/file/ServiceFileStore.java +++ b/plugin-common/src/main/java/org/apache/ranger/plugin/store/file/ServiceFileStore.java @@ -26,15 +26,20 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.Predicate; +import org.apache.commons.collections.PredicateUtils; import org.apache.commons.lang.ObjectUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; import org.apache.ranger.plugin.model.RangerPolicy; +import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItem; import org.apache.ranger.plugin.model.RangerService; import org.apache.ranger.plugin.model.RangerServiceDef; import org.apache.ranger.plugin.store.ServiceStore; +import org.apache.ranger.plugin.util.SearchFilter; import org.apache.ranger.plugin.util.ServicePolicies; @@ -60,13 +65,23 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { LOG.debug("==> ServiceFileStore.ServiceFileStore()"); } - init(); - if(LOG.isDebugEnabled()) { LOG.debug("<== ServiceFileStore.ServiceFileStore()"); } } + @Override + public void init() throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> ServiceFileStore.init()"); + } + + super.initStore(); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== ServiceFileStore.init()"); + } + } @Override public RangerServiceDef createServiceDef(RangerServiceDef serviceDef) throws Exception { @@ -74,7 +89,7 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { LOG.debug("==> ServiceDefFileStore.createServiceDef(" + serviceDef + ")"); } - RangerServiceDef existing = findServiceDefByName(serviceDef.getName()); + RangerServiceDef existing = getServiceDefByName(serviceDef.getName()); if(existing != null) { throw new Exception(serviceDef.getName() + ": service-def already exists (id=" + existing.getId() + ")"); @@ -109,7 +124,7 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { LOG.debug("==> ServiceDefFileStore.updateServiceDef(" + serviceDef + ")"); } - RangerServiceDef existing = findServiceDefById(serviceDef.getId()); + RangerServiceDef existing = getServiceDef(serviceDef.getId()); if(existing == null) { throw new Exception(serviceDef.getId() + ": service-def does not exist"); @@ -163,7 +178,7 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { LOG.debug("==> ServiceDefFileStore.deleteServiceDef(" + id + ")"); } - RangerServiceDef existing = findServiceDefById(id); + RangerServiceDef existing = getServiceDef(id); if(existing == null) { throw new Exception("service-def does not exist. id=" + id); @@ -202,7 +217,17 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { LOG.debug("==> ServiceDefFileStore.getServiceDef(" + id + ")"); } - RangerServiceDef ret = findServiceDefById(id); + RangerServiceDef ret = null; + + List<RangerServiceDef> serviceDefs = getAllServiceDefs(); + + for(RangerServiceDef sd : serviceDefs) { + if(sd != null && sd.getId() != null && sd.getId().longValue() == id) { + ret = sd; + + break; + } + } if(LOG.isDebugEnabled()) { LOG.debug("<== ServiceDefFileStore.getServiceDef(" + id + "): " + ret); @@ -217,7 +242,17 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { LOG.debug("==> ServiceDefFileStore.getServiceDefByName(" + name + ")"); } - RangerServiceDef ret = findServiceDefByName(name); + RangerServiceDef ret = null; + + List<RangerServiceDef> serviceDefs = getAllServiceDefs(); + + for(RangerServiceDef sd : serviceDefs) { + if(sd != null && StringUtils.equalsIgnoreCase(sd.getName(), name)) { + ret = sd; + + break; + } + } if(LOG.isDebugEnabled()) { LOG.debug("<== ServiceDefFileStore.getServiceDefByName(" + name + "): " + ret); @@ -227,69 +262,19 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { } @Override - public List<RangerServiceDef> getAllServiceDefs() throws Exception { + public List<RangerServiceDef> getServiceDefs(SearchFilter filter) throws Exception { if(LOG.isDebugEnabled()) { - LOG.debug("==> ServiceDefFileStore.getAllServiceDefs()"); + LOG.debug("==> ServiceDefFileStore.getServiceDefs()"); } - List<RangerServiceDef> ret = new ArrayList<RangerServiceDef>(); + List<RangerServiceDef> ret = getAllServiceDefs(); - try { - // load definitions for legacy services from embedded resources - String[] legacyServiceDefResources = { - "/service-defs/ranger-servicedef-hdfs.json", - "/service-defs/ranger-servicedef-hive.json", - "/service-defs/ranger-servicedef-hbase.json", - "/service-defs/ranger-servicedef-knox.json", - "/service-defs/ranger-servicedef-storm.json", - }; - - for(String resource : legacyServiceDefResources) { - RangerServiceDef sd = loadFromResource(resource, RangerServiceDef.class); - - if(sd != null) { - ret.add(sd); - } - } - nextServiceDefId = getMaxId(ret) + 1; - - // load service definitions from file system - List<RangerServiceDef> sds = loadFromDir(new Path(getDataDir()), FILE_PREFIX_SERVICE_DEF, RangerServiceDef.class); - - if(sds != null) { - for(RangerServiceDef sd : sds) { - if(sd != null) { - if(isLegacyServiceDef(sd)) { - LOG.warn("Found in-built service-def '" + sd.getName() + "' under " + getDataDir() + ". Ignorning"); - - continue; - } - - // if the ServiceDef is already found, remove the earlier definition - for(int i = 0; i < ret.size(); i++) { - RangerServiceDef currSd = ret.get(i); - - if(StringUtils.equals(currSd.getName(), sd.getName()) || - ObjectUtils.equals(currSd.getId(), sd.getId())) { - ret.remove(i); - } - } - - ret.add(sd); - } - } - } - nextServiceDefId = getMaxId(ret) + 1; - } catch(Exception excp) { - LOG.error("ServiceDefFileStore.getAllServiceDefs(): failed to read service-defs", excp); + if(ret != null && filter != null) { + CollectionUtils.filter(ret, getServiceDefPredicate(filter)); } if(LOG.isDebugEnabled()) { - LOG.debug("<== ServiceDefFileStore.getAllServiceDefs(): count=" + (ret == null ? 0 : ret.size())); - } - - if(ret != null) { - Collections.sort(ret, RangerServiceDef.idComparator); + LOG.debug("<== ServiceDefFileStore.getServiceDefs(): count=" + (ret == null ? 0 : ret.size())); } return ret; @@ -464,27 +449,19 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { } @Override - public List<RangerService> getAllServices() throws Exception { + public List<RangerService> getServices(SearchFilter filter) throws Exception { if(LOG.isDebugEnabled()) { - LOG.debug("==> ServiceFileStore.getAllServices()"); + LOG.debug("==> ServiceFileStore.getServices()"); } - List<RangerService> ret = null; - - try { - ret = loadFromDir(new Path(getDataDir()), FILE_PREFIX_SERVICE, RangerService.class); + List<RangerService> ret = getAllServices(); - nextServiceId = getMaxId(ret) + 1; - } catch(Exception excp) { - LOG.error("ServiceFileStore.getAllServices(): failed to read services", excp); + if(ret != null && filter != null) { + CollectionUtils.filter(ret, getServicePredicate(filter)); } if(LOG.isDebugEnabled()) { - LOG.debug("<== ServiceFileStore.getAllServices(): count=" + (ret == null ? 0 : ret.size())); - } - - if(ret != null) { - Collections.sort(ret, RangerService.idComparator); + LOG.debug("<== ServiceFileStore.getServices(): count=" + (ret == null ? 0 : ret.size())); } return ret; @@ -655,36 +632,28 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { } @Override - public List<RangerPolicy> getAllPolicies() throws Exception { + public List<RangerPolicy> getPolicies(SearchFilter filter) throws Exception { if(LOG.isDebugEnabled()) { - LOG.debug("==> ServiceFileStore.getAllPolicies()"); + LOG.debug("==> ServiceFileStore.getPolicies()"); } - List<RangerPolicy> ret = null; - - try { - ret = loadFromDir(new Path(getDataDir()), FILE_PREFIX_POLICY, RangerPolicy.class); + List<RangerPolicy> ret = getAllPolicies(); - nextPolicyId = getMaxId(ret) + 1; - } catch(Exception excp) { - LOG.error("ServiceFileStore.getAllPolicies(): failed to read policies", excp); + if(ret != null) { + CollectionUtils.filter(ret, getPolicyPredicate(filter)); } if(LOG.isDebugEnabled()) { - LOG.debug("<== ServiceFileStore.getAllPolicies(): count=" + (ret == null ? 0 : ret.size())); - } - - if(ret != null) { - Collections.sort(ret, RangerPolicy.idComparator); + LOG.debug("<== ServiceFileStore.getPolicies(): count=" + (ret == null ? 0 : ret.size())); } return ret; } @Override - public List<RangerPolicy> getServicePolicies(Long serviceId) throws Exception { + public List<RangerPolicy> getServicePolicies(Long serviceId, SearchFilter filter) throws Exception { if(LOG.isDebugEnabled()) { - LOG.debug("==> ServiceFileStore.getPolicies(" + serviceId + ")"); + LOG.debug("==> ServiceFileStore.getServicePolicies(" + serviceId + ")"); } RangerService service = getService(serviceId); @@ -693,37 +662,25 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { throw new Exception("service does not exist - id='" + serviceId); } - List<RangerPolicy> ret = getServicePolicies(service.getName()); + List<RangerPolicy> ret = getServicePolicies(service.getName(), filter); if(LOG.isDebugEnabled()) { - LOG.debug("<== ServiceFileStore.getPolicies(" + serviceId + "): " + ((ret == null) ? 0 : ret.size())); + LOG.debug("<== ServiceFileStore.getServicePolicies(" + serviceId + "): " + ((ret == null) ? 0 : ret.size())); } return ret; } @Override - public List<RangerPolicy> getServicePolicies(String serviceName) throws Exception { + public List<RangerPolicy> getServicePolicies(String serviceName, SearchFilter filter) throws Exception { if(LOG.isDebugEnabled()) { - LOG.debug("==> ServiceFileStore.getPolicies(" + serviceName + ")"); - } - - RangerService service = getServiceByName(serviceName); - - if(service == null) { - throw new Exception("service does not exist - name='" + serviceName); - } - - RangerServiceDef serviceDef = findServiceDefByName(service.getType()); - - if(serviceDef == null) { - throw new Exception(service.getType() + ": unknown service-def)"); + LOG.debug("==> ServiceFileStore.getServicePolicies(" + serviceName + ")"); } List<RangerPolicy> ret = new ArrayList<RangerPolicy>(); try { - List<RangerPolicy> policies = getAllPolicies(); + List<RangerPolicy> policies = getPolicies(filter); if(policies != null) { for(RangerPolicy policy : policies) { @@ -733,11 +690,11 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { } } } catch(Exception excp) { - LOG.error("ServiceFileStore.getPolicies(" + serviceName + "): failed to read policies", excp); + LOG.error("ServiceFileStore.getServicePolicies(" + serviceName + "): failed to read policies", excp); } if(LOG.isDebugEnabled()) { - LOG.debug("<== ServiceFileStore.getPolicies(" + serviceName + "): count=" + ((ret == null) ? 0 : ret.size())); + LOG.debug("<== ServiceFileStore.getServicePolicies(" + serviceName + "): count=" + ((ret == null) ? 0 : ret.size())); } if(ret != null) { @@ -759,7 +716,7 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { throw new Exception("service does not exist - name=" + serviceName); } - RangerServiceDef serviceDef = findServiceDefByName(service.getType()); + RangerServiceDef serviceDef = getServiceDefByName(service.getType()); if(serviceDef == null) { throw new Exception(service.getType() + ": unknown service-def)"); @@ -801,19 +758,6 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { return ret; } - @Override - protected void init() { - if(LOG.isDebugEnabled()) { - LOG.debug("==> ServiceFileStore.init()"); - } - - super.init(); - - if(LOG.isDebugEnabled()) { - LOG.debug("<== ServiceFileStore.init()"); - } - } - private void handleServiceRename(RangerService service, String oldName) throws Exception { List<RangerPolicy> policies = getAllPolicies(); @@ -872,38 +816,6 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { saveToFile(service, true); } - private RangerServiceDef findServiceDefById(long id) throws Exception { - RangerServiceDef ret = null; - - List<RangerServiceDef> serviceDefs = getAllServiceDefs(); - - for(RangerServiceDef sd : serviceDefs) { - if(sd != null && sd.getId() != null && sd.getId().longValue() == id) { - ret = sd; - - break; - } - } - - return ret; - } - - private RangerServiceDef findServiceDefByName(String sdName) throws Exception { - RangerServiceDef ret = null; - - List<RangerServiceDef> serviceDefs = getAllServiceDefs(); - - for(RangerServiceDef sd : serviceDefs) { - if(sd != null && StringUtils.equalsIgnoreCase(sd.getName(), sdName)) { - ret = sd; - - break; - } - } - - return ret; - } - private RangerPolicy findPolicyByName(String serviceName, String policyName) throws Exception { if(LOG.isDebugEnabled()) { LOG.debug("==> ServiceFileStore.findPolicyByName(" + serviceName + ", " + policyName + ")"); @@ -952,4 +864,447 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { private boolean isLegacyServiceDef(Long id) { return id == null ? false : legacyServiceDefs.containsValue(id); } + + private List<RangerServiceDef> getAllServiceDefs() throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> ServiceDefFileStore.getAllServiceDefs()"); + } + + List<RangerServiceDef> ret = new ArrayList<RangerServiceDef>(); + + try { + // load definitions for legacy services from embedded resources + String[] legacyServiceDefResources = { + "/service-defs/ranger-servicedef-hdfs.json", + "/service-defs/ranger-servicedef-hive.json", + "/service-defs/ranger-servicedef-hbase.json", + "/service-defs/ranger-servicedef-knox.json", + "/service-defs/ranger-servicedef-storm.json", + }; + + for(String resource : legacyServiceDefResources) { + RangerServiceDef sd = loadFromResource(resource, RangerServiceDef.class); + + if(sd != null) { + ret.add(sd); + } + } + nextServiceDefId = getMaxId(ret) + 1; + + // load service definitions from file system + List<RangerServiceDef> sds = loadFromDir(new Path(getDataDir()), FILE_PREFIX_SERVICE_DEF, RangerServiceDef.class); + + if(sds != null) { + for(RangerServiceDef sd : sds) { + if(sd != null) { + if(isLegacyServiceDef(sd)) { + LOG.warn("Found in-built service-def '" + sd.getName() + "' under " + getDataDir() + ". Ignorning"); + + continue; + } + + // if the ServiceDef is already found, remove the earlier definition + for(int i = 0; i < ret.size(); i++) { + RangerServiceDef currSd = ret.get(i); + + if(StringUtils.equals(currSd.getName(), sd.getName()) || + ObjectUtils.equals(currSd.getId(), sd.getId())) { + ret.remove(i); + } + } + + ret.add(sd); + } + } + } + nextServiceDefId = getMaxId(ret) + 1; + } catch(Exception excp) { + LOG.error("ServiceDefFileStore.getAllServiceDefs(): failed to read service-defs", excp); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== ServiceDefFileStore.getAllServiceDefs(): count=" + (ret == null ? 0 : ret.size())); + } + + if(ret != null) { + Collections.sort(ret, RangerServiceDef.idComparator); + } + + return ret; + } + + private List<RangerService> getAllServices() throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> ServiceFileStore.getAllServices()"); + } + + List<RangerService> ret = null; + + try { + ret = loadFromDir(new Path(getDataDir()), FILE_PREFIX_SERVICE, RangerService.class); + + nextServiceId = getMaxId(ret) + 1; + } catch(Exception excp) { + LOG.error("ServiceFileStore.getAllServices(): failed to read services", excp); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== ServiceFileStore.getAllServices(): count=" + (ret == null ? 0 : ret.size())); + } + + if(ret != null) { + Collections.sort(ret, RangerService.idComparator); + } + + return ret; + } + + private List<RangerPolicy> getAllPolicies() throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> ServiceFileStore.getAllPolicies()"); + } + + List<RangerPolicy> ret = null; + + try { + ret = loadFromDir(new Path(getDataDir()), FILE_PREFIX_POLICY, RangerPolicy.class); + + nextPolicyId = getMaxId(ret) + 1; + } catch(Exception excp) { + LOG.error("ServiceFileStore.getAllPolicies(): failed to read policies", excp); + } + + if(ret != null) { + Collections.sort(ret, RangerPolicy.idComparator); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== ServiceFileStore.getAllPolicies(): count=" + (ret == null ? 0 : ret.size())); + } + + return ret; + } + + private String getServiceType(String serviceName) { + RangerService service = null; + + try { + service = getServiceByName(serviceName); + } catch(Exception excp) { + // ignore + } + + return service != null ? service.getType() : null; + } + + private Long getServiceId(String serviceName) { + RangerService service = null; + + try { + service = getServiceByName(serviceName); + } catch(Exception excp) { + // ignore + } + + return service != null ? service.getId() : null; + } + + /* + public static final String LOGIN_USER = "loginUser"; + public static final String SERVICE_TYPE = "serviceType"; + public static final String SERVICE_NAME = "serviceName"; + public static final String SERVICE_ID = "serviceId"; + public static final String POLICY_NAME = "policyName"; + public static final String RESOURCE_PREFIX = "resource:"; + public static final String STATUS = "status"; + public static final String USER_NAME = "userName"; + public static final String GROUP_NAME = "groupName"; + public static final String START_INDEX = "startIndex"; + public static final String PAGE_SIZE = "pageSize"; + public static final String SORT_BY = "sortBy"; + */ + + private Predicate getServiceDefPredicate(SearchFilter filter) { + if(filter == null) { + return null; + } + + List<Predicate> predicates = new ArrayList<Predicate>(); + + final String serviceType = filter.getParam(SearchFilter.SERVICE_TYPE); + if(! StringUtils.isEmpty(serviceType)) { + Predicate p = new Predicate() { + @Override + public boolean evaluate(Object object) { + boolean ret = false; + + if(object != null && object instanceof RangerServiceDef) { + RangerServiceDef serviceDef = (RangerServiceDef)object; + + ret = StringUtils.equals(serviceType, serviceDef.getName()); + } + + return ret; + } + }; + + predicates.add(p); + } + + Predicate ret = CollectionUtils.isEmpty(predicates) ? null : PredicateUtils.allPredicate(predicates); + + return ret; + } + + private Predicate getServicePredicate(SearchFilter filter) { + if(filter == null) { + return null; + } + + List<Predicate> predicates = new ArrayList<Predicate>(); + + final String serviceType = filter.getParam(SearchFilter.SERVICE_TYPE); + if(! StringUtils.isEmpty(serviceType)) { + Predicate p = new Predicate() { + @Override + public boolean evaluate(Object object) { + boolean ret = false; + + if(object != null && object instanceof RangerService) { + RangerService service = (RangerService)object; + + ret = StringUtils.equals(serviceType, service.getType()); + } + + return ret; + } + }; + + predicates.add(p); + } + + final String serviceName = filter.getParam(SearchFilter.SERVICE_NAME); + if(! StringUtils.isEmpty(serviceName)) { + Predicate p = new Predicate() { + @Override + public boolean evaluate(Object object) { + boolean ret = false; + + if(object != null && object instanceof RangerService) { + RangerService service = (RangerService)object; + + ret = StringUtils.equals(serviceName, service.getName()); + } + + return ret; + } + }; + + predicates.add(p); + } + + final String serviceId = filter.getParam(SearchFilter.SERVICE_ID); + if(! StringUtils.isEmpty(serviceId)) { + Predicate p = new Predicate() { + @Override + public boolean evaluate(Object object) { + boolean ret = false; + + if(object != null && object instanceof RangerService) { + RangerService service = (RangerService)object; + Long svcId = service.getId(); + + if(svcId != null) { + ret = StringUtils.equals(serviceId, svcId.toString()); + } + } + + return ret; + } + }; + + predicates.add(p); + } + + Predicate ret = CollectionUtils.isEmpty(predicates) ? null : PredicateUtils.allPredicate(predicates); + + return ret; + } + + private Predicate getPolicyPredicate(SearchFilter filter) { + if(filter == null) { + return null; + } + + List<Predicate> predicates = new ArrayList<Predicate>(); + + final String loginUser = filter.getParam(SearchFilter.LOGIN_USER); + if(! StringUtils.isEmpty(loginUser)) { + Predicate p = new Predicate() { + @Override + public boolean evaluate(Object object) { + boolean ret = false; + + if(object != null && object instanceof RangerPolicy) { + RangerPolicy policy = (RangerPolicy)object; + + for(RangerPolicyItem policyItem : policy.getPolicyItems()) { + if(!policyItem.getDelegateAdmin()) { + continue; + } + + if(policyItem.getUsers().contains(loginUser)) { // TODO: group membership check + ret = true; + + break; + } + } + } + + return ret; + } + }; + + predicates.add(p); + } + + final String serviceType = filter.getParam(SearchFilter.SERVICE_TYPE); + if(! StringUtils.isEmpty(serviceType)) { + Predicate p = new Predicate() { + @Override + public boolean evaluate(Object object) { + boolean ret = false; + + if(object != null && object instanceof RangerPolicy) { + RangerPolicy policy = (RangerPolicy)object; + + ret = StringUtils.equals(serviceType, getServiceType(policy.getService())); + } + + return ret; + } + }; + + predicates.add(p); + } + + final String serviceName = filter.getParam(SearchFilter.SERVICE_NAME); + if(! StringUtils.isEmpty(serviceName)) { + Predicate p = new Predicate() { + @Override + public boolean evaluate(Object object) { + boolean ret = false; + + if(object != null && object instanceof RangerPolicy) { + RangerPolicy policy = (RangerPolicy)object; + + ret = StringUtils.equals(serviceName, policy.getService()); + } + + return ret; + } + }; + + predicates.add(p); + } + + final String serviceId = filter.getParam(SearchFilter.SERVICE_ID); + if(! StringUtils.isEmpty(serviceId)) { + Predicate p = new Predicate() { + @Override + public boolean evaluate(Object object) { + boolean ret = false; + + if(object != null && object instanceof RangerPolicy) { + RangerPolicy policy = (RangerPolicy)object; + Long svcId = getServiceId(policy.getService()); + + if(svcId != null) { + ret = StringUtils.equals(serviceId, svcId.toString()); + } + } + + return ret; + } + }; + + predicates.add(p); + } + + final String policyName = filter.getParam(SearchFilter.POLICY_NAME); + if(! StringUtils.isEmpty(policyName)) { + Predicate p = new Predicate() { + @Override + public boolean evaluate(Object object) { + boolean ret = false; + + if(object != null && object instanceof RangerPolicy) { + RangerPolicy policy = (RangerPolicy)object; + + ret = StringUtils.equals(policyName, policy.getName()); + } + + return ret; + } + }; + + predicates.add(p); + } + + final String userName = filter.getParam(SearchFilter.USER_NAME); + if(! StringUtils.isEmpty(userName)) { + Predicate p = new Predicate() { + @Override + public boolean evaluate(Object object) { + boolean ret = false; + + if(object != null && object instanceof RangerPolicy) { + RangerPolicy policy = (RangerPolicy)object; + + for(RangerPolicyItem policyItem : policy.getPolicyItems()) { + if(policyItem.getUsers().contains(userName)) { // TODO: group membership check + ret = true; + + break; + } + } + } + + return ret; + } + }; + + predicates.add(p); + } + + final String groupName = filter.getParam(SearchFilter.GROUP_NAME); + if(! StringUtils.isEmpty(groupName)) { + Predicate p = new Predicate() { + @Override + public boolean evaluate(Object object) { + boolean ret = false; + + if(object != null && object instanceof RangerPolicy) { + RangerPolicy policy = (RangerPolicy)object; + + for(RangerPolicyItem policyItem : policy.getPolicyItems()) { + if(policyItem.getGroups().contains(groupName)) { + ret = true; + + break; + } + } + } + + return ret; + } + }; + + predicates.add(p); + } + + Predicate ret = CollectionUtils.isEmpty(predicates) ? null : PredicateUtils.allPredicate(predicates); + + return ret; + } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ff4a817e/plugin-common/src/main/java/org/apache/ranger/plugin/store/rest/ServiceRESTStore.java ---------------------------------------------------------------------- diff --git a/plugin-common/src/main/java/org/apache/ranger/plugin/store/rest/ServiceRESTStore.java b/plugin-common/src/main/java/org/apache/ranger/plugin/store/rest/ServiceRESTStore.java index cdb2fa5..dd3624b 100644 --- a/plugin-common/src/main/java/org/apache/ranger/plugin/store/rest/ServiceRESTStore.java +++ b/plugin-common/src/main/java/org/apache/ranger/plugin/store/rest/ServiceRESTStore.java @@ -24,11 +24,13 @@ import java.util.List; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.ranger.admin.client.datatype.RESTResponse; +import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; import org.apache.ranger.plugin.model.RangerPolicy; import org.apache.ranger.plugin.model.RangerService; import org.apache.ranger.plugin.model.RangerServiceDef; import org.apache.ranger.plugin.store.ServiceStore; import org.apache.ranger.plugin.util.RangerRESTClient; +import org.apache.ranger.plugin.util.SearchFilter; import org.apache.ranger.plugin.util.ServicePolicies; import com.sun.jersey.api.client.ClientResponse; @@ -62,23 +64,32 @@ public class ServiceRESTStore implements ServiceStore { public final String REST_URL_POLICY_GET_ALL = "/service/plugins/policies"; public final String REST_URL_POLICY_GET_FOR_SERVICE = "/service/plugins/policies/service/"; public final String REST_URL_POLICY_GET_FOR_SERVICE_BY_NAME = "/service/plugins/policies/service/name/"; + public final String REST_URL_POLICY_GET_FOR_SERVICE_IF_UPDATED = "/service/plugins/policies/service/name/"; public static final String REST_MIME_TYPE_JSON = "application/json" ; private RangerRESTClient restClient; - public ServiceRESTStore(RangerRESTClient restClient) { + public ServiceRESTStore() { if(LOG.isDebugEnabled()) { - LOG.debug("==> ServiceRESTStore.ServiceRESTStore(" + restClient + ")"); + LOG.debug("==> ServiceRESTStore.ServiceRESTStore()"); } - this.restClient = restClient; - if(LOG.isDebugEnabled()) { - LOG.debug("<== ServiceRESTStore.ServiceRESTStore(" + restClient + ")"); + LOG.debug("<== ServiceRESTStore.ServiceRESTStore()"); } } + @Override + public void init() throws Exception { + String restUrl = RangerConfiguration.getInstance().get("ranger.service.store.rest.url", "http://localhost:6080";); + String restUsername = RangerConfiguration.getInstance().get("ranger.service.store.rest.username", "admin"); + String restPassword = RangerConfiguration.getInstance().get("ranger.service.store.rest.password", "admin"); + String sslConfigFile = RangerConfiguration.getInstance().get("ranger.service.store.rest.ssl.config.file", ""); + + restClient = new RangerRESTClient(restUrl, sslConfigFile); + restClient.setBasicAuthInfo(restUsername, restPassword); + } @Override public RangerServiceDef createServiceDef(RangerServiceDef serviceDef) throws Exception { @@ -88,7 +99,7 @@ public class ServiceRESTStore implements ServiceStore { RangerServiceDef ret = null; - WebResource webResource = restClient.getResource(REST_URL_SERVICEDEF_CREATE); + WebResource webResource = createWebResource(REST_URL_SERVICEDEF_CREATE); ClientResponse response = webResource.accept(REST_MIME_TYPE_JSON).type(REST_MIME_TYPE_JSON).post(ClientResponse.class, restClient.toJson(serviceDef)); if(response != null && response.getStatus() == 200) { @@ -114,7 +125,7 @@ public class ServiceRESTStore implements ServiceStore { RangerServiceDef ret = null; - WebResource webResource = restClient.getResource(REST_URL_SERVICEDEF_UPDATE + serviceDef.getId()); + WebResource webResource = createWebResource(REST_URL_SERVICEDEF_UPDATE + serviceDef.getId()); ClientResponse response = webResource.accept(REST_MIME_TYPE_JSON).type(REST_MIME_TYPE_JSON).put(ClientResponse.class, restClient.toJson(serviceDef)); if(response != null && response.getStatus() == 200) { @@ -138,7 +149,7 @@ public class ServiceRESTStore implements ServiceStore { LOG.debug("==> ServiceRESTStore.deleteServiceDef(" + id + ")"); } - WebResource webResource = restClient.getResource(REST_URL_SERVICEDEF_DELETE + id); + WebResource webResource = createWebResource(REST_URL_SERVICEDEF_DELETE + id); ClientResponse response = webResource.accept(REST_MIME_TYPE_JSON).delete(ClientResponse.class); if(response == null || (response.getStatus() != 200 && response.getStatus() != 204)) { @@ -160,7 +171,7 @@ public class ServiceRESTStore implements ServiceStore { RangerServiceDef ret = null; - WebResource webResource = restClient.getResource(REST_URL_SERVICEDEF_GET + id); + WebResource webResource = createWebResource(REST_URL_SERVICEDEF_GET + id); ClientResponse response = webResource.accept(REST_MIME_TYPE_JSON).get(ClientResponse.class); if(response != null && response.getStatus() == 200) { @@ -186,7 +197,7 @@ public class ServiceRESTStore implements ServiceStore { RangerServiceDef ret = null; - WebResource webResource = restClient.getResource(REST_URL_SERVICEDEF_GET_BY_NAME + name); + WebResource webResource = createWebResource(REST_URL_SERVICEDEF_GET_BY_NAME + name); ClientResponse response = webResource.accept(REST_MIME_TYPE_JSON).get(ClientResponse.class); if(response != null && response.getStatus() == 200) { @@ -205,14 +216,14 @@ public class ServiceRESTStore implements ServiceStore { } @Override - public List<RangerServiceDef> getAllServiceDefs() throws Exception { + public List<RangerServiceDef> getServiceDefs(SearchFilter filter) throws Exception { if(LOG.isDebugEnabled()) { - LOG.debug("==> ServiceRESTStore.getAllServiceDefs()"); + LOG.debug("==> ServiceRESTStore.getServiceDefs()"); } List<RangerServiceDef> ret = null; - WebResource webResource = restClient.getResource(REST_URL_SERVICEDEF_GET_ALL); + WebResource webResource = createWebResource(REST_URL_SERVICEDEF_GET_ALL, filter); ClientResponse response = webResource.accept(REST_MIME_TYPE_JSON).get(ClientResponse.class); if(response != null && response.getStatus() == 200) { @@ -238,7 +249,7 @@ public class ServiceRESTStore implements ServiceStore { RangerService ret = null; - WebResource webResource = restClient.getResource(REST_URL_SERVICE_CREATE); + WebResource webResource = createWebResource(REST_URL_SERVICE_CREATE); ClientResponse response = webResource.accept(REST_MIME_TYPE_JSON).type(REST_MIME_TYPE_JSON).post(ClientResponse.class, restClient.toJson(service)); if(response != null && response.getStatus() == 200) { @@ -264,7 +275,7 @@ public class ServiceRESTStore implements ServiceStore { RangerService ret = null; - WebResource webResource = restClient.getResource(REST_URL_SERVICE_UPDATE + service.getId()); + WebResource webResource = createWebResource(REST_URL_SERVICE_UPDATE + service.getId()); ClientResponse response = webResource.accept(REST_MIME_TYPE_JSON).type(REST_MIME_TYPE_JSON).put(ClientResponse.class, restClient.toJson(service)); if(response != null && response.getStatus() == 200) { @@ -288,7 +299,7 @@ public class ServiceRESTStore implements ServiceStore { LOG.debug("==> ServiceRESTStore.deleteService(" + id + ")"); } - WebResource webResource = restClient.getResource(REST_URL_SERVICE_DELETE + id); + WebResource webResource = createWebResource(REST_URL_SERVICE_DELETE + id); ClientResponse response = webResource.accept(REST_MIME_TYPE_JSON).delete(ClientResponse.class); if(response == null || (response.getStatus() != 200 && response.getStatus() != 204)) { @@ -310,7 +321,7 @@ public class ServiceRESTStore implements ServiceStore { RangerService ret = null; - WebResource webResource = restClient.getResource(REST_URL_SERVICE_GET + id); + WebResource webResource = createWebResource(REST_URL_SERVICE_GET + id); ClientResponse response = webResource.accept(REST_MIME_TYPE_JSON).get(ClientResponse.class); if(response != null && response.getStatus() == 200) { @@ -336,7 +347,7 @@ public class ServiceRESTStore implements ServiceStore { RangerService ret = null; - WebResource webResource = restClient.getResource(REST_URL_SERVICE_GET_BY_NAME + name); + WebResource webResource = createWebResource(REST_URL_SERVICE_GET_BY_NAME + name); ClientResponse response = webResource.accept(REST_MIME_TYPE_JSON).get(ClientResponse.class); if(response != null && response.getStatus() == 200) { @@ -355,14 +366,14 @@ public class ServiceRESTStore implements ServiceStore { } @Override - public List<RangerService> getAllServices() throws Exception { + public List<RangerService> getServices(SearchFilter filter) throws Exception { if(LOG.isDebugEnabled()) { - LOG.debug("==> ServiceRESTStore.getAllServices()"); + LOG.debug("==> ServiceRESTStore.getServices()"); } List<RangerService> ret = null; - WebResource webResource = restClient.getResource(REST_URL_SERVICE_GET_ALL); + WebResource webResource = createWebResource(REST_URL_SERVICE_GET_ALL, filter); ClientResponse response = webResource.accept(REST_MIME_TYPE_JSON).get(ClientResponse.class); if(response != null && response.getStatus() == 200) { @@ -374,7 +385,7 @@ public class ServiceRESTStore implements ServiceStore { } if(LOG.isDebugEnabled()) { - LOG.debug("<== ServiceRESTStore.getAllServices(): " + ret); + LOG.debug("<== ServiceRESTStore.getServices(): " + ret); } return ret; @@ -388,7 +399,7 @@ public class ServiceRESTStore implements ServiceStore { RangerPolicy ret = null; - WebResource webResource = restClient.getResource(REST_URL_POLICY_CREATE); + WebResource webResource = createWebResource(REST_URL_POLICY_CREATE); ClientResponse response = webResource.accept(REST_MIME_TYPE_JSON).type(REST_MIME_TYPE_JSON).post(ClientResponse.class, restClient.toJson(policy)); if(response != null && response.getStatus() == 200) { @@ -414,7 +425,7 @@ public class ServiceRESTStore implements ServiceStore { RangerPolicy ret = null; - WebResource webResource = restClient.getResource(REST_URL_POLICY_UPDATE + policy.getId()); + WebResource webResource = createWebResource(REST_URL_POLICY_UPDATE + policy.getId()); ClientResponse response = webResource.accept(REST_MIME_TYPE_JSON).type(REST_MIME_TYPE_JSON).put(ClientResponse.class, restClient.toJson(policy)); if(response != null && response.getStatus() == 200) { @@ -438,7 +449,7 @@ public class ServiceRESTStore implements ServiceStore { LOG.debug("==> ServiceRESTStore.deletePolicy(" + id + ")"); } - WebResource webResource = restClient.getResource(REST_URL_POLICY_DELETE + id); + WebResource webResource = createWebResource(REST_URL_POLICY_DELETE + id); ClientResponse response = webResource.accept(REST_MIME_TYPE_JSON).delete(ClientResponse.class); if(response == null || (response.getStatus() != 200 && response.getStatus() != 204)) { @@ -460,7 +471,7 @@ public class ServiceRESTStore implements ServiceStore { RangerPolicy ret = null; - WebResource webResource = restClient.getResource(REST_URL_POLICY_GET + id); + WebResource webResource = createWebResource(REST_URL_POLICY_GET + id); ClientResponse response = webResource.accept(REST_MIME_TYPE_JSON).get(ClientResponse.class); if(response != null && response.getStatus() == 200) { @@ -479,14 +490,14 @@ public class ServiceRESTStore implements ServiceStore { } @Override - public List<RangerPolicy> getAllPolicies() throws Exception { + public List<RangerPolicy> getPolicies(SearchFilter filter) throws Exception { if(LOG.isDebugEnabled()) { - LOG.debug("==> ServiceRESTStore.getAllPolicies()"); + LOG.debug("==> ServiceRESTStore.getPolicies()"); } List<RangerPolicy> ret = null; - WebResource webResource = restClient.getResource(REST_URL_POLICY_GET_ALL); + WebResource webResource = createWebResource(REST_URL_POLICY_GET_ALL, filter); ClientResponse response = webResource.accept(REST_MIME_TYPE_JSON).get(ClientResponse.class); if(response != null && response.getStatus() == 200) { @@ -498,21 +509,21 @@ public class ServiceRESTStore implements ServiceStore { } if(LOG.isDebugEnabled()) { - LOG.debug("<== ServiceRESTStore.getAllPolicies(): " + ret); + LOG.debug("<== ServiceRESTStore.getPolicies(): " + ret); } return ret; } @Override - public List<RangerPolicy> getServicePolicies(Long serviceId) throws Exception { + public List<RangerPolicy> getServicePolicies(Long serviceId, SearchFilter filter) throws Exception { if(LOG.isDebugEnabled()) { LOG.debug("==> ServiceRESTStore.getServicePolicies(" + serviceId + ")"); } List<RangerPolicy> ret = null; - WebResource webResource = restClient.getResource(REST_URL_POLICY_GET_FOR_SERVICE + serviceId); + WebResource webResource = createWebResource(REST_URL_POLICY_GET_FOR_SERVICE + serviceId, filter); ClientResponse response = webResource.accept(REST_MIME_TYPE_JSON).get(ClientResponse.class); if(response != null && response.getStatus() == 200) { @@ -531,14 +542,14 @@ public class ServiceRESTStore implements ServiceStore { } @Override - public List<RangerPolicy> getServicePolicies(String serviceName) throws Exception { + public List<RangerPolicy> getServicePolicies(String serviceName, SearchFilter filter) throws Exception { if(LOG.isDebugEnabled()) { LOG.debug("==> ServiceRESTStore.getServicePolicies(" + serviceName + ")"); } List<RangerPolicy> ret = null; - WebResource webResource = restClient.getResource(REST_URL_POLICY_GET_FOR_SERVICE_BY_NAME + serviceName); + WebResource webResource = createWebResource(REST_URL_POLICY_GET_FOR_SERVICE_BY_NAME + serviceName, filter); ClientResponse response = webResource.accept(REST_MIME_TYPE_JSON).get(ClientResponse.class); if(response != null && response.getStatus() == 200) { @@ -557,9 +568,42 @@ public class ServiceRESTStore implements ServiceStore { } @Override - public ServicePolicies getServicePoliciesIfUpdated(String serviceName, - Long lastKnownVersion) throws Exception { - // TODO Auto-generated method stub - return null; + public ServicePolicies getServicePoliciesIfUpdated(String serviceName, Long lastKnownVersion) throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> ServiceRESTStore.getServicePoliciesIfUpdated(" + serviceName + ", " + lastKnownVersion + ")"); + } + + ServicePolicies ret = null; + + WebResource webResource = createWebResource(REST_URL_POLICY_GET_FOR_SERVICE_IF_UPDATED + serviceName + "/" + lastKnownVersion); + ClientResponse response = webResource.accept(REST_MIME_TYPE_JSON).get(ClientResponse.class); + + if(response != null && response.getStatus() == 200) { + ret = response.getEntity(ServicePolicies.class); + } else { + RESTResponse resp = RESTResponse.fromClientResponse(response); + + throw new Exception(resp.getMessage()); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== ServiceRESTStore.getServicePoliciesIfUpdated(" + serviceName + ", " + lastKnownVersion + "): " + ret); + } + + return ret; + } + + private WebResource createWebResource(String url) { + return createWebResource(url, null); + } + + private WebResource createWebResource(String url, SearchFilter filter) { + WebResource ret = restClient.getResource(url); + + if(filter != null) { + // TODO: add query params for filter + } + + return ret; } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ff4a817e/plugin-common/src/main/java/org/apache/ranger/plugin/util/PolicyRefresher.java ---------------------------------------------------------------------- diff --git a/plugin-common/src/main/java/org/apache/ranger/plugin/util/PolicyRefresher.java b/plugin-common/src/main/java/org/apache/ranger/plugin/util/PolicyRefresher.java index 1ff87ce..2437b3e 100644 --- a/plugin-common/src/main/java/org/apache/ranger/plugin/util/PolicyRefresher.java +++ b/plugin-common/src/main/java/org/apache/ranger/plugin/util/PolicyRefresher.java @@ -21,7 +21,6 @@ package org.apache.ranger.plugin.util; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; import org.apache.ranger.plugin.policyengine.RangerPolicyEngine; import org.apache.ranger.plugin.store.ServiceStore; @@ -30,24 +29,25 @@ public class PolicyRefresher extends Thread { private static final Log LOG = LogFactory.getLog(PolicyRefresher.class); private RangerPolicyEngine policyEngine = null; + private String serviceType = null; private String serviceName = null; private ServiceStore serviceStore = null; - private ServicePolicies lastKnownPolicies = null; + private long pollingIntervalMs = 30 * 1000; - private boolean shutdownFlag = false; - private long pollingIntervalMilliSeconds = 30 * 1000; + private boolean shutdownFlag = false; + private ServicePolicies lastKnownPolicies = null; - public PolicyRefresher(RangerPolicyEngine policyEngine, String serviceName, ServiceStore serviceStore) { + public PolicyRefresher(RangerPolicyEngine policyEngine, String serviceType, String serviceName, ServiceStore serviceStore, long pollingIntervalMs, String cacheDir) { if(LOG.isDebugEnabled()) { LOG.debug("==> PolicyRefresher.PolicyRefresher(serviceName=" + serviceName + ")"); } - this.policyEngine = policyEngine; - this.serviceName = serviceName; - this.serviceStore = serviceStore; - - this.pollingIntervalMilliSeconds = RangerConfiguration.getInstance().getLong("xasecure.hdfs.policymgr.url.reloadIntervalInMillis", 30 * 1000); + this.policyEngine = policyEngine; + this.serviceType = serviceType; + this.serviceName = serviceName; + this.serviceStore = serviceStore; + this.pollingIntervalMs = pollingIntervalMs; if(LOG.isDebugEnabled()) { LOG.debug("<== PolicyRefresher.PolicyRefresher(serviceName=" + serviceName + ")"); @@ -62,6 +62,13 @@ public class PolicyRefresher extends Thread { } /** + * @return the serviceType + */ + public String getServiceType() { + return serviceType; + } + + /** * @return the serviceName */ public String getServiceName() { @@ -69,17 +76,24 @@ public class PolicyRefresher extends Thread { } /** + * @return the serviceStore + */ + public ServiceStore getServiceStore() { + return serviceStore; + } + + /** * @return the pollingIntervalMilliSeconds */ - public long getPollingIntervalMilliSeconds() { - return pollingIntervalMilliSeconds; + public long getPollingIntervalMs() { + return pollingIntervalMs; } /** * @param pollingIntervalMilliSeconds the pollingIntervalMilliSeconds to set */ public void setPollingIntervalMilliSeconds(long pollingIntervalMilliSeconds) { - this.pollingIntervalMilliSeconds = pollingIntervalMilliSeconds; + this.pollingIntervalMs = pollingIntervalMilliSeconds; } public void startRefresher() { @@ -125,7 +139,7 @@ public class PolicyRefresher extends Thread { } try { - Thread.sleep(pollingIntervalMilliSeconds); + Thread.sleep(pollingIntervalMs); } catch(Exception excp) { LOG.error("PolicyRefresher(serviceName=" + serviceName + ").run(): error while sleep. exiting thread", excp); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ff4a817e/plugin-common/src/test/java/org/apache/ranger/plugin/store/TestServiceStore.java ---------------------------------------------------------------------- diff --git a/plugin-common/src/test/java/org/apache/ranger/plugin/store/TestServiceStore.java b/plugin-common/src/test/java/org/apache/ranger/plugin/store/TestServiceStore.java index d0ef299..a365240 100644 --- a/plugin-common/src/test/java/org/apache/ranger/plugin/store/TestServiceStore.java +++ b/plugin-common/src/test/java/org/apache/ranger/plugin/store/TestServiceStore.java @@ -31,12 +31,14 @@ import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItem; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource; import org.apache.ranger.plugin.store.ServiceStore; import org.apache.ranger.plugin.store.ServiceStoreFactory; +import org.apache.ranger.plugin.util.SearchFilter; import org.apache.ranger.plugin.util.ServicePolicies; import org.junit.BeforeClass; import org.junit.Test; public class TestServiceStore { - static ServiceStore svcStore = null; + static ServiceStore svcStore = null; + static SearchFilter filter = null; static final String sdName = "svcDef-unit-test-TestServiceStore"; static final String serviceName = "svc-unit-test-TestServiceStore"; @@ -47,14 +49,14 @@ public class TestServiceStore { svcStore = ServiceStoreFactory.instance().getServiceStore(); // cleanup if the test service and service-def if they already exist - List<RangerService> services = svcStore.getAllServices(); + List<RangerService> services = svcStore.getServices(filter); for(RangerService service : services) { if(service.getName().startsWith(serviceName)) { svcStore.deleteService(service.getId()); } } - List<RangerServiceDef> serviceDefs = svcStore.getAllServiceDefs(); + List<RangerServiceDef> serviceDefs = svcStore.getServiceDefs(filter); for(RangerServiceDef serviceDef : serviceDefs) { if(serviceDef.getName().startsWith(sdName)) { svcStore.deleteServiceDef(serviceDef.getId()); @@ -66,7 +68,7 @@ public class TestServiceStore { public void testServiceStore() throws Exception { String updatedName, updatedDescription; - List<RangerServiceDef> sds = svcStore.getAllServiceDefs(); + List<RangerServiceDef> sds = svcStore.getServiceDefs(filter); int initSdCount = sds == null ? 0 : sds.size(); @@ -75,7 +77,7 @@ public class TestServiceStore { RangerServiceDef createdSd = svcStore.createServiceDef(sd); assertNotNull("createServiceDef() failed", createdSd != null); - sds = svcStore.getAllServiceDefs(); + sds = svcStore.getServiceDefs(filter); assertEquals("createServiceDef() failed", initSdCount + 1, sds == null ? 0 : sds.size()); updatedDescription = sd.getDescription() + ": updated"; @@ -84,7 +86,7 @@ public class TestServiceStore { assertNotNull("updateServiceDef(updatedDescription) failed", updatedSd); assertEquals("updateServiceDef(updatedDescription) failed", updatedDescription, updatedSd.getDescription()); - sds = svcStore.getAllServiceDefs(); + sds = svcStore.getServiceDefs(filter); assertEquals("updateServiceDef(updatedDescription) failed", initSdCount + 1, sds == null ? 0 : sds.size()); /* @@ -98,7 +100,7 @@ public class TestServiceStore { assertEquals("updateServiceDef(updatedName) failed", initSdCount + 1, sds == null ? 0 : sds.size()); */ - List<RangerService> services = svcStore.getAllServices(); + List<RangerService> services = svcStore.getServices(filter); int initServiceCount = services == null ? 0 : services.size(); @@ -107,7 +109,7 @@ public class TestServiceStore { RangerService createdSvc = svcStore.createService(svc); assertNotNull("createService() failed", createdSvc); - services = svcStore.getAllServices(); + services = svcStore.getServices(filter); assertEquals("createServiceDef() failed", initServiceCount + 1, services == null ? 0 : services.size()); updatedDescription = createdSvc.getDescription() + ": updated"; @@ -116,7 +118,7 @@ public class TestServiceStore { assertNotNull("updateService(updatedDescription) failed", updatedSvc); assertEquals("updateService(updatedDescription) failed", updatedDescription, updatedSvc.getDescription()); - services = svcStore.getAllServices(); + services = svcStore.getServices(filter); assertEquals("updateService(updatedDescription) failed", initServiceCount + 1, services == null ? 0 : services.size()); updatedName = serviceName + "-Renamed"; @@ -125,10 +127,10 @@ public class TestServiceStore { assertNotNull("updateService(updatedName) failed", updatedSvc); assertEquals("updateService(updatedName) failed", updatedName, updatedSvc.getName()); - services = svcStore.getAllServices(); + services = svcStore.getServices(filter); assertEquals("updateService(updatedName) failed", initServiceCount + 1, services == null ? 0 : services.size()); - List<RangerPolicy> policies = svcStore.getAllPolicies(); + List<RangerPolicy> policies = svcStore.getPolicies(filter); int initPolicyCount = policies == null ? 0 : policies.size(); @@ -171,7 +173,7 @@ public class TestServiceStore { assertNotNull(createItem2.getGroups()); assertEquals(createItem2.getGroups().size(), 1); - policies = svcStore.getAllPolicies(); + policies = svcStore.getPolicies(filter); assertEquals("createPolicy() failed", initPolicyCount + 1, policies == null ? 0 : policies.size()); updatedDescription = policy.getDescription() + ":updated"; @@ -179,7 +181,7 @@ public class TestServiceStore { RangerPolicy updatedPolicy = svcStore.updatePolicy(createdPolicy); assertNotNull("updatePolicy(updatedDescription) failed", updatedPolicy != null); - policies = svcStore.getAllPolicies(); + policies = svcStore.getPolicies(filter); assertEquals("updatePolicy(updatedDescription) failed", initPolicyCount + 1, policies == null ? 0 : policies.size()); updatedName = policyName + "-Renamed"; @@ -187,7 +189,7 @@ public class TestServiceStore { updatedPolicy = svcStore.updatePolicy(updatedPolicy); assertNotNull("updatePolicy(updatedName) failed", updatedPolicy); - policies = svcStore.getAllPolicies(); + policies = svcStore.getPolicies(filter); assertEquals("updatePolicy(updatedName) failed", initPolicyCount + 1, policies == null ? 0 : policies.size()); // rename the service; all the policies for this service should reflect the new service name @@ -197,7 +199,7 @@ public class TestServiceStore { assertNotNull("updateService(updatedName2) failed", updatedSvc); assertEquals("updateService(updatedName2) failed", updatedName, updatedSvc.getName()); - services = svcStore.getAllServices(); + services = svcStore.getServices(filter); assertEquals("updateService(updatedName2) failed", initServiceCount + 1, services == null ? 0 : services.size()); updatedPolicy = svcStore.getPolicy(createdPolicy.getId()); @@ -219,16 +221,28 @@ public class TestServiceStore { assertNotNull(updatedPolicies); assertEquals(0, updatedPolicies.getPolicies().size()); + filter = new SearchFilter(); + filter.setParam(SearchFilter.POLICY_NAME, policyName); + policies = svcStore.getPolicies(filter); + assertEquals("getPolicies(filter=origPolicyName) failed", 0, policies == null ? 0 : policies.size()); + filter = null; + + filter = new SearchFilter(); + filter.setParam(SearchFilter.POLICY_NAME, updatedPolicy.getName()); + policies = svcStore.getPolicies(filter); + assertEquals("getPolicies(filter=origPolicyName) failed", 1, policies == null ? 0 : policies.size()); + filter = null; + svcStore.deletePolicy(policy.getId()); - policies = svcStore.getAllPolicies(); + policies = svcStore.getPolicies(filter); assertEquals("deletePolicy() failed", initPolicyCount, policies == null ? 0 : policies.size()); svcStore.deleteService(svc.getId()); - services = svcStore.getAllServices(); + services = svcStore.getServices(filter); assertEquals("deleteService() failed", initServiceCount, services == null ? 0 : services.size()); svcStore.deleteServiceDef(sd.getId()); - sds = svcStore.getAllServiceDefs(); + sds = svcStore.getServiceDefs(filter); assertEquals("deleteServiceDef() failed", initSdCount, sds == null ? 0 : sds.size()); } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ff4a817e/plugin-common/src/test/java/org/apache/ranger/plugin/util/TestPolicyRefresher.java ---------------------------------------------------------------------- diff --git a/plugin-common/src/test/java/org/apache/ranger/plugin/util/TestPolicyRefresher.java b/plugin-common/src/test/java/org/apache/ranger/plugin/util/TestPolicyRefresher.java index a1da8c2..3dedbec 100644 --- a/plugin-common/src/test/java/org/apache/ranger/plugin/util/TestPolicyRefresher.java +++ b/plugin-common/src/test/java/org/apache/ranger/plugin/util/TestPolicyRefresher.java @@ -78,8 +78,7 @@ public class TestPolicyRefresher { } }; - refresher = new PolicyRefresher(policyEngine, svcName, svcStore); - refresher.setPollingIntervalMilliSeconds(pollingIntervalInMs); + refresher = new PolicyRefresher(policyEngine, sdName, svcName, svcStore, pollingIntervalInMs, null); refresher.start(); // create a service http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ff4a817e/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java index cfe07d0..1cfc0a8 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java @@ -39,6 +39,7 @@ import org.apache.ranger.plugin.model.RangerService; import org.apache.ranger.plugin.model.RangerServiceDef; import org.apache.ranger.plugin.store.ServiceStore; import org.apache.ranger.plugin.store.ServiceStoreFactory; +import org.apache.ranger.plugin.util.SearchFilter; import org.apache.ranger.plugin.util.ServicePolicies; import org.apache.ranger.view.VXResponse; import org.springframework.beans.factory.annotation.Autowired; @@ -196,8 +197,10 @@ public class ServiceREST { List<RangerServiceDef> ret = null; + SearchFilter filter = getSearchFilter(request); + try { - ret = svcStore.getAllServiceDefs(); + ret = svcStore.getServiceDefs(filter); } catch(Exception excp) { throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, excp.getMessage(), true); } @@ -340,8 +343,10 @@ public class ServiceREST { List<RangerService> ret = null; + SearchFilter filter = getSearchFilter(request); + try { - ret = svcStore.getAllServices(); + ret = svcStore.getServices(filter); } catch(Exception excp) { throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, excp.getMessage(), true); } @@ -506,8 +511,10 @@ public class ServiceREST { List<RangerPolicy> ret = null; + SearchFilter filter = getSearchFilter(request); + try { - ret = svcStore.getAllPolicies(); + ret = svcStore.getPolicies(filter); } catch(Exception excp) { throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, excp.getMessage(), true); } @@ -522,7 +529,7 @@ public class ServiceREST { @GET @Path("/policies/count") @Produces({ "application/json", "application/xml" }) - public Long countPolicies(@Context HttpServletRequest request) { + public Long countPolicies( @Context HttpServletRequest request) { if(LOG.isDebugEnabled()) { LOG.debug("==> ServiceREST.countPolicies():"); } @@ -554,8 +561,10 @@ public class ServiceREST { List<RangerPolicy> ret = null; + SearchFilter filter = getSearchFilter(request); + try { - ret = svcStore.getServicePolicies(serviceId); + ret = svcStore.getServicePolicies(serviceId, filter); } catch(Exception excp) { throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, excp.getMessage(), true); } @@ -581,8 +590,10 @@ public class ServiceREST { List<RangerPolicy> ret = null; + SearchFilter filter = getSearchFilter(request); + try { - ret = svcStore.getServicePolicies(serviceName); + ret = svcStore.getServicePolicies(serviceName, filter); } catch(Exception excp) { throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, excp.getMessage(), true); } @@ -624,4 +635,13 @@ public class ServiceREST { return ret; } + + + private SearchFilter getSearchFilter(HttpServletRequest request) { + SearchFilter ret = null; + + // TODO: create SearchFilter from HttpServletRequest params + + return ret; + } }
