Repository: incubator-ranger Updated Branches: refs/heads/stack f49cac435 -> ea89bb5f4
RANGER-203: 1) ServiceDefStore merged with ServiceStore 2) added SeviceStore.getServicePoliciesIfUpdated() 3) PolicyRefresher added 4) RangerBasePlugin updated to use policy-refresher 5) RangerResourceDef.type renamed to RangerResourceDef.uiType Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/50c639ab Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/50c639ab Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/50c639ab Branch: refs/heads/stack Commit: 50c639ab13eb4718466cc8dc1a75cd795de14774 Parents: f49cac4 Author: Madhan Neethiraj <[email protected]> Authored: Tue Jan 13 15:21:25 2015 -0800 Committer: Madhan Neethiraj <[email protected]> Committed: Tue Jan 13 15:21:25 2015 -0800 ---------------------------------------------------------------------- .../ranger/plugin/model/RangerService.java | 45 +- .../ranger/plugin/model/RangerServiceDef.java | 38 +- .../ranger/plugin/service/RangerBasePlugin.java | 31 +- .../ranger/plugin/store/ServiceDefStore.java | 38 -- .../plugin/store/ServiceDefStoreFactory.java | 70 --- .../ranger/plugin/store/ServiceStore.java | 32 +- .../ranger/plugin/store/file/BaseFileStore.java | 37 +- .../plugin/store/file/ServiceDefFileStore.java | 358 ------------- .../plugin/store/file/ServiceFileStore.java | 531 ++++++++++++++++--- .../ranger/plugin/util/PolicyRefresher.java | 103 ++++ .../ranger/plugin/util/ServicePolicies.java | 125 +++++ .../ranger/plugin/store/TestServiceStore.java | 63 ++- .../ranger/plugin/util/TestPolicyRefresher.java | 193 +++++++ .../org/apache/ranger/rest/ServiceREST.java | 32 +- 14 files changed, 1075 insertions(+), 621 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/50c639ab/plugin-common/src/main/java/org/apache/ranger/plugin/model/RangerService.java ---------------------------------------------------------------------- diff --git a/plugin-common/src/main/java/org/apache/ranger/plugin/model/RangerService.java b/plugin-common/src/main/java/org/apache/ranger/plugin/model/RangerService.java index ea2182a..a810a1a 100644 --- a/plugin-common/src/main/java/org/apache/ranger/plugin/model/RangerService.java +++ b/plugin-common/src/main/java/org/apache/ranger/plugin/model/RangerService.java @@ -19,6 +19,7 @@ package org.apache.ranger.plugin.model; +import java.util.Date; import java.util.HashMap; import java.util.Map; @@ -29,7 +30,6 @@ import javax.xml.bind.annotation.XmlRootElement; import org.codehaus.jackson.annotate.JsonAutoDetect; import org.codehaus.jackson.annotate.JsonIgnoreProperties; import org.codehaus.jackson.annotate.JsonAutoDetect.Visibility; -import org.codehaus.jackson.map.annotate.JsonDeserialize; import org.codehaus.jackson.map.annotate.JsonSerialize; @@ -41,11 +41,13 @@ import org.codehaus.jackson.map.annotate.JsonSerialize; public class RangerService extends RangerBaseModelObject implements java.io.Serializable { private static final long serialVersionUID = 1L; - private String type = null; - private String name = null; - private String description = null; - private Boolean isEnabled = null; - private Map<String, String> configs = null; + private String type = null; + private String name = null; + private String description = null; + private Boolean isEnabled = null; + private Map<String, String> configs = null; + private Long policyVersion = null; + private Date policyUpdateTime = null; /** @@ -166,6 +168,34 @@ public class RangerService extends RangerBaseModelObject implements java.io.Seri } } + /** + * @return the policyVersion + */ + public Long getPolicyVersion() { + return policyVersion; + } + + /** + * @param policyVersion the policyVersion to set + */ + public void setPolicyVersion(Long policyVersion) { + this.policyVersion = policyVersion; + } + + /** + * @return the policyUpdateTime + */ + public Date getPolicyUpdateTime() { + return policyUpdateTime; + } + + /** + * @param policyUpdateTime the policyUpdateTime to set + */ + public void setPolicyUpdateTime(Date policyUpdateTime) { + this.policyUpdateTime = policyUpdateTime; + } + @Override public String toString( ) { StringBuilder sb = new StringBuilder(); @@ -192,6 +222,9 @@ public class RangerService extends RangerBaseModelObject implements java.io.Seri } sb.append("} "); + sb.append("policyVersion={").append(policyVersion).append("} "); + sb.append("policyUpdateTime={").append(policyUpdateTime).append("} "); + sb.append("}"); return sb; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/50c639ab/plugin-common/src/main/java/org/apache/ranger/plugin/model/RangerServiceDef.java ---------------------------------------------------------------------- diff --git a/plugin-common/src/main/java/org/apache/ranger/plugin/model/RangerServiceDef.java b/plugin-common/src/main/java/org/apache/ranger/plugin/model/RangerServiceDef.java index f1fe7a8..e701762 100644 --- a/plugin-common/src/main/java/org/apache/ranger/plugin/model/RangerServiceDef.java +++ b/plugin-common/src/main/java/org/apache/ranger/plugin/model/RangerServiceDef.java @@ -635,7 +635,7 @@ public class RangerServiceDef extends RangerBaseModelObject implements java.io.S } /** - * @param type the type to set + * @param uiType the type to set */ public void setSubType(String subType) { this.subType = subType; @@ -756,7 +756,6 @@ public class RangerServiceDef extends RangerBaseModelObject implements java.io.S private static final long serialVersionUID = 1L; private String name = null; - private String type = null; private Integer level = null; private String parent = null; private Boolean mandatory = null; @@ -767,6 +766,7 @@ public class RangerServiceDef extends RangerBaseModelObject implements java.io.S private String matcherOptions = null; private String label = null; private String description = null; + private String uiType = null; private String rbKeyLabel = null; private String rbKeyDescription = null; @@ -775,9 +775,8 @@ public class RangerServiceDef extends RangerBaseModelObject implements java.io.S this(null, null, null, null, null, null, null, null, null, null, null, null, null, null); } - public RangerResourceDef(String name, String type, Integer level, String parent, Boolean mandatory, Boolean lookupSupported, Boolean recursiveSupported, Boolean excludesSupported, String matcher, String matcherOptions, String label, String description, String rbKeyLabel, String rbKeyDescription) { + public RangerResourceDef(String name, Integer level, String parent, Boolean mandatory, Boolean lookupSupported, Boolean recursiveSupported, Boolean excludesSupported, String matcher, String matcherOptions, String label, String description, String uiType, String rbKeyLabel, String rbKeyDescription) { setName(name); - setType(type); setLevel(level); setParent(parent); setMandatory(mandatory); @@ -788,6 +787,7 @@ public class RangerServiceDef extends RangerBaseModelObject implements java.io.S setMatcher(matcherOptions); setLabel(label); setDescription(description); + setUIType(uiType); setRbKeyLabel(rbKeyLabel); setRbKeyDescription(rbKeyDescription); } @@ -807,20 +807,6 @@ public class RangerServiceDef extends RangerBaseModelObject implements java.io.S } /** - * @return the type - */ - public String getType() { - return type; - } - - /** - * @param type the type to set - */ - public void setType(String type) { - this.type = type; - } - - /** * @return the level */ public Integer getLevel() { @@ -961,6 +947,20 @@ public class RangerServiceDef extends RangerBaseModelObject implements java.io.S } /** + * @return the uiType + */ + public String getUIType() { + return uiType; + } + + /** + * @param uiType the uiType to set + */ + public void setUIType(String uiType) { + this.uiType = uiType; + } + + /** * @return the rbKeyLabel */ public String getRbKeyLabel() { @@ -1000,7 +1000,6 @@ public class RangerServiceDef extends RangerBaseModelObject implements java.io.S public StringBuilder toString(StringBuilder sb) { sb.append("RangerResourceDef={"); sb.append("name={").append(name).append("} "); - sb.append("type={").append(type).append("} "); sb.append("level={").append(level).append("} "); sb.append("parent={").append(parent).append("} "); sb.append("mandatory={").append(mandatory).append("} "); @@ -1011,6 +1010,7 @@ public class RangerServiceDef extends RangerBaseModelObject implements java.io.S sb.append("matcherOptions={").append(matcherOptions).append("} "); sb.append("label={").append(label).append("} "); sb.append("description={").append(description).append("} "); + sb.append("uiType={").append(uiType).append("} "); sb.append("rbKeyLabel={").append(rbKeyLabel).append("} "); sb.append("rbKeyDescription={").append(rbKeyDescription).append("} "); sb.append("}"); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/50c639ab/plugin-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java ---------------------------------------------------------------------- diff --git a/plugin-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java b/plugin-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java index 29c1082..d27733b 100644 --- a/plugin-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java +++ b/plugin-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java @@ -20,26 +20,27 @@ package org.apache.ranger.plugin.service; import org.apache.ranger.plugin.policyengine.RangerPolicyEngine; +import org.apache.ranger.plugin.store.ServiceStore; +import org.apache.ranger.plugin.store.ServiceStoreFactory; +import org.apache.ranger.plugin.util.PolicyRefresher; public abstract class RangerBasePlugin { - private boolean initDone = false; + private boolean initDone = false; + private PolicyRefresher refresher = null; - public boolean init() { + + public boolean init(RangerPolicyEngine policyEngine) { if(!initDone) { synchronized(this) { if(! initDone) { - /* TODO: - loadConfig(); // to get serviceName, policy download URL, local cache file details, etc - - initAuditFramework(); - - loadLocallyCachedPolicies(); + String serviceName = System.getProperty("ranger.plugin.service.name", "hbasedev"); // TODO: read from configuration - getPolicyEngine().setPolicies(serviceDef, policies); + ServiceStore serviceStore = ServiceStoreFactory.instance().getServiceStore(); - setupPolicyRefresher(); // to poll for policy updates - */ + refresher = new PolicyRefresher(policyEngine, serviceName, serviceStore); + + refresher.start(); initDone = true; } @@ -50,8 +51,10 @@ public abstract class RangerBasePlugin { } public void cleanup() { - // TODO: + PolicyRefresher refresher = this.refresher; + + if(refresher != null) { + refresher.stopRefresher(); + } } - - public abstract RangerPolicyEngine getPolicyEngine(); } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/50c639ab/plugin-common/src/main/java/org/apache/ranger/plugin/store/ServiceDefStore.java ---------------------------------------------------------------------- diff --git a/plugin-common/src/main/java/org/apache/ranger/plugin/store/ServiceDefStore.java b/plugin-common/src/main/java/org/apache/ranger/plugin/store/ServiceDefStore.java deleted file mode 100644 index 5489031..0000000 --- a/plugin-common/src/main/java/org/apache/ranger/plugin/store/ServiceDefStore.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.ranger.plugin.store; - -import java.util.List; - -import org.apache.ranger.plugin.model.RangerServiceDef; - -public interface ServiceDefStore { - RangerServiceDef create(RangerServiceDef serviceDef) throws Exception; - - RangerServiceDef update(RangerServiceDef serviceDef) throws Exception; - - void delete(Long id) throws Exception; - - RangerServiceDef get(Long id) throws Exception; - - RangerServiceDef getByName(String name) throws Exception; - - List<RangerServiceDef> getAll() throws Exception; -} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/50c639ab/plugin-common/src/main/java/org/apache/ranger/plugin/store/ServiceDefStoreFactory.java ---------------------------------------------------------------------- diff --git a/plugin-common/src/main/java/org/apache/ranger/plugin/store/ServiceDefStoreFactory.java b/plugin-common/src/main/java/org/apache/ranger/plugin/store/ServiceDefStoreFactory.java deleted file mode 100644 index 3192efc..0000000 --- a/plugin-common/src/main/java/org/apache/ranger/plugin/store/ServiceDefStoreFactory.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.ranger.plugin.store; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.ranger.plugin.store.file.ServiceDefFileStore; - - -public class ServiceDefStoreFactory { - private static final Log LOG = LogFactory.getLog(ServiceDefStoreFactory.class); - - private static ServiceDefStoreFactory sInstance = null; - - private ServiceDefStore serviceDefStore = null; - - - public static ServiceDefStoreFactory instance() { - if(sInstance == null) { - sInstance = new ServiceDefStoreFactory(); - } - - return sInstance; - } - - public ServiceDefStore getServiceDefStore() { - return serviceDefStore; - } - - private ServiceDefStoreFactory() { - if(LOG.isDebugEnabled()) { - LOG.debug("==> ServiceDefStoreFactory.ServiceDefStoreFactory()"); - } - - init(); - - if(LOG.isDebugEnabled()) { - LOG.debug("<== ServiceDefStoreFactory.ServiceDefStoreFactory()"); - } - } - - private void init() { - if(LOG.isDebugEnabled()) { - LOG.debug("==> ServiceDefStoreFactory.init()"); - } - - serviceDefStore = new ServiceDefFileStore(); // TODO: configurable store implementation - - if(LOG.isDebugEnabled()) { - LOG.debug("<== ServiceDefStoreFactory.init()"); - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/50c639ab/plugin-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java ---------------------------------------------------------------------- diff --git a/plugin-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java b/plugin-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java index c5b0724..570f20f 100644 --- a/plugin-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java +++ b/plugin-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java @@ -19,23 +19,39 @@ package org.apache.ranger.plugin.store; +import java.util.ArrayList; import java.util.List; import org.apache.ranger.plugin.model.RangerPolicy; import org.apache.ranger.plugin.model.RangerService; +import org.apache.ranger.plugin.model.RangerServiceDef; +import org.apache.ranger.plugin.util.ServicePolicies; public interface ServiceStore { - RangerService create(RangerService service) throws Exception; + RangerServiceDef createServiceDef(RangerServiceDef serviceDef) throws Exception; - RangerService update(RangerService service) throws Exception; + RangerServiceDef updateServiceDef(RangerServiceDef serviceDef) throws Exception; - void delete(Long id) throws Exception; + void deleteServiceDef(Long id) throws Exception; - RangerService get(Long id) throws Exception; + RangerServiceDef getServiceDef(Long id) throws Exception; - RangerService getByName(String name) throws Exception; + RangerServiceDef getServiceDefByName(String name) throws Exception; - List<RangerService> getAll() throws Exception; + List<RangerServiceDef> getAllServiceDefs() throws Exception; + + + RangerService createService(RangerService service) throws Exception; + + RangerService updateService(RangerService service) throws Exception; + + void deleteService(Long id) throws Exception; + + RangerService getService(Long id) throws Exception; + + RangerService getServiceByName(String name) throws Exception; + + List<RangerService> getAllServices() throws Exception; RangerPolicy createPolicy(RangerPolicy policy) throws Exception; @@ -48,9 +64,11 @@ public interface ServiceStore { RangerPolicy getPolicyByName(String serviceName, String policyName) throws Exception; + List<RangerPolicy> getAllPolicies() throws Exception; + List<RangerPolicy> getServicePolicies(String serviceName) throws Exception; List<RangerPolicy> getServicePolicies(Long serviceId) throws Exception; - List<RangerPolicy> getAllPolicies() throws Exception; + ServicePolicies getServicePoliciesIfUpdated(String serviceName, Long lastKnownVersion) throws Exception; } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/50c639ab/plugin-common/src/main/java/org/apache/ranger/plugin/store/file/BaseFileStore.java ---------------------------------------------------------------------- diff --git a/plugin-common/src/main/java/org/apache/ranger/plugin/store/file/BaseFileStore.java b/plugin-common/src/main/java/org/apache/ranger/plugin/store/file/BaseFileStore.java index da20ba2..8717495 100644 --- a/plugin-common/src/main/java/org/apache/ranger/plugin/store/file/BaseFileStore.java +++ b/plugin-common/src/main/java/org/apache/ranger/plugin/store/file/BaseFileStore.java @@ -38,6 +38,9 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; import org.apache.ranger.plugin.model.RangerBaseModelObject; +import org.apache.ranger.plugin.model.RangerPolicy; +import org.apache.ranger.plugin.model.RangerService; +import org.apache.ranger.plugin.model.RangerServiceDef; import com.google.gson.Gson; import com.google.gson.GsonBuilder; @@ -55,7 +58,7 @@ public class BaseFileStore { protected void init() { - dataDir = System.getProperty("org.apache.ranger.datastore.dir", "/etc/ranger/data"); // TODO: read from configuration + dataDir = System.getProperty("ranger.policystore.file.dir", "/etc/ranger/data"); // TODO: read from configuration try { gsonBuilder = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z").setPrettyPrinting().create(); @@ -253,6 +256,38 @@ public class BaseFileStore { return ret; } + protected RangerServiceDef saveToFile(RangerServiceDef serviceDef, boolean overWrite) throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> BaseFileStore.saveToFile(" + serviceDef + "," + overWrite + ")"); + } + + Path filePath = new Path(getServiceDefFile(serviceDef.getId())); + + RangerServiceDef ret = saveToFile(serviceDef, filePath, overWrite); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== BaseFileStore.saveToFile(" + serviceDef + "," + overWrite + "): "); + } + + return ret; + } + + protected RangerService saveToFile(RangerService service, boolean overWrite) throws Exception { + Path filePath = new Path(getServiceFile(service.getId())); + + RangerService ret = saveToFile(service, filePath, overWrite); + + return ret; + } + + protected RangerPolicy saveToFile(RangerPolicy policy, long serviceId, boolean overWrite) throws Exception { + Path filePath = new Path(getPolicyFile(serviceId, policy.getId())); + + RangerPolicy ret = saveToFile(policy, filePath, overWrite); + + return ret; + } + protected long getMaxId(List<? extends RangerBaseModelObject> objs) { long ret = -1; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/50c639ab/plugin-common/src/main/java/org/apache/ranger/plugin/store/file/ServiceDefFileStore.java ---------------------------------------------------------------------- diff --git a/plugin-common/src/main/java/org/apache/ranger/plugin/store/file/ServiceDefFileStore.java b/plugin-common/src/main/java/org/apache/ranger/plugin/store/file/ServiceDefFileStore.java deleted file mode 100644 index 5f6fabf..0000000 --- a/plugin-common/src/main/java/org/apache/ranger/plugin/store/file/ServiceDefFileStore.java +++ /dev/null @@ -1,358 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.ranger.plugin.store.file; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.apache.commons.lang.StringUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.fs.Path; -import org.apache.ranger.plugin.model.RangerServiceDef; -import org.apache.ranger.plugin.store.ServiceDefStore; - - -public class ServiceDefFileStore extends BaseFileStore implements ServiceDefStore { - private static final Log LOG = LogFactory.getLog(ServiceDefFileStore.class); - - private List<RangerServiceDef> serviceDefs = null; - private long nextServiceDefId = 0; - - static Map<String, Long> legacyServiceTypes = new HashMap<String, Long>(); - - static { - legacyServiceTypes.put("hdfs", new Long(1)); - legacyServiceTypes.put("hbase", new Long(2)); - legacyServiceTypes.put("hive", new Long(3)); - legacyServiceTypes.put("knox", new Long(5)); - legacyServiceTypes.put("storm", new Long(6)); - } - - public ServiceDefFileStore() { - if(LOG.isDebugEnabled()) { - LOG.debug("==> ServiceDefFileStore.ServiceDefManagerFile()"); - } - - init(); - - if(LOG.isDebugEnabled()) { - LOG.debug("<== ServiceDefFileStore.ServiceDefManagerFile()"); - } - } - - @Override - public RangerServiceDef create(RangerServiceDef serviceDef) throws Exception { - if(LOG.isDebugEnabled()) { - LOG.debug("==> ServiceDefFileStore.create(" + serviceDef + ")"); - } - - RangerServiceDef existing = findServiceDefByName(serviceDef.getName()); - - if(existing != null) { - throw new Exception(serviceDef.getName() + ": service-def already exists (id=" + existing.getId() + ")"); - } - - RangerServiceDef ret = null; - - try { - preCreate(serviceDef); - - serviceDef.setId(nextServiceDefId++); - - Path filePath = new Path(getServiceDefFile(serviceDef.getId())); - - ret = saveToFile(serviceDef, filePath, false); - - addServiceDef(ret); - - postCreate(ret); - } catch(Exception excp) { - LOG.warn("ServiceDefFileStore.create(): failed to save service-def '" + serviceDef.getName() + "'", excp); - - throw new Exception("failed to save service-def '" + serviceDef.getName() + "'", excp); - } - - if(LOG.isDebugEnabled()) { - LOG.debug("<== ServiceDefFileStore.create(" + serviceDef + ")"); - } - - return ret; - } - - @Override - public RangerServiceDef update(RangerServiceDef serviceDef) throws Exception { - if(LOG.isDebugEnabled()) { - LOG.debug("==> ServiceDefFileStore.update(" + serviceDef + ")"); - } - - RangerServiceDef existing = findServiceDefById(serviceDef.getId()); - - if(existing == null) { - throw new Exception(serviceDef.getId() + ": service-def does not exist"); - } - - if(isLegacyServiceType(existing)) { - String msg = existing.getName() + ": is an in-built service-def. Update not allowed"; - - LOG.warn(msg); - - throw new Exception(msg); - } - - String existingName = existing.getName(); - - boolean renamed = !StringUtils.equalsIgnoreCase(serviceDef.getName(), existingName); - - // renaming service-def would require updating services that refer to this service-def - if(renamed) { - LOG.warn("ServiceDefFileStore.update(): service-def renaming not supported. " + existingName + " ==> " + serviceDef.getName()); - - throw new Exception("service-def renaming not supported. " + existingName + " ==> " + serviceDef.getName()); - } - - RangerServiceDef ret = null; - - try { - existing.updateFrom(serviceDef); - - preUpdate(existing); - - Path filePath = new Path(getServiceDefFile(existing.getId())); - - ret = saveToFile(existing, filePath, true); - - postUpdate(ret); - } catch(Exception excp) { - LOG.warn("ServiceDefFileStore.update(): failed to save service-def '" + existing.getName() + "'", excp); - - throw new Exception("failed to save service-def '" + existing.getName() + "'", excp); - } - - if(LOG.isDebugEnabled()) { - LOG.debug("<== ServiceDefFileStore.update(" + serviceDef + "): " + ret); - } - - return ret; - } - - @Override - public void delete(Long id) throws Exception { - if(LOG.isDebugEnabled()) { - LOG.debug("==> ServiceDefFileStore.delete(" + id + ")"); - } - - RangerServiceDef existing = findServiceDefById(id); - - if(existing == null) { - throw new Exception("service-def does not exist. id=" + id); - } - - if(isLegacyServiceType(existing)) { - String msg = existing.getName() + ": is an in-built service-def. Update not allowed"; - - LOG.warn(msg); - - throw new Exception(msg); - } - - // TODO: deleting service-def would require deleting services that refer to this service-def - - try { - preDelete(existing); - - Path filePath = new Path(getServiceDefFile(id)); - - deleteFile(filePath); - - removeServiceDef(existing); - - postDelete(existing); - } catch(Exception excp) { - throw new Exception("failed to delete service-def. id=" + id + "; name=" + existing.getName(), excp); - } - - if(LOG.isDebugEnabled()) { - LOG.debug("<== ServiceDefFileStore.delete(" + id + ")"); - } - } - - @Override - public RangerServiceDef get(Long id) throws Exception { - if(LOG.isDebugEnabled()) { - LOG.debug("==> ServiceDefFileStore.get(" + id + ")"); - } - - RangerServiceDef ret = findServiceDefById(id); - - if(LOG.isDebugEnabled()) { - LOG.debug("<== ServiceDefFileStore.get(" + id + "): " + ret); - } - - return ret; - } - - @Override - public RangerServiceDef getByName(String name) throws Exception { - if(LOG.isDebugEnabled()) { - LOG.debug("==> ServiceDefFileStore.getByName(" + name + ")"); - } - - RangerServiceDef ret = findServiceDefByName(name); - - if(LOG.isDebugEnabled()) { - LOG.debug("<== ServiceDefFileStore.getByName(" + name + "): " + ret); - } - - return ret; - } - - @Override - public List<RangerServiceDef> getAll() throws Exception { - if(LOG.isDebugEnabled()) { - LOG.debug("==> ServiceDefFileStore.getAll()"); - } - - List<RangerServiceDef> ret = serviceDefs; - - if(LOG.isDebugEnabled()) { - LOG.debug("<== ServiceDefFileStore.getAll(): count=" + (ret == null ? 0 : ret.size())); - } - - return ret; - } - - @Override - protected void init() { - if(LOG.isDebugEnabled()) { - LOG.debug("==> ServiceDefFileStore.init()"); - } - - super.init(); - - try { - serviceDefs = new ArrayList<RangerServiceDef>(); - - // load definitions for legacy services from embedded resources - String[] legacyServiceDefResources = { - "/service-defs/ranger-servicedef-hdfs.json", - "/service-defs/ranger-servicedef-hive.json", - "/service-defs/ranger-servicedef-hbase.json", - "/service-defs/ranger-servicedef-knox.json", - "/service-defs/ranger-servicedef-storm.json", - }; - - for(String resource : legacyServiceDefResources) { - RangerServiceDef sd = loadFromResource(resource, RangerServiceDef.class); - - if(sd != null) { - serviceDefs.add(sd); - } - } - nextServiceDefId = getMaxId(serviceDefs) + 1; - - // load service definitions from file system - List<RangerServiceDef> sds = loadFromDir(new Path(getDataDir()), FILE_PREFIX_SERVICE_DEF, RangerServiceDef.class); - - if(sds != null) { - for(RangerServiceDef sd : sds) { - if(sd != null) { - if(isLegacyServiceType(sd)) { - LOG.warn("Found in-built service-def '" + sd.getName() + "' under " + getDataDir() + ". Ignorning"); - - continue; - } - - RangerServiceDef existingSd = findServiceDefByName(sd.getName()); - - if(existingSd != null) { - removeServiceDef(existingSd); - } - - existingSd = findServiceDefById(sd.getId()); - - if(existingSd != null) { - removeServiceDef(existingSd); - } - - serviceDefs.add(sd); - } - } - } - nextServiceDefId = getMaxId(serviceDefs) + 1; - } catch(Exception excp) { - LOG.error("ServiceDefFileStore.init(): failed to read service-defs", excp); - } - - if(LOG.isDebugEnabled()) { - LOG.debug("<== ServiceDefFileStore.init()"); - } - } - - private RangerServiceDef findServiceDefById(long id) { - RangerServiceDef ret = null; - - for(RangerServiceDef sd : serviceDefs) { - if(sd != null && sd.getId() != null && sd.getId().longValue() == id) { - ret = sd; - - break; - } - } - - return ret; - } - - private RangerServiceDef findServiceDefByName(String sdName) { - RangerServiceDef ret = null; - - for(RangerServiceDef sd : serviceDefs) { - if(sd != null && StringUtils.equalsIgnoreCase(sd.getName(), sdName)) { - ret = sd; - - break; - } - } - - return ret; - } - - private void addServiceDef(RangerServiceDef sd) { - serviceDefs.add(sd); - } - - private void removeServiceDef(RangerServiceDef sd) { - serviceDefs.remove(sd); - } - - private boolean isLegacyServiceType(RangerServiceDef sd) { - return sd == null ? false : (isLegacyServiceType(sd.getName()) || isLegacyServiceType(sd.getId())); - } - - private boolean isLegacyServiceType(String name) { - return name == null ? false : legacyServiceTypes.containsKey(name); - } - - private boolean isLegacyServiceType(Long id) { - return id == null ? false : legacyServiceTypes.containsValue(id); - } -} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/50c639ab/plugin-common/src/main/java/org/apache/ranger/plugin/store/file/ServiceFileStore.java ---------------------------------------------------------------------- diff --git a/plugin-common/src/main/java/org/apache/ranger/plugin/store/file/ServiceFileStore.java b/plugin-common/src/main/java/org/apache/ranger/plugin/store/file/ServiceFileStore.java index 85524db..ed6d168 100644 --- a/plugin-common/src/main/java/org/apache/ranger/plugin/store/file/ServiceFileStore.java +++ b/plugin-common/src/main/java/org/apache/ranger/plugin/store/file/ServiceFileStore.java @@ -20,7 +20,10 @@ package org.apache.ranger.plugin.store.file; import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; import java.util.List; +import java.util.Map; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; @@ -28,34 +31,228 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; import org.apache.ranger.plugin.model.RangerPolicy; import org.apache.ranger.plugin.model.RangerService; +import org.apache.ranger.plugin.model.RangerServiceDef; import org.apache.ranger.plugin.store.ServiceStore; +import org.apache.ranger.plugin.util.ServicePolicies; public class ServiceFileStore extends BaseFileStore implements ServiceStore { private static final Log LOG = LogFactory.getLog(ServiceFileStore.class); - private long nextServiceId = 0; - private long nextPolicyId = 0; + private List<RangerServiceDef> serviceDefs = null; + + private long nextServiceDefId = 0; + private long nextServiceId = 0; + private long nextPolicyId = 0; + + static Map<String, Long> legacyServiceDefs = new HashMap<String, Long>(); + + static { + legacyServiceDefs.put("hdfs", new Long(1)); + legacyServiceDefs.put("hbase", new Long(2)); + legacyServiceDefs.put("hive", new Long(3)); + legacyServiceDefs.put("knox", new Long(5)); + legacyServiceDefs.put("storm", new Long(6)); + } public ServiceFileStore() { if(LOG.isDebugEnabled()) { - LOG.debug("==> ServiceFileStore.ServiceManagerFile()"); + LOG.debug("==> ServiceFileStore.ServiceFileStore()"); } init(); if(LOG.isDebugEnabled()) { - LOG.debug("<== ServiceFileStore.ServiceManagerFile()"); + LOG.debug("<== ServiceFileStore.ServiceFileStore()"); + } + } + + + @Override + public RangerServiceDef createServiceDef(RangerServiceDef serviceDef) throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> ServiceDefFileStore.createServiceDef(" + serviceDef + ")"); + } + + RangerServiceDef existing = findServiceDefByName(serviceDef.getName()); + + if(existing != null) { + throw new Exception(serviceDef.getName() + ": service-def already exists (id=" + existing.getId() + ")"); + } + + RangerServiceDef ret = null; + + try { + preCreate(serviceDef); + + serviceDef.setId(nextServiceDefId++); + + ret = saveToFile(serviceDef, false); + + addServiceDef(ret); + + postCreate(ret); + } catch(Exception excp) { + LOG.warn("ServiceDefFileStore.createServiceDef(): failed to save service-def '" + serviceDef.getName() + "'", excp); + + throw new Exception("failed to save service-def '" + serviceDef.getName() + "'", excp); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== ServiceDefFileStore.createServiceDef(" + serviceDef + ")"); } + + return ret; } @Override - public RangerService create(RangerService service) throws Exception { + public RangerServiceDef updateServiceDef(RangerServiceDef serviceDef) throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> ServiceDefFileStore.updateServiceDef(" + serviceDef + ")"); + } + + RangerServiceDef existing = findServiceDefById(serviceDef.getId()); + + if(existing == null) { + throw new Exception(serviceDef.getId() + ": service-def does not exist"); + } + + if(isLegacyServiceDef(existing)) { + String msg = existing.getName() + ": is an in-built service-def. Update not allowed"; + + LOG.warn(msg); + + throw new Exception(msg); + } + + String existingName = existing.getName(); + + boolean renamed = !StringUtils.equalsIgnoreCase(serviceDef.getName(), existingName); + + // renaming service-def would require updating services that refer to this service-def + if(renamed) { + LOG.warn("ServiceDefFileStore.updateServiceDef(): service-def renaming not supported. " + existingName + " ==> " + serviceDef.getName()); + + throw new Exception("service-def renaming not supported. " + existingName + " ==> " + serviceDef.getName()); + } + + RangerServiceDef ret = null; + + try { + existing.updateFrom(serviceDef); + + preUpdate(existing); + + ret = saveToFile(existing, true); + + postUpdate(ret); + } catch(Exception excp) { + LOG.warn("ServiceDefFileStore.updateServiceDef(): failed to save service-def '" + existing.getName() + "'", excp); + + throw new Exception("failed to save service-def '" + existing.getName() + "'", excp); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== ServiceDefFileStore.updateServiceDef(" + serviceDef + "): " + ret); + } + + return ret; + } + + @Override + public void deleteServiceDef(Long id) throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> ServiceDefFileStore.deleteServiceDef(" + id + ")"); + } + + RangerServiceDef existing = findServiceDefById(id); + + if(existing == null) { + throw new Exception("service-def does not exist. id=" + id); + } + + if(isLegacyServiceDef(existing)) { + String msg = existing.getName() + ": is an in-built service-def. Update not allowed"; + + LOG.warn(msg); + + throw new Exception(msg); + } + + // TODO: deleting service-def would require deleting services that refer to this service-def + + try { + preDelete(existing); + + Path filePath = new Path(getServiceDefFile(id)); + + deleteFile(filePath); + + removeServiceDef(existing); + + postDelete(existing); + } catch(Exception excp) { + throw new Exception("failed to delete service-def. id=" + id + "; name=" + existing.getName(), excp); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== ServiceDefFileStore.deleteServiceDef(" + id + ")"); + } + } + + @Override + public RangerServiceDef getServiceDef(Long id) throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> ServiceDefFileStore.getServiceDef(" + id + ")"); + } + + RangerServiceDef ret = findServiceDefById(id); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== ServiceDefFileStore.getServiceDef(" + id + "): " + ret); + } + + return ret; + } + + @Override + public RangerServiceDef getServiceDefByName(String name) throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> ServiceDefFileStore.getServiceDefByName(" + name + ")"); + } + + RangerServiceDef ret = findServiceDefByName(name); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== ServiceDefFileStore.getServiceDefByName(" + name + "): " + ret); + } + + return ret; + } + + @Override + public List<RangerServiceDef> getAllServiceDefs() throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> ServiceDefFileStore.getAllServiceDefs()"); + } + + List<RangerServiceDef> ret = serviceDefs; + + if(LOG.isDebugEnabled()) { + LOG.debug("<== ServiceDefFileStore.getAllServiceDefs(): count=" + (ret == null ? 0 : ret.size())); + } + + return ret; + } + + + @Override + public RangerService createService(RangerService service) throws Exception { if(LOG.isDebugEnabled()) { - LOG.debug("==> ServiceFileStore.create(" + service + ")"); + LOG.debug("==> ServiceFileStore.createService(" + service + ")"); } - RangerService existing = getByName(service.getName()); + RangerService existing = getServiceByName(service.getName()); if(existing != null) { throw new Exception("service already exists - '" + service.getName() + "'. ID=" + existing.getId()); @@ -68,9 +265,7 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { service.setId(nextServiceId++); - Path filePath = new Path(getServiceFile(service.getId())); - - ret = saveToFile(service, filePath, false); + ret = saveToFile(service, false); postCreate(service); } catch(Exception excp) { @@ -78,19 +273,19 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { } if(LOG.isDebugEnabled()) { - LOG.debug("<== ServiceFileStore.create(" + service + "): " + ret); + LOG.debug("<== ServiceFileStore.createService(" + service + "): " + ret); } return ret; } @Override - public RangerService update(RangerService service) throws Exception { + public RangerService updateService(RangerService service) throws Exception { if(LOG.isDebugEnabled()) { - LOG.debug("==> ServiceFileStore.update(" + service + ")"); + LOG.debug("==> ServiceFileStore.updateService(" + service + ")"); } - RangerService existing = get(service.getId()); + RangerService existing = getService(service.getId()); if(existing == null) { throw new Exception("no service exists with ID=" + service.getId()); @@ -101,7 +296,7 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { boolean renamed = !StringUtils.equalsIgnoreCase(service.getName(), existingName); if(renamed) { - RangerService newNameService = getByName(service.getName()); + RangerService newNameService = getServiceByName(service.getName()); if(newNameService != null) { throw new Exception("another service already exists with name '" + service.getName() + "'. ID=" + newNameService.getId()); @@ -115,9 +310,7 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { preUpdate(existing); - Path filePath = new Path(getServiceFile(existing.getId())); - - ret = saveToFile(existing, filePath, true); + ret = saveToFile(existing, true); postUpdate(ret); @@ -129,19 +322,19 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { } if(LOG.isDebugEnabled()) { - LOG.debug("<== ServiceFileStore.update(" + service + "): " + ret); + LOG.debug("<== ServiceFileStore.updateService(" + service + "): " + ret); } return ret; } @Override - public void delete(Long id) throws Exception { + public void deleteService(Long id) throws Exception { if(LOG.isDebugEnabled()) { - LOG.debug("==> ServiceFileStore.delete(" + id + ")"); + LOG.debug("==> ServiceFileStore.deleteService(" + id + ")"); } - RangerService existing = get(id); + RangerService existing = getService(id); if(existing == null) { throw new Exception("no service exists with ID=" + id); @@ -162,14 +355,14 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { } if(LOG.isDebugEnabled()) { - LOG.debug("<== ServiceFileStore.delete(" + id + ")"); + LOG.debug("<== ServiceFileStore.deleteService(" + id + ")"); } } @Override - public RangerService get(Long id) throws Exception { + public RangerService getService(Long id) throws Exception { if(LOG.isDebugEnabled()) { - LOG.debug("==> ServiceFileStore.get(" + id + ")"); + LOG.debug("==> ServiceFileStore.getService(" + id + ")"); } RangerService ret = null; @@ -179,26 +372,26 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { ret = loadFromFile(filePath, RangerService.class); } catch(Exception excp) { - LOG.error("ServiceFileStore.get(" + id + "): failed to read service", excp); + LOG.error("ServiceFileStore.getService(" + id + "): failed to read service", excp); } if(LOG.isDebugEnabled()) { - LOG.debug("<== ServiceFileStore.get(" + id + "): " + ret); + LOG.debug("<== ServiceFileStore.getService(" + id + "): " + ret); } return ret; } @Override - public RangerService getByName(String name) throws Exception { + public RangerService getServiceByName(String name) throws Exception { if(LOG.isDebugEnabled()) { - LOG.debug("==> ServiceFileStore.getByName(" + name + ")"); + LOG.debug("==> ServiceFileStore.getServiceByName(" + name + ")"); } RangerService ret = null; try { - List<RangerService> services = getAll(); + List<RangerService> services = getAllServices(); if(services != null) { for(RangerService service : services) { @@ -210,20 +403,20 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { } } } catch(Exception excp) { - LOG.error("ServiceFileStore.getByName(" + name + "): failed to read service", excp); + LOG.error("ServiceFileStore.getServiceByName(" + name + "): failed to read service", excp); } if(LOG.isDebugEnabled()) { - LOG.debug("<== ServiceFileStore.getByName(" + name + "): " + ret); + LOG.debug("<== ServiceFileStore.getServiceByName(" + name + "): " + ret); } return ret; } @Override - public List<RangerService> getAll() throws Exception { + public List<RangerService> getAllServices() throws Exception { if(LOG.isDebugEnabled()) { - LOG.debug("==> ServiceFileStore.getAll()"); + LOG.debug("==> ServiceFileStore.getAllServices()"); } List<RangerService> ret = null; @@ -231,11 +424,11 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { try { ret = loadFromDir(new Path(getDataDir()), FILE_PREFIX_SERVICE, RangerService.class); } catch(Exception excp) { - LOG.error("ServiceFileStore.getAll(): failed to read services", excp); + LOG.error("ServiceFileStore.getAllServices(): failed to read services", excp); } if(LOG.isDebugEnabled()) { - LOG.debug("<== ServiceFileStore.getAll(): count=" + (ret == null ? 0 : ret.size())); + LOG.debug("<== ServiceFileStore.getAllServices(): count=" + (ret == null ? 0 : ret.size())); } return ret; @@ -247,7 +440,7 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { LOG.debug("==> ServiceFileStore.createPolicy(" + policy + ")"); } - RangerService service = getByName(policy.getService()); + RangerService service = getServiceByName(policy.getService()); if(service == null) { throw new Exception("service does not exist - name=" + policy.getService()); @@ -266,9 +459,9 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { policy.setId(nextPolicyId++); - Path filePath = new Path(getPolicyFile(service.getId(), policy.getId())); + ret = saveToFile(policy, service.getId(), false); - ret = saveToFile(policy, filePath, false); + handlePolicyUpdate(service); postCreate(ret); } catch(Exception excp) { @@ -294,7 +487,7 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { throw new Exception("no policy exists with ID=" + policy.getId()); } - RangerService service = getByName(policy.getService()); + RangerService service = getServiceByName(policy.getService()); if(service == null) { throw new Exception("service does not exist - name=" + policy.getService()); @@ -321,9 +514,9 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { preUpdate(existing); - Path filePath = new Path(getPolicyFile(service.getId(), existing.getId())); + ret = saveToFile(existing, service.getId(), true); - ret = saveToFile(existing, filePath, true); + handlePolicyUpdate(service); postUpdate(ret); } catch(Exception excp) { @@ -349,7 +542,7 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { throw new Exception("no policy exists with ID=" + id); } - RangerService service = getByName(existing.getService()); + RangerService service = getServiceByName(existing.getService()); if(service == null) { throw new Exception("service does not exist - name='" + existing.getService()); @@ -362,6 +555,8 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { deleteFile(filePath); + handlePolicyUpdate(service); + postDelete(existing); } catch(Exception excp) { throw new Exception(existing.getId() + ": failed to delete policy", excp); @@ -409,7 +604,7 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { LOG.debug("==> ServiceFileStore.getPolicyByName(" + serviceName + ", " + policyName + ")"); } - RangerService service = getByName(serviceName); + RangerService service = getServiceByName(serviceName); if(service == null) { throw new Exception("service does not exist - name='" + serviceName); @@ -418,11 +613,12 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { RangerPolicy ret = null; try { - List<RangerPolicy> policies = getServicePolicies(service.getId()); + List<RangerPolicy> policies = getAllPolicies(); if(policies != null) { for(RangerPolicy policy : policies) { - if(StringUtils.equals(policy.getName(), policyName)) { + if(StringUtils.equals(policy.getService(), service.getName()) && + StringUtils.equals(policy.getName(), policyName)) { ret = policy; break; @@ -441,17 +637,44 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { } @Override + public List<RangerPolicy> getAllPolicies() throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> ServiceFileStore.getAllPolicies()"); + } + + List<RangerPolicy> ret = null; + + try { + ret = loadFromDir(new Path(getDataDir()), FILE_PREFIX_POLICY, RangerPolicy.class); + } catch(Exception excp) { + LOG.error("ServiceFileStore.getAllPolicies(): failed to read policies", excp); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== ServiceFileStore.getAllPolicies(): count=" + (ret == null ? 0 : ret.size())); + } + + return ret; + } + + @Override public List<RangerPolicy> getServicePolicies(String serviceName) throws Exception { if(LOG.isDebugEnabled()) { LOG.debug("==> ServiceFileStore.getPolicies(" + serviceName + ")"); } - RangerService service = getByName(serviceName); + RangerService service = getServiceByName(serviceName); if(service == null) { throw new Exception("service does not exist - name='" + serviceName); } + RangerServiceDef serviceDef = findServiceDefByName(service.getType()); + + if(serviceDef == null) { + throw new Exception(service.getType() + ": unknown service-def)"); + } + List<RangerPolicy> ret = new ArrayList<RangerPolicy>(); try { @@ -469,7 +692,7 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { } if(LOG.isDebugEnabled()) { - LOG.debug("<== ServiceFileStore.getPolicies(" + serviceName + "): count=" + (ret == null ? 0 : ret.size())); + LOG.debug("<== ServiceFileStore.getPolicies(" + serviceName + "): count=" + ((ret == null) ? 0 : ret.size())); } return ret; @@ -481,7 +704,7 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { LOG.debug("==> ServiceFileStore.getPolicies(" + serviceId + ")"); } - RangerService service = get(serviceId); + RangerService service = getService(serviceId); if(service == null) { throw new Exception("service does not exist - id='" + serviceId); @@ -490,28 +713,57 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { List<RangerPolicy> ret = getServicePolicies(service.getName()); if(LOG.isDebugEnabled()) { - LOG.debug("<== ServiceFileStore.getPolicies(" + serviceId + "): " + (ret == null ? 0 : ret.size())); + LOG.debug("<== ServiceFileStore.getPolicies(" + serviceId + "): " + ((ret == null) ? 0 : ret.size())); } return ret; } @Override - public List<RangerPolicy> getAllPolicies() throws Exception { + public ServicePolicies getServicePoliciesIfUpdated(String serviceName, Long lastKnownVersion) throws Exception { if(LOG.isDebugEnabled()) { - LOG.debug("==> ServiceFileStore.getAllPolicies()"); + LOG.debug("==> ServiceFileStore.getServicePoliciesIfUpdated(" + serviceName + ", " + lastKnownVersion + ")"); } - List<RangerPolicy> ret = null; + RangerService service = getServiceByName(serviceName); - try { - ret = loadFromDir(new Path(getDataDir()), FILE_PREFIX_POLICY, RangerPolicy.class); - } catch(Exception excp) { - LOG.error("ServiceFileStore.getAllPolicies(): failed to read policies", excp); + if(service == null) { + throw new Exception("service does not exist - name='" + serviceName); + } + + RangerServiceDef serviceDef = findServiceDefByName(service.getType()); + + if(serviceDef == null) { + throw new Exception(service.getType() + ": unknown service-def)"); + } + + ServicePolicies ret = new ServicePolicies(); + ret.setServiceId(service.getId()); + ret.setServiceName(service.getName()); + ret.setPolicyVersion(service.getPolicyVersion()); + ret.setPolicyUpdateTime(service.getPolicyUpdateTime()); + ret.setServiceDef(serviceDef); + ret.setPolicies(new ArrayList<RangerPolicy>()); + + if(lastKnownVersion == null || service.getPolicyVersion() == null || lastKnownVersion.longValue() != service.getPolicyVersion().longValue()) { + + try { + List<RangerPolicy> policies = getAllPolicies(); + + if(policies != null) { + for(RangerPolicy policy : policies) { + if(StringUtils.equals(policy.getService(), serviceName)) { + ret.getPolicies().add(policy); + } + } + } + } catch(Exception excp) { + LOG.error("ServiceFileStore.getServicePoliciesIfUpdated(" + serviceName + "): failed to read policies", excp); + } } if(LOG.isDebugEnabled()) { - LOG.debug("<== ServiceFileStore.getAllPolicies(): count=" + (ret == null ? 0 : ret.size())); + LOG.debug("<== ServiceFileStore.getServicePoliciesIfUpdated(" + serviceName + ", " + lastKnownVersion + "): count=" + ((ret == null || ret.getPolicies() == null) ? 0 : ret.getPolicies().size())); } return ret; @@ -524,6 +776,88 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { } super.init(); + + initServiceDef(); + initService(); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== ServiceFileStore.init()"); + } + } + + + private void initServiceDef() { + if(LOG.isDebugEnabled()) { + LOG.debug("==> ServiceDefFileStore.initServiceDef()"); + } + + super.init(); + + try { + serviceDefs = new ArrayList<RangerServiceDef>(); + + // load definitions for legacy services from embedded resources + String[] legacyServiceDefResources = { + "/service-defs/ranger-servicedef-hdfs.json", + "/service-defs/ranger-servicedef-hive.json", + "/service-defs/ranger-servicedef-hbase.json", + "/service-defs/ranger-servicedef-knox.json", + "/service-defs/ranger-servicedef-storm.json", + }; + + for(String resource : legacyServiceDefResources) { + RangerServiceDef sd = loadFromResource(resource, RangerServiceDef.class); + + if(sd != null) { + serviceDefs.add(sd); + } + } + nextServiceDefId = getMaxId(serviceDefs) + 1; + + // load service definitions from file system + List<RangerServiceDef> sds = loadFromDir(new Path(getDataDir()), FILE_PREFIX_SERVICE_DEF, RangerServiceDef.class); + + if(sds != null) { + for(RangerServiceDef sd : sds) { + if(sd != null) { + if(isLegacyServiceDef(sd)) { + LOG.warn("Found in-built service-def '" + sd.getName() + "' under " + getDataDir() + ". Ignorning"); + + continue; + } + + RangerServiceDef existingSd = findServiceDefByName(sd.getName()); + + if(existingSd != null) { + removeServiceDef(existingSd); + } + + existingSd = findServiceDefById(sd.getId()); + + if(existingSd != null) { + removeServiceDef(existingSd); + } + + serviceDefs.add(sd); + } + } + } + nextServiceDefId = getMaxId(serviceDefs) + 1; + } catch(Exception excp) { + LOG.error("ServiceDefFileStore.initServiceDef(): failed to read service-defs", excp); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== ServiceDefFileStore.initServiceDef()"); + } + } + + private void initService() { + if(LOG.isDebugEnabled()) { + LOG.debug("==> ServiceFileStore.initService()"); + } + + super.init(); try { List<RangerService> services = loadFromDir(new Path(getDataDir()), FILE_PREFIX_SERVICE, RangerService.class); @@ -532,11 +866,11 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { nextServiceId = getMaxId(services) + 1; nextPolicyId = getMaxId(policies) + 1; } catch(Exception excp) { - LOG.error("ServiceDefFileStore.init() failed", excp); + LOG.error("ServiceDefFileStore.initService() failed", excp); } if(LOG.isDebugEnabled()) { - LOG.debug("<== ServiceFileStore.init()"); + LOG.debug("<== ServiceFileStore.initService()"); } } @@ -550,9 +884,7 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { preUpdate(policy); - Path filePath = new Path(getPolicyFile(service.getId(), policy.getId())); - - saveToFile(policy, filePath, true); + saveToFile(policy, service.getId(), true); postUpdate(policy); } @@ -561,10 +893,14 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { } private void handleServiceDelete(RangerService service) throws Exception { - List<RangerPolicy> policies = getServicePolicies(service.getName()); + List<RangerPolicy> policies = getAllPolicies(); if(policies != null) { for(RangerPolicy policy : policies) { + if(! StringUtils.equals(policy.getService(), service.getName())) { + continue; + } + preDelete(policy); Path filePath = new Path(getPolicyFile(service.getId(), policy.getId())); @@ -575,4 +911,71 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { } } } + + private void handlePolicyUpdate(RangerService service) throws Exception { + if(service == null) { + return; + } + + Long policyVersion = service.getPolicyVersion(); + + if(policyVersion == null) { + policyVersion = new Long(1); + } else { + policyVersion = new Long(policyVersion.longValue() + 1); + } + + service.setPolicyVersion(policyVersion); + service.setPolicyUpdateTime(new Date()); + + saveToFile(service, true); + } + + private RangerServiceDef findServiceDefById(long id) { + RangerServiceDef ret = null; + + for(RangerServiceDef sd : serviceDefs) { + if(sd != null && sd.getId() != null && sd.getId().longValue() == id) { + ret = sd; + + break; + } + } + + return ret; + } + + private RangerServiceDef findServiceDefByName(String sdName) { + RangerServiceDef ret = null; + + for(RangerServiceDef sd : serviceDefs) { + if(sd != null && StringUtils.equalsIgnoreCase(sd.getName(), sdName)) { + ret = sd; + + break; + } + } + + return ret; + } + + private void addServiceDef(RangerServiceDef sd) { + serviceDefs.add(sd); + } + + private void removeServiceDef(RangerServiceDef sd) { + serviceDefs.remove(sd); + } + + private boolean isLegacyServiceDef(RangerServiceDef sd) { + return sd == null ? false : (isLegacyServiceDef(sd.getName()) || isLegacyServiceDef(sd.getId())); + } + + private boolean isLegacyServiceDef(String name) { + return name == null ? false : legacyServiceDefs.containsKey(name); + } + + private boolean isLegacyServiceDef(Long id) { + return id == null ? false : legacyServiceDefs.containsValue(id); + } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/50c639ab/plugin-common/src/main/java/org/apache/ranger/plugin/util/PolicyRefresher.java ---------------------------------------------------------------------- diff --git a/plugin-common/src/main/java/org/apache/ranger/plugin/util/PolicyRefresher.java b/plugin-common/src/main/java/org/apache/ranger/plugin/util/PolicyRefresher.java new file mode 100644 index 0000000..489b5c0 --- /dev/null +++ b/plugin-common/src/main/java/org/apache/ranger/plugin/util/PolicyRefresher.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.util; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.ranger.plugin.policyengine.RangerPolicyEngine; +import org.apache.ranger.plugin.store.ServiceStore; + + +public class PolicyRefresher extends Thread { + private static final Log LOG = LogFactory.getLog(PolicyRefresher.class); + + private RangerPolicyEngine policyEngine = null; + private String serviceName = null; + private ServiceStore serviceStore = null; + private ServicePolicies lastKnownPolicies = null; + + private boolean shutdownFlag = false; + private long pollingIntervalMilliSeconds = 30 * 1000; + + + public PolicyRefresher(RangerPolicyEngine policyEngine, String serviceName, ServiceStore serviceStore) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> PolicyRefresher.PolicyRefresher(serviceName=" + serviceName + ")"); + } + + this.policyEngine = policyEngine; + this.serviceName = serviceName; + this.serviceStore = serviceStore; + + this.pollingIntervalMilliSeconds = 30 * 1000; // TODO: read from configuration + + if(LOG.isDebugEnabled()) { + LOG.debug("<== PolicyRefresher.PolicyRefresher(serviceName=" + serviceName + ")"); + } + } + + public void stopRefresher() { + shutdownFlag = true; + } + + public void run() { + if(LOG.isDebugEnabled()) { + LOG.debug("==> PolicyRefresher.run()"); + } + + while(! shutdownFlag) { + try { + long lastKnownVersion = (lastKnownPolicies == null || lastKnownPolicies.getPolicyVersion() == null) ? 0 : lastKnownPolicies.getPolicyVersion().longValue(); + + ServicePolicies svcPolicies = serviceStore.getServicePoliciesIfUpdated(serviceName, lastKnownVersion); + + long newVersion = (svcPolicies == null || svcPolicies.getPolicyVersion() == null) ? 0 : svcPolicies.getPolicyVersion().longValue(); + + boolean isUpdated = newVersion != 0 && lastKnownVersion != newVersion; + + if(isUpdated) { + LOG.info("PolicyRefresher(serviceName=" + serviceName + ").run(): found updated version. lastKnownVersion=" + lastKnownVersion + "; newVersion=" + newVersion); + + policyEngine.setPolicies(serviceName, svcPolicies.getServiceDef(), svcPolicies.getPolicies()); + + lastKnownPolicies = svcPolicies; + } else { + if(LOG.isDebugEnabled()) { + LOG.info("PolicyRefresher(serviceName=" + serviceName + ").run(): no update found. lastKnownVersion=" + lastKnownVersion + "; newVersion=" + newVersion); + } + } + } catch(Exception excp) { + LOG.error("PolicyRefresher(serviceName=" + serviceName + ").run(): ", excp); + } + + try { + Thread.sleep(pollingIntervalMilliSeconds); + } catch(Exception excp) { + LOG.error("PolicyRefresher(serviceName=" + serviceName + ").run(): error while sleep. exiting thread", excp); + + throw new RuntimeException(excp); + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== PolicyRefresher.run()"); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/50c639ab/plugin-common/src/main/java/org/apache/ranger/plugin/util/ServicePolicies.java ---------------------------------------------------------------------- diff --git a/plugin-common/src/main/java/org/apache/ranger/plugin/util/ServicePolicies.java b/plugin-common/src/main/java/org/apache/ranger/plugin/util/ServicePolicies.java new file mode 100644 index 0000000..f1c8adf --- /dev/null +++ b/plugin-common/src/main/java/org/apache/ranger/plugin/util/ServicePolicies.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.util; + + +import java.util.Date; +import java.util.List; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.ranger.plugin.model.RangerPolicy; +import org.apache.ranger.plugin.model.RangerServiceDef; +import org.codehaus.jackson.annotate.JsonAutoDetect; +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.annotate.JsonAutoDetect.Visibility; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +@JsonAutoDetect(getterVisibility=Visibility.NONE, setterVisibility=Visibility.NONE, fieldVisibility=Visibility.ANY) +@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL ) +@JsonIgnoreProperties(ignoreUnknown=true) +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class ServicePolicies implements java.io.Serializable { + private static final long serialVersionUID = 1L; + + private String serviceName; + private Long serviceId; + private RangerServiceDef serviceDef; + private Long policyVersion; + private Date policyUpdateTime; + private List<RangerPolicy> policies; + + + /** + * @return the serviceName + */ + public String getServiceName() { + return serviceName; + } + /** + * @param serviceName the serviceName to set + */ + public void setServiceName(String serviceName) { + this.serviceName = serviceName; + } + /** + * @return the serviceId + */ + public Long getServiceId() { + return serviceId; + } + /** + * @param serviceId the serviceId to set + */ + public void setServiceId(Long serviceId) { + this.serviceId = serviceId; + } + /** + * @return the serviceDef + */ + public RangerServiceDef getServiceDef() { + return serviceDef; + } + /** + * @param serviceDef the serviceDef to set + */ + public void setServiceDef(RangerServiceDef serviceDef) { + this.serviceDef = serviceDef; + } + /** + * @return the policyVersion + */ + public Long getPolicyVersion() { + return policyVersion; + } + /** + * @param policyVersion the policyVersion to set + */ + public void setPolicyVersion(Long policyVersion) { + this.policyVersion = policyVersion; + } + /** + * @return the policyUpdateTime + */ + public Date getPolicyUpdateTime() { + return policyUpdateTime; + } + /** + * @param policyUpdateTime the policyUpdateTime to set + */ + public void setPolicyUpdateTime(Date policyUpdateTime) { + this.policyUpdateTime = policyUpdateTime; + } + /** + * @return the policies + */ + public List<RangerPolicy> getPolicies() { + return policies; + } + /** + * @param policies the policies to set + */ + public void setPolicies(List<RangerPolicy> policies) { + this.policies = policies; + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/50c639ab/plugin-common/src/test/java/org/apache/ranger/plugin/store/TestServiceStore.java ---------------------------------------------------------------------- diff --git a/plugin-common/src/test/java/org/apache/ranger/plugin/store/TestServiceStore.java b/plugin-common/src/test/java/org/apache/ranger/plugin/store/TestServiceStore.java index dcd2125..6bf5811 100644 --- a/plugin-common/src/test/java/org/apache/ranger/plugin/store/TestServiceStore.java +++ b/plugin-common/src/test/java/org/apache/ranger/plugin/store/TestServiceStore.java @@ -29,16 +29,14 @@ import org.apache.ranger.plugin.model.RangerService; import org.apache.ranger.plugin.model.RangerServiceDef; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItem; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource; -import org.apache.ranger.plugin.store.ServiceDefStore; -import org.apache.ranger.plugin.store.ServiceDefStoreFactory; import org.apache.ranger.plugin.store.ServiceStore; import org.apache.ranger.plugin.store.ServiceStoreFactory; +import org.apache.ranger.plugin.util.ServicePolicies; import org.junit.BeforeClass; import org.junit.Test; public class TestServiceStore { - static ServiceDefStore svcDefStore = null; - static ServiceStore svcStore = null; + static ServiceStore svcStore = null; static String sdName = "HdfsTest"; static String serviceName = "HdfsTest-dev"; @@ -46,35 +44,34 @@ public class TestServiceStore { @BeforeClass public static void setupTest() { - svcDefStore = ServiceDefStoreFactory.instance().getServiceDefStore(); - svcStore = ServiceStoreFactory.instance().getServiceStore(); + svcStore = ServiceStoreFactory.instance().getServiceStore(); } @Test public void testServiceManager() throws Exception { - List<RangerServiceDef> sds = svcDefStore.getAll(); + List<RangerServiceDef> sds = svcStore.getAllServiceDefs(); int initSdCount = sds == null ? 0 : sds.size(); RangerServiceDef sd = new RangerServiceDef(sdName, "org.apache.ranger.services.TestService", "TestService", "test servicedef description", null, null, null, null, null); - RangerServiceDef createdSd = svcDefStore.create(sd); + RangerServiceDef createdSd = svcStore.createServiceDef(sd); assertNotNull("createServiceDef() failed", createdSd != null); - sds = svcDefStore.getAll(); + sds = svcStore.getAllServiceDefs(); assertEquals("createServiceDef() failed", initSdCount + 1, sds == null ? 0 : sds.size()); String updatedDescription = sd.getDescription() + ": updated"; createdSd.setDescription(updatedDescription); - RangerServiceDef updatedSd = svcDefStore.update(createdSd); + RangerServiceDef updatedSd = svcStore.updateServiceDef(createdSd); assertNotNull("updateServiceDef(updatedDescription) failed", updatedSd); assertEquals("updateServiceDef(updatedDescription) failed", updatedDescription, updatedSd.getDescription()); - sds = svcDefStore.getAll(); + sds = svcStore.getAllServiceDefs(); assertEquals("updateServiceDef(updatedDescription) failed", initSdCount + 1, sds == null ? 0 : sds.size()); - String updatedName = sd.getName() + "-Renamed"; /* + String updatedName = sd.getName() + "-Renamed"; updatedSd.setName(updatedName); updatedSd = sdMgr.update(updatedSd); assertNotNull("updateServiceDef(updatedName) failed", updatedSd); @@ -84,34 +81,34 @@ public class TestServiceStore { assertEquals("updateServiceDef(updatedName) failed", initSdCount + 1, sds == null ? 0 : sds.size()); */ - List<RangerService> services = svcStore.getAll(); + List<RangerService> services = svcStore.getAllServices(); int initServiceCount = services == null ? 0 : services.size(); RangerService svc = new RangerService(sdName, serviceName, "test service description", Boolean.TRUE, null); - RangerService createdSvc = svcStore.create(svc); + RangerService createdSvc = svcStore.createService(svc); assertNotNull("createService() failed", createdSvc); - services = svcStore.getAll(); + services = svcStore.getAllServices(); assertEquals("createServiceDef() failed", initServiceCount + 1, services == null ? 0 : services.size()); updatedDescription = createdSvc.getDescription() + ": updated"; createdSvc.setDescription(updatedDescription); - RangerService updatedSvc = svcStore.update(createdSvc); + RangerService updatedSvc = svcStore.updateService(createdSvc); assertNotNull("updateService(updatedDescription) failed", updatedSvc); assertEquals("updateService(updatedDescription) failed", updatedDescription, updatedSvc.getDescription()); - services = svcStore.getAll(); + services = svcStore.getAllServices(); assertEquals("updateService(updatedDescription) failed", initServiceCount + 1, services == null ? 0 : services.size()); - updatedName = serviceName + "-Renamed"; + String updatedName = serviceName + "-Renamed"; updatedSvc.setName(updatedName); - updatedSvc = svcStore.update(updatedSvc); + updatedSvc = svcStore.updateService(updatedSvc); assertNotNull("updateService(updatedName) failed", updatedSvc); assertEquals("updateService(updatedName) failed", updatedName, updatedSvc.getName()); - services = svcStore.getAll(); + services = svcStore.getAllServices(); assertEquals("updateService(updatedName) failed", initServiceCount + 1, services == null ? 0 : services.size()); List<RangerPolicy> policies = svcStore.getAllPolicies(); @@ -179,27 +176,41 @@ public class TestServiceStore { // rename the service; all the policies for this service should reflect the new service name updatedName = serviceName + "-Renamed2"; updatedSvc.setName(updatedName); - updatedSvc = svcStore.update(updatedSvc); + updatedSvc = svcStore.updateService(updatedSvc); assertNotNull("updateService(updatedName2) failed", updatedSvc); assertEquals("updateService(updatedName2) failed", updatedName, updatedSvc.getName()); - services = svcStore.getAll(); + services = svcStore.getAllServices(); assertEquals("updateService(updatedName2) failed", initServiceCount + 1, services == null ? 0 : services.size()); updatedPolicy = svcStore.getPolicy(createdPolicy.getId()); assertNotNull("updateService(updatedName2) failed", updatedPolicy); assertEquals("updateService(updatedName2) failed", updatedPolicy.getService(), updatedSvc.getName()); + ServicePolicies svcPolicies = svcStore.getServicePoliciesIfUpdated(updatedSvc.getName(), 0l); + assertNotNull("getServicePolicies(" + updatedSvc.getName() + ") failed", svcPolicies); + assertNotNull("getServicePolicies(" + updatedSvc.getName() + ") failed", svcPolicies.getPolicies()); + assertEquals("getServicePolicies(" + updatedSvc.getName() + ") failed", svcPolicies.getServiceName(), updatedSvc.getName()); + assertEquals("getServicePolicies(" + updatedSvc.getName() + ") failed", svcPolicies.getServiceId(), updatedSvc.getId()); + assertEquals("getServicePolicies(" + updatedSvc.getName() + ") failed", svcPolicies.getPolicyVersion(), updatedSvc.getPolicyVersion()); + assertEquals("getServicePolicies(" + updatedSvc.getName() + ") failed", svcPolicies.getPolicyUpdateTime(), updatedSvc.getPolicyUpdateTime()); + assertEquals("getServicePolicies(" + updatedSvc.getName() + ") failed", svcPolicies.getServiceDef(), updatedSd); + assertEquals("getServicePolicies(" + updatedSvc.getName() + ") failed", svcPolicies.getPolicies().size(), 1); + assertEquals("getServicePolicies(" + updatedSvc.getName() + ") failed", svcPolicies.getPolicies().get(0).getName(), updatedPolicy.getName()); + + ServicePolicies updatedPolicies = svcStore.getServicePoliciesIfUpdated(updatedSvc.getName(), svcPolicies.getPolicyVersion()); + assertNull(updatedPolicies); + svcStore.deletePolicy(policy.getId()); policies = svcStore.getAllPolicies(); assertEquals("deletePolicy() failed", initPolicyCount, policies == null ? 0 : policies.size()); - svcStore.delete(svc.getId()); - services = svcStore.getAll(); + svcStore.deleteService(svc.getId()); + services = svcStore.getAllServices(); assertEquals("deleteService() failed", initServiceCount, services == null ? 0 : services.size()); - svcDefStore.delete(sd.getId()); - sds = svcDefStore.getAll(); + svcStore.deleteServiceDef(sd.getId()); + sds = svcStore.getAllServiceDefs(); assertEquals("deleteServiceDef() failed", initSdCount, sds == null ? 0 : sds.size()); } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/50c639ab/plugin-common/src/test/java/org/apache/ranger/plugin/util/TestPolicyRefresher.java ---------------------------------------------------------------------- diff --git a/plugin-common/src/test/java/org/apache/ranger/plugin/util/TestPolicyRefresher.java b/plugin-common/src/test/java/org/apache/ranger/plugin/util/TestPolicyRefresher.java new file mode 100644 index 0000000..6baa613 --- /dev/null +++ b/plugin-common/src/test/java/org/apache/ranger/plugin/util/TestPolicyRefresher.java @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.util; + +import static org.junit.Assert.*; + +import java.util.List; + +import org.apache.ranger.plugin.model.RangerPolicy; +import org.apache.ranger.plugin.model.RangerService; +import org.apache.ranger.plugin.model.RangerServiceDef; +import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItem; +import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemAccess; +import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource; +import org.apache.ranger.plugin.policyengine.RangerPolicyEngineImpl; +import org.apache.ranger.plugin.store.ServiceStore; +import org.apache.ranger.plugin.store.ServiceStoreFactory; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + + +public class TestPolicyRefresher { + static RangerPolicyEngineImpl policyEngine = null; + static ServiceStore svcStore = null; + static PolicyRefresher refresher = null; + + static long sleepTimeInMs = 45 * 1000; + static String sdName = "hbase"; + static String svcName = "unit-test-TestPolicyRefresher"; + static RangerService svc = null; + static RangerPolicy policy1 = null; + static RangerPolicy policy2 = null; + + static boolean isPolicyRefreshed = false; + static long policyCount = 0; + + + /** + * @throws java.lang.Exception + */ + @BeforeClass + public static void setUpBeforeClass() throws Exception { + policyEngine = new RangerPolicyEngineImpl() { + @Override + public void setPolicies(String serviceName, RangerServiceDef serviceDef, List<RangerPolicy> policies) { + isPolicyRefreshed = true; + policyCount = policies != null ? policies.size() : 0; + + super.setPolicies(serviceName, serviceDef, policies); + } + }; + + svcStore = ServiceStoreFactory.instance().getServiceStore(); + + refresher = new PolicyRefresher(policyEngine, svcName, svcStore); + refresher.start(); + + // cleanup if the test service already exists + svc = svcStore.getServiceByName(svcName); + if(svc != null) { + svcStore.deleteService(svc.getId()); + } + + // create a service + svc = new RangerService(sdName, svcName, "test service description", Boolean.TRUE, null); + + svc = svcStore.createService(svc); + assertNotNull("createService(" + svcName + ") failed", svc); + } + + /** + * @throws java.lang.Exception + */ + @AfterClass + public static void tearDownAfterClass() throws Exception { + if(svcStore == null) { + return; + } + + if(policy1 != null) { + svcStore.deletePolicy(policy1.getId()); + } + + if(policy2 != null) { + svcStore.deletePolicy(policy2.getId()); + } + + if(svc != null) { + svcStore.deleteService(svc.getId()); + } + + if(refresher != null) { + refresher.stopRefresher(); + } + } + + @Test + public void testRefresher() throws Exception { + assertEquals("policy count - initial", 0, policyCount); + + RangerPolicy policy = new RangerPolicy(svc.getName(), "policy1", "test policy description", Boolean.TRUE, null, null); + policy.getResources().put("table", new RangerPolicyResource("employee", Boolean.FALSE, Boolean.TRUE)); + policy.getResources().put("column-family", new RangerPolicyResource("personal", Boolean.FALSE, Boolean.TRUE)); + policy.getResources().put("column", new RangerPolicyResource("ssn", Boolean.FALSE, Boolean.TRUE)); + + RangerPolicyItem item1 = new RangerPolicyItem(); + item1.getAccesses().add(new RangerPolicyItemAccess("admin")); + item1.getUsers().add("admin"); + item1.getGroups().add("hr"); + + RangerPolicyItem item2 = new RangerPolicyItem(); + item2.getAccesses().add(new RangerPolicyItemAccess("read")); + item2.getGroups().add("public"); + + policy.getPolicyItems().add(item1); + policy.getPolicyItems().add(item2); + + policy1 = svcStore.createPolicy(policy); + + Thread.sleep(sleepTimeInMs); + + assertTrue("policy refresh - after one new policy", isPolicyRefreshed); + assertEquals("policy count - after one new policy", 1, policyCount); + isPolicyRefreshed = false; + + policy = new RangerPolicy(svc.getName(), "policy2", "test policy description", Boolean.TRUE, null, null); + policy.getResources().put("table", new RangerPolicyResource("employee", Boolean.FALSE, Boolean.TRUE)); + policy.getResources().put("column-family", new RangerPolicyResource("finance", Boolean.FALSE, Boolean.TRUE)); + policy.getResources().put("column", new RangerPolicyResource("balance", Boolean.FALSE, Boolean.TRUE)); + + item1 = new RangerPolicyItem(); + item1.getAccesses().add(new RangerPolicyItemAccess("admin")); + item1.getUsers().add("admin"); + item1.getGroups().add("finance"); + + policy.getPolicyItems().add(item1); + + policy2 = svcStore.createPolicy(policy); + + Thread.sleep(sleepTimeInMs); + + assertTrue("policy refresh - after two new policies", isPolicyRefreshed); + assertEquals("policy count - after two new policies", 2, policyCount); + isPolicyRefreshed = false; + + Thread.sleep(sleepTimeInMs); + + assertFalse("policy refresh - after no new policies", isPolicyRefreshed); + assertEquals("policy count - after no new policies", 2, policyCount); + isPolicyRefreshed = false; + + item2 = new RangerPolicyItem(); + item2.getAccesses().add(new RangerPolicyItemAccess("read")); + item2.getGroups().add("public"); + policy2.getPolicyItems().add(item2); + + policy2 = svcStore.updatePolicy(policy2); + + Thread.sleep(sleepTimeInMs); + + assertTrue("policy refresh - after update policy", isPolicyRefreshed); + assertEquals("policy count - after update policy", 2, policyCount); + isPolicyRefreshed = false; + + svcStore.deletePolicy(policy2.getId()); + + Thread.sleep(sleepTimeInMs); + + assertTrue("policy refresh - after delete policy", isPolicyRefreshed); + assertEquals("policy count - after delete policy", 1, policyCount); + isPolicyRefreshed = false; + policy2 = null; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/50c639ab/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java index b21528c..2ecd347 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java @@ -37,8 +37,6 @@ import org.apache.commons.logging.LogFactory; import org.apache.ranger.plugin.model.RangerPolicy; import org.apache.ranger.plugin.model.RangerService; import org.apache.ranger.plugin.model.RangerServiceDef; -import org.apache.ranger.plugin.store.ServiceDefStore; -import org.apache.ranger.plugin.store.ServiceDefStoreFactory; import org.apache.ranger.plugin.store.ServiceStore; import org.apache.ranger.plugin.store.ServiceStoreFactory; import org.apache.ranger.view.VXResponse; @@ -58,12 +56,10 @@ public class ServiceREST { @Autowired RESTErrorUtil restErrorUtil; - private ServiceDefStore serviceDefStore = null; - private ServiceStore svcStore = null; + private ServiceStore svcStore = null; public ServiceREST() { - serviceDefStore = ServiceDefStoreFactory.instance().getServiceDefStore(); - svcStore = ServiceStoreFactory.instance().getServiceStore(); + svcStore = ServiceStoreFactory.instance().getServiceStore(); } @GET @@ -77,7 +73,7 @@ public class ServiceREST { RangerServiceDef ret = null; try { - ret = serviceDefStore.get(id); + ret = svcStore.getServiceDef(id); } catch(Exception excp) { throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, excp.getMessage(), true); } @@ -104,7 +100,7 @@ public class ServiceREST { RangerServiceDef ret = null; try { - ret = serviceDefStore.getByName(name); + ret = svcStore.getServiceDefByName(name); } catch(Exception excp) { throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, excp.getMessage(), true); } @@ -131,7 +127,7 @@ public class ServiceREST { List<RangerServiceDef> ret = null; try { - ret = serviceDefStore.getAll(); + ret = svcStore.getAllServiceDefs(); } catch(Exception excp) { throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, excp.getMessage(), true); } @@ -155,7 +151,7 @@ public class ServiceREST { RangerServiceDef ret = null; try { - ret = serviceDefStore.create(serviceDef); + ret = svcStore.createServiceDef(serviceDef); } catch(Exception excp) { throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, excp.getMessage(), true); } @@ -179,7 +175,7 @@ public class ServiceREST { RangerServiceDef ret = null; try { - ret = serviceDefStore.update(serviceDef); + ret = svcStore.updateServiceDef(serviceDef); } catch(Exception excp) { throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, excp.getMessage(), true); } @@ -201,7 +197,7 @@ public class ServiceREST { } try { - serviceDefStore.delete(id); + svcStore.deleteServiceDef(id); } catch(Exception excp) { throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, excp.getMessage(), true); } @@ -223,7 +219,7 @@ public class ServiceREST { RangerService ret = null; try { - ret = svcStore.get(id); + ret = svcStore.getService(id); } catch(Exception excp) { throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, excp.getMessage(), true); } @@ -250,7 +246,7 @@ public class ServiceREST { RangerService ret = null; try { - ret = svcStore.getByName(name); + ret = svcStore.getServiceByName(name); } catch(Exception excp) { throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, excp.getMessage(), true); } @@ -277,7 +273,7 @@ public class ServiceREST { List<RangerService> ret = null; try { - ret = svcStore.getAll(); + ret = svcStore.getAllServices(); } catch(Exception excp) { throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, excp.getMessage(), true); } @@ -325,7 +321,7 @@ public class ServiceREST { RangerService ret = null; try { - ret = svcStore.create(service); + ret = svcStore.createService(service); } catch(Exception excp) { throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, excp.getMessage(), true); } @@ -348,7 +344,7 @@ public class ServiceREST { RangerService ret = null; try { - ret = svcStore.update(service); + ret = svcStore.updateService(service); } catch(Exception excp) { throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, excp.getMessage(), true); } @@ -370,7 +366,7 @@ public class ServiceREST { } try { - svcStore.delete(id); + svcStore.deleteService(id); } catch(Exception excp) { throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, excp.getMessage(), true); }
