RANGER-274: Added service-def for Tags; added TagStore/TagFileStore, TagREST to add/retrieve tags for resources
Signed-off-by: Madhan Neethiraj <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/9578f94c Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/9578f94c Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/9578f94c Branch: refs/heads/tag-policy Commit: 9578f94cdf0dd0e6ea31bb4c524edfda82576ef4 Parents: b5a23b2 Author: Abhay Kulkarni <[email protected]> Authored: Mon May 18 14:34:21 2015 -0700 Committer: Madhan Neethiraj <[email protected]> Committed: Tue May 19 20:14:43 2015 -0700 ---------------------------------------------------------------------- .../ranger/plugin/model/RangerResource.java | 143 +++++ .../ranger/plugin/model/RangerServiceDef.java | 7 + .../ranger/plugin/model/RangerTagDef.java | 133 +++++ .../plugin/store/AbstractServiceStore.java | 372 ++++++++++++ .../ranger/plugin/store/AbstractTagStore.java | 88 +++ .../plugin/store/EmbeddedServiceDefsUtil.java | 11 +- .../org/apache/ranger/plugin/store/PList.java | 189 ++++++ .../plugin/store/RangerPolicyPaginatedList.java | 62 ++ .../store/RangerServiceDefPaginatedList.java | 61 ++ .../store/RangerServicePaginatedList.java | 62 ++ .../ranger/plugin/store/ServiceStore.java | 22 +- .../ranger/plugin/store/TagPredicateUtil.java | 212 +++++++ .../apache/ranger/plugin/store/TagStore.java | 59 ++ .../ranger/plugin/store/file/BaseFileStore.java | 395 ------------- .../ranger/plugin/store/file/FileStoreUtil.java | 317 +++++++++++ .../plugin/store/file/ServiceFileStore.java | 59 +- .../ranger/plugin/store/file/TagFileStore.java | 569 +++++++++++++++++++ .../plugin/store/rest/ServiceRESTStore.java | 3 +- .../apache/ranger/plugin/util/SearchFilter.java | 9 + .../service-defs/ranger-servicedef-_tag_.json | 73 +++ .../org/apache/ranger/biz/ServiceDBStore.java | 68 ++- .../org/apache/ranger/common/view/VList.java | 7 +- .../org/apache/ranger/rest/ServiceREST.java | 73 ++- .../java/org/apache/ranger/rest/TagREST.java | 409 +++++++++++++ .../apache/ranger/rest/TagRESTConstants.java | 39 ++ .../ranger/service/RangerServiceDefService.java | 33 +- .../apache/ranger/biz/TestServiceDBStore.java | 18 +- .../org/apache/ranger/rest/TestServiceREST.java | 12 +- 28 files changed, 3029 insertions(+), 476 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/9578f94c/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerResource.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerResource.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerResource.java new file mode 100644 index 0000000..23bb098 --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerResource.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.model; + +import org.codehaus.jackson.annotate.JsonAutoDetect; +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * This class represents a RangerResource including the service-type (such as hdfs, hive, etc.) in which it is supported. + * This implies that there is one-to-one mapping between service-type and the resource-type which is a valid assumption. + * Service-type must be one of service-types supported by Ranger. + * + * This class also contains a list of (tag-name, JSON-string-representing-tagattribute-tagattributevalue-pairs) + * + */ + +@JsonAutoDetect(getterVisibility= JsonAutoDetect.Visibility.NONE, setterVisibility= JsonAutoDetect.Visibility.NONE, fieldVisibility= JsonAutoDetect.Visibility.ANY) +@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL ) +@JsonIgnoreProperties(ignoreUnknown=true) +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) + +public class RangerResource extends RangerBaseModelObject { + private static final long serialVersionUID = 1L; + + private String serviceType = null; // one of any supported by any component + private Map<String, RangerPolicy.RangerPolicyResource> resourceSpec = null; // + private String tagServiceName = null; + private List<RangerResourceTag> tagsAndValues = null; + + public RangerResource(String serviceType, Map<String, RangerPolicy.RangerPolicyResource> resourceSpecs, String tagServiceName, List<RangerResourceTag> tagsAndValues) { + super(); + setServiceType(serviceType); + setResourceSpecs(resourceSpecs); + setTagServiceName(tagServiceName); + setTagsAndValues(tagsAndValues); + } + + public RangerResource() { + this(null, null, null, null); + } + + public String getServiceType() { + return serviceType; + } + + public Map<String, RangerPolicy.RangerPolicyResource> getResourceSpecs() { + return resourceSpec; + } + + public String getTagServiceName() { + return tagServiceName; + } + + public List<RangerResourceTag> getTagsAndValues() { + return tagsAndValues; + } + + // And corresponding set methods + public void setServiceType(String serviceType) { + this.serviceType = serviceType == null ? new String() : serviceType; + } + + public void setResourceSpecs(Map<String, RangerPolicy.RangerPolicyResource> fullName) { + this.resourceSpec = resourceSpec == null ? new HashMap<String, RangerPolicy.RangerPolicyResource>() : resourceSpec; + } + + public void setTagServiceName(String tagServiceName) { + this.tagServiceName = tagServiceName == null ? new String() : tagServiceName; + } + + public void setTagsAndValues(List<RangerResourceTag> tagsAndValues) { + this.tagsAndValues = tagsAndValues == null ? new ArrayList<RangerResourceTag>() : tagsAndValues; + } + + /** + * Represents a tag and its attribute-values for a resource. + */ + + @JsonAutoDetect(getterVisibility= JsonAutoDetect.Visibility.NONE, setterVisibility= JsonAutoDetect.Visibility.NONE, fieldVisibility= JsonAutoDetect.Visibility.ANY) + @JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL ) + @JsonIgnoreProperties(ignoreUnknown=true) + @XmlRootElement + @XmlAccessorType(XmlAccessType.FIELD) + + public static class RangerResourceTag implements java.io.Serializable { + + private String name = null; + private Map<String, Object> attributeValues = null; // Will be JSON string with (name, value) pairs of tag attributes in database + + public RangerResourceTag(String name, Map<String, Object> attributeValues) { + super(); + setName(name); + setAttributeValues(attributeValues); + } + + public RangerResourceTag() { + this(null, null); + } + + public String getName() { + return name; + } + + public Map<String, Object> getAttributeValues() { + return attributeValues; + } + + public void setName(String name) { + this.name = name; + } + + public void setAttributeValues(Map<String, Object> attributeValues) { + this.attributeValues = attributeValues; + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/9578f94c/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerServiceDef.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerServiceDef.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerServiceDef.java index 302da2f..49ee3a1 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerServiceDef.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerServiceDef.java @@ -1672,6 +1672,13 @@ public class RangerServiceDef extends RangerBaseModelObject implements java.io.S setImpliedGrants(impliedGrants); } + public RangerAccessTypeDef(RangerAccessTypeDef other) { + this.setName(other.getName()); + this.setLabel(other.getLabel()); + this.setRbKeyLabel(other.getRbKeyLabel()); + this.setImpliedGrants(other.getImpliedGrants()); + } + /** * @return the itemId */ http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/9578f94c/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerTagDef.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerTagDef.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerTagDef.java new file mode 100644 index 0000000..0f58c96 --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerTagDef.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.model; + +import org.codehaus.jackson.annotate.JsonAutoDetect; +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.ArrayList; +import java.util.List; + +@JsonAutoDetect(getterVisibility= JsonAutoDetect.Visibility.NONE, setterVisibility= JsonAutoDetect.Visibility.NONE, fieldVisibility= JsonAutoDetect.Visibility.ANY) +@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL ) +@JsonIgnoreProperties(ignoreUnknown=true) +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) + +/** + * Represents a TAG definition known to Ranger. In general, this will be provided + * by some external system identified by 'source'. + * + */ + +public class RangerTagDef extends RangerBaseModelObject { + private static final long serialVersionUID = 1L; + + private String name = null; + private String source = null; + + private List<RangerTagAttributeDef> attributeDefs; + + public RangerTagDef() { + this(null, "Internal"); + } + + public RangerTagDef(String name) { + this(name, "Internal"); + } + + public RangerTagDef(String name, String source) { + super(); + setName(name); + setSource(source); + } + + public String getName() { + return name; + } + + public void setName(String name) { + + this.name = name == null ? new String() : name; + } + + public String getSource() { + return source; + } + + public void setSource(String source) { + this.source = source == null ? new String() : source; + } + + public List<RangerTagAttributeDef> getAttributeDefs() { + return attributeDefs; + } + + public void setAttributeDefs(List<RangerTagAttributeDef> attributeDefs) { + this.attributeDefs = attributeDefs == null ? new ArrayList<RangerTagAttributeDef>() : attributeDefs; + } + + /** + * Represents one attribute for a TAG. TAG-Attribute consists of a name and type. + * name provides a handle for possible specification of additional information + * associated with the TAG. + * Interpretation of type is up to the policy-engine. + */ + + @JsonAutoDetect(getterVisibility= JsonAutoDetect.Visibility.NONE, setterVisibility= JsonAutoDetect.Visibility.NONE, fieldVisibility= JsonAutoDetect.Visibility.ANY) + @JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL ) + @JsonIgnoreProperties(ignoreUnknown=true) + @XmlRootElement + @XmlAccessorType(XmlAccessType.FIELD) + + public static class RangerTagAttributeDef implements java.io.Serializable { + + private String name = null; + private String type = null; + + public RangerTagAttributeDef() { + this(null, null); + } + + public RangerTagAttributeDef(String name, String type) { + setName(name); + setType(type); + } + + public String getName() { + return name; + } + + public String getType() { + return type; + } + + public void setName(String name) { + this.name = name == null ? new String() : name; + } + public void setType(String type) { + this.type = type == null ? new String() : type; + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/9578f94c/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java new file mode 100644 index 0000000..e30535a --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java @@ -0,0 +1,372 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.store; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.ranger.plugin.model.RangerBaseModelObject; +import org.apache.ranger.plugin.model.RangerPolicy; +import org.apache.ranger.plugin.model.RangerService; +import org.apache.ranger.plugin.model.RangerServiceDef; +import org.apache.ranger.plugin.util.SearchFilter; + +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.UUID; + +public abstract class AbstractServiceStore implements ServiceStore { + private static final Log LOG = LogFactory.getLog(AbstractServiceStore.class); + + + private static final int MAX_ACCESS_TYPES_IN_SERVICE_DEF = 1000; + + @Override + public void updateTagServiceDefForAccessTypes() throws Exception { + if (LOG.isDebugEnabled()) { + LOG.debug("==> ServiceDefDBStore.updateTagServiceDefForAccessTypes()"); + } + List<RangerServiceDef> allServiceDefs = getServiceDefs(new SearchFilter()); + for (RangerServiceDef serviceDef : allServiceDefs) { + if (StringUtils.isEmpty(serviceDef.getName()) || serviceDef.getName().equals(EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME)) { + continue; + } + updateTagServiceDefForUpdatingAccessTypes(serviceDef); + } + if (LOG.isDebugEnabled()) { + LOG.debug("<== ServiceDefDBStore.updateTagServiceDefForAccessTypes()"); + } + return; + } + + @Override + public void deleteServiceDef(Long id, Boolean forceDelete) throws Exception { + deleteServiceDef(id); // Ignore forceDelete flag + } + + @Override + public RangerServiceDefPaginatedList getPaginatedServiceDefs(SearchFilter filter) throws Exception { + List<RangerServiceDef> resultList = getServiceDefs(filter); + + RangerServiceDefPaginatedList ret = new RangerServiceDefPaginatedList(); + + ret.setResultSize(resultList.size()); + ret.setPageSize(resultList.size()); + ret.setSortBy(filter.getSortBy()); + ret.setSortType(filter.getSortType()); + ret.setStartIndex(0); + ret.setTotalCount(resultList.size()); + + ret.setServiceDefs(resultList); + + return ret; + } + + @Override + public RangerServicePaginatedList getPaginatedServices(SearchFilter filter) throws Exception { + List<RangerService> resultList = getServices(filter); + + RangerServicePaginatedList ret = new RangerServicePaginatedList(); + + ret.setResultSize(resultList.size()); + ret.setPageSize(resultList.size()); + ret.setSortBy(filter.getSortBy()); + ret.setSortType(filter.getSortType()); + ret.setStartIndex(0); + ret.setTotalCount(resultList.size()); + + ret.setServices(resultList); + + return ret; + } + + @Override + public RangerPolicyPaginatedList getPaginatedPolicies(SearchFilter filter) throws Exception { + List<RangerPolicy> resultList = getPolicies(filter); + + RangerPolicyPaginatedList ret = new RangerPolicyPaginatedList(); + + ret.setResultSize(resultList.size()); + ret.setPageSize(resultList.size()); + ret.setSortBy(filter.getSortBy()); + ret.setSortType(filter.getSortType()); + ret.setStartIndex(0); + ret.setTotalCount(resultList.size()); + + ret.setPolicies(resultList); + + return ret; + } + + @Override + public RangerPolicyPaginatedList getPaginatedServicePolicies(Long serviceId, SearchFilter filter) throws Exception { + List<RangerPolicy> resultList = getServicePolicies(serviceId, filter); + + RangerPolicyPaginatedList ret = new RangerPolicyPaginatedList(); + + ret.setResultSize(resultList.size()); + ret.setPageSize(resultList.size()); + ret.setSortBy(filter.getSortBy()); + ret.setSortType(filter.getSortType()); + ret.setStartIndex(0); + ret.setTotalCount(resultList.size()); + + ret.setPolicies(resultList); + + return ret; + } + + @Override + public RangerPolicyPaginatedList getPaginatedServicePolicies(String serviceName, SearchFilter filter) throws Exception { + List<RangerPolicy> resultList = getServicePolicies(serviceName, filter); + + RangerPolicyPaginatedList ret = new RangerPolicyPaginatedList(); + + ret.setResultSize(resultList.size()); + ret.setPageSize(resultList.size()); + ret.setSortBy(filter.getSortBy()); + ret.setSortType(filter.getSortType()); + ret.setStartIndex(0); + ret.setTotalCount(resultList.size()); + + ret.setPolicies(resultList); + + return ret; + + } + + @Override + public RangerPolicy getPolicyFromEventTime(String eventTimeStr, Long policyId) { + RangerPolicy ret = null; + try { + ret = getPolicy(policyId); + } catch (Exception e) { + // Do nothing + } + return ret; + } + + @Override + public RangerPolicy getPolicyForVersionNumber(Long policyId, Integer versionNo) { + RangerPolicy ret = null; + try { + ret = getPolicy(policyId); + } catch (Exception e) { + // Do nothing + } + return ret; + } + + @Override + public String getPolicyForVersionNumber(Long policyId) { + RangerPolicy ret = null; + try { + ret = getPolicy(policyId); + } catch (Exception e) { + // Do nothing + } + return ret == null ? null : ret.getName(); + } + + protected void preCreate(RangerBaseModelObject obj) throws Exception { + obj.setId(new Long(0)); + obj.setGuid(UUID.randomUUID().toString()); + obj.setCreateTime(new Date()); + obj.setUpdateTime(obj.getCreateTime()); + obj.setVersion(new Long(1)); + } + + protected void preCreate(RangerService service) throws Exception { + preCreate((RangerBaseModelObject)service); + + service.setPolicyVersion(new Long(0)); + service.setPolicyUpdateTime(service.getCreateTime()); + } + + protected void postCreate(RangerBaseModelObject obj) throws Exception { + if(obj instanceof RangerServiceDef) { + updateTagServiceDefForAddingAccessTypes((RangerServiceDef)obj); + } + } + + protected void preUpdate(RangerBaseModelObject obj) throws Exception { + if(obj.getId() == null) { + obj.setId(new Long(0)); + } + + if(obj.getGuid() == null) { + obj.setGuid(UUID.randomUUID().toString()); + } + + if(obj.getCreateTime() == null) { + obj.setCreateTime(new Date()); + } + + Long version = obj.getVersion(); + + if(version == null) { + version = new Long(1); + } else { + version = new Long(version.longValue() + 1); + } + + obj.setVersion(version); + obj.setUpdateTime(new Date()); + } + + protected void postUpdate(RangerBaseModelObject obj) throws Exception { + if(obj instanceof RangerServiceDef) { + updateTagServiceDefForUpdatingAccessTypes((RangerServiceDef) obj); + } + } + + protected void preDelete(RangerBaseModelObject obj) throws Exception { + // TODO: + } + + protected void postDelete(RangerBaseModelObject obj) throws Exception { + if(obj instanceof RangerServiceDef) { + updateTagServiceDefForDeletingAccessTypes(((RangerServiceDef) obj).getName()); + } + } + + protected long getMaxId(List<? extends RangerBaseModelObject> objs) { + long ret = -1; + + if (objs != null) { + for (RangerBaseModelObject obj : objs) { + if (obj.getId() > ret) { + ret = obj.getId(); + } + } + } + return ret; + } + + private void updateTagServiceDefForAddingAccessTypes(RangerServiceDef serviceDef) throws Exception { + if (serviceDef.getName().equals(EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME)) { + return; + } + + RangerServiceDef tagServiceDef = null; + try { + tagServiceDef = this.getServiceDef(EmbeddedServiceDefsUtil.instance().getTagServiceDefId()); + } catch (Exception e) { + LOG.error("AbstractServiceStore.updateTagServiceDefForAddingAccessTypes -- Could not find TAG ServiceDef.. ", e); + throw e; + } + List<RangerServiceDef.RangerAccessTypeDef> accessTypes = new ArrayList<RangerServiceDef.RangerAccessTypeDef>(); + + for (RangerServiceDef.RangerAccessTypeDef accessType : serviceDef.getAccessTypes()) { + RangerServiceDef.RangerAccessTypeDef newAccessType = new RangerServiceDef.RangerAccessTypeDef(accessType); + + newAccessType.setItemId(serviceDef.getId()*(MAX_ACCESS_TYPES_IN_SERVICE_DEF + 1) + accessType.getItemId()); + newAccessType.setName(serviceDef.getName() + ":" + accessType.getName()); + accessTypes.add(newAccessType); + } + + tagServiceDef.getAccessTypes().addAll(accessTypes); + try { + updateServiceDef(tagServiceDef); + LOG.info("AbstractServiceStore.updateTagServiceDefForAddingAccessTypes -- updated TAG service def with " + serviceDef.getName() + " access types"); + } catch (Exception e) { + LOG.error("AbstractServiceStore.updateTagServiceDefForAddingAccessTypes -- Failed to update TAG ServiceDef.. ", e); + throw e; + } + } + + private void updateTagServiceDefForUpdatingAccessTypes(RangerServiceDef serviceDef) throws Exception { + if (serviceDef.getName().equals(EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME)) { + return; + } + + String serviceDefName = serviceDef.getName(); + + RangerServiceDef tagServiceDef = null; + try { + tagServiceDef = this.getServiceDef(EmbeddedServiceDefsUtil.instance().getTagServiceDefId()); + } catch (Exception e) { + LOG.error("AbstractServiceStore.updateTagServiceDefForDeletingAccessTypes -- Could not find TAG ServiceDef.. ", e); + throw e; + } + + List<RangerServiceDef.RangerAccessTypeDef> tagSvcDefAccessTypes = new ArrayList<RangerServiceDef.RangerAccessTypeDef>(); + + for (RangerServiceDef.RangerAccessTypeDef accessType : tagServiceDef.getAccessTypes()) { + if (accessType.getName().startsWith(serviceDefName + ":")) { + RangerServiceDef.RangerAccessTypeDef tagSvcDefAccessType = new RangerServiceDef.RangerAccessTypeDef(accessType); + tagSvcDefAccessTypes.add(tagSvcDefAccessType); + } + } + + List<RangerServiceDef.RangerAccessTypeDef> svcDefAccessTypes = new ArrayList<RangerServiceDef.RangerAccessTypeDef>(); + + for (RangerServiceDef.RangerAccessTypeDef accessType : serviceDef.getAccessTypes()) { + RangerServiceDef.RangerAccessTypeDef svcDefAccessType = new RangerServiceDef.RangerAccessTypeDef(accessType); + svcDefAccessType.setItemId(serviceDef.getId()*(MAX_ACCESS_TYPES_IN_SERVICE_DEF + 1) + accessType.getItemId()); + svcDefAccessType.setName(serviceDefName + ":" + accessType.getName()); + svcDefAccessTypes.add(svcDefAccessType); + } + + tagServiceDef.getAccessTypes().removeAll(tagSvcDefAccessTypes); + tagServiceDef.getAccessTypes().addAll(svcDefAccessTypes); + + try { + updateServiceDef(tagServiceDef); + LOG.info("AbstractServiceStore.updateTagServiceDefForUpdatingAccessTypes -- updated TAG service def with " + serviceDefName + " access types"); + } catch (Exception e) { + LOG.error("AbstractServiceStore.updateTagServiceDefForUpdatingAccessTypes -- Failed to update TAG ServiceDef.. ", e); + throw e; + } + + } + + private void updateTagServiceDefForDeletingAccessTypes(String serviceDefName) throws Exception { + if (serviceDefName.equals(EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME)) { + return; + } + + RangerServiceDef tagServiceDef = null; + try { + tagServiceDef = this.getServiceDef(EmbeddedServiceDefsUtil.instance().getTagServiceDefId()); + } catch (Exception e) { + LOG.error("AbstractServiceStore.updateTagServiceDefForDeletingAccessTypes -- Could not find TAG ServiceDef.. ", e); + throw e; + } + List<RangerServiceDef.RangerAccessTypeDef> accessTypes = new ArrayList<RangerServiceDef.RangerAccessTypeDef>(); + + for (RangerServiceDef.RangerAccessTypeDef accessType : tagServiceDef.getAccessTypes()) { + if (accessType.getName().startsWith(serviceDefName + ":")) { + RangerServiceDef.RangerAccessTypeDef newAccessType = new RangerServiceDef.RangerAccessTypeDef(accessType); + accessTypes.add(newAccessType); + } + } + + tagServiceDef.getAccessTypes().removeAll(accessTypes); + try { + updateServiceDef(tagServiceDef); + LOG.info("AbstractServiceStore.updateTagServiceDefForDeletingAccessTypes -- updated TAG service def with " + serviceDefName + " access types"); + } catch (Exception e) { + LOG.error("AbstractServiceStore.updateTagServiceDefForDeletingAccessTypes -- Failed to update TAG ServiceDef.. ", e); + throw e; + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/9578f94c/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractTagStore.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractTagStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractTagStore.java new file mode 100644 index 0000000..9be733d --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractTagStore.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.store; + +import org.apache.ranger.plugin.model.RangerBaseModelObject; +import org.apache.ranger.plugin.model.RangerService; + +import java.util.Date; +import java.util.List; +import java.util.UUID; + +public abstract class AbstractTagStore implements TagStore { + protected void preCreate(RangerBaseModelObject obj) throws Exception { + obj.setId(new Long(0)); + obj.setGuid(UUID.randomUUID().toString()); + obj.setCreateTime(new Date()); + obj.setUpdateTime(obj.getCreateTime()); + obj.setVersion(new Long(1)); + } + + protected void postCreate(RangerBaseModelObject obj) throws Exception { + } + + protected void preUpdate(RangerBaseModelObject obj) throws Exception { + if(obj.getId() == null) { + obj.setId(new Long(0)); + } + + if(obj.getGuid() == null) { + obj.setGuid(UUID.randomUUID().toString()); + } + + if(obj.getCreateTime() == null) { + obj.setCreateTime(new Date()); + } + + Long version = obj.getVersion(); + + if(version == null) { + version = new Long(1); + } else { + version = new Long(version.longValue() + 1); + } + + obj.setVersion(version); + obj.setUpdateTime(new Date()); + } + + protected void postUpdate(RangerBaseModelObject obj) throws Exception { + } + + protected void preDelete(RangerBaseModelObject obj) throws Exception { + // TODO: + } + + protected void postDelete(RangerBaseModelObject obj) throws Exception { + } + + protected long getMaxId(List<? extends RangerBaseModelObject> objs) { + long ret = -1; + + if (objs != null) { + for (RangerBaseModelObject obj : objs) { + if (obj.getId() > ret) { + ret = obj.getId(); + } + } + } + return ret; + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/9578f94c/agents-common/src/main/java/org/apache/ranger/plugin/store/EmbeddedServiceDefsUtil.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/EmbeddedServiceDefsUtil.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/EmbeddedServiceDefsUtil.java index 2930606..dcf6288 100755 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/EmbeddedServiceDefsUtil.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/EmbeddedServiceDefsUtil.java @@ -43,6 +43,7 @@ public class EmbeddedServiceDefsUtil { private static final Log LOG = LogFactory.getLog(EmbeddedServiceDefsUtil.class); + public static final String EMBEDDED_SERVICEDEF_TAG_NAME = "_tag_"; public static final String EMBEDDED_SERVICEDEF_HDFS_NAME = "hdfs"; public static final String EMBEDDED_SERVICEDEF_HBASE_NAME = "hbase"; public static final String EMBEDDED_SERVICEDEF_HIVE_NAME = "hive"; @@ -55,6 +56,7 @@ public class EmbeddedServiceDefsUtil { private static EmbeddedServiceDefsUtil instance = new EmbeddedServiceDefsUtil(); private boolean createEmbeddedServiceDefs = true; + private RangerServiceDef tagServiceDef = null; private RangerServiceDef hdfsServiceDef = null; private RangerServiceDef hBaseServiceDef = null; private RangerServiceDef hiveServiceDef = null; @@ -82,6 +84,7 @@ public class EmbeddedServiceDefsUtil { gsonBuilder = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z").setPrettyPrinting().create(); + tagServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_TAG_NAME); hdfsServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_HDFS_NAME); hBaseServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_HBASE_NAME); hiveServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_HIVE_NAME); @@ -89,6 +92,9 @@ public class EmbeddedServiceDefsUtil { stormServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_STORM_NAME); yarnServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_YARN_NAME); kmsServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_KMS_NAME); + + // Ensure that tag service def is updated with access types of all service defs + store.updateTagServiceDefForAccessTypes(); } catch(Throwable excp) { LOG.fatal("EmbeddedServiceDefsUtil.init(): failed", excp); } @@ -96,6 +102,8 @@ public class EmbeddedServiceDefsUtil { LOG.info("<== EmbeddedServiceDefsUtil.init()"); } + public long getTagServiceDefId() { return getId(tagServiceDef); } + public long getHdfsServiceDefId() { return getId(hdfsServiceDef); } @@ -141,7 +149,8 @@ public class EmbeddedServiceDefsUtil { ret = loadEmbeddedServiceDef(serviceDefName); LOG.info("creating embedded service-def " + serviceDefName); - store.createServiceDef(ret); + ret = store.createServiceDef(ret); + LOG.info("created embedded service-def " + serviceDefName); } } catch(Exception excp) { LOG.fatal("EmbeddedServiceDefsUtil.getOrCreateServiceDef(): failed to load/create serviceType " + serviceDefName, excp); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/9578f94c/agents-common/src/main/java/org/apache/ranger/plugin/store/PList.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/PList.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/PList.java new file mode 100644 index 0000000..8c3e0a9 --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/PList.java @@ -0,0 +1,189 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.store; + +import java.util.List; + +public abstract class PList implements java.io.Serializable { + private static final long serialVersionUID = 1L; + + /** + * Start index for the result + */ + protected int startIndex; + /** + * Page size used for the result + */ + protected int pageSize; + /** + * Total records in the database for the given search conditions + */ + protected long totalCount; + /** + * Number of rows returned for the search condition + */ + protected int resultSize; + /** + * Sort type. Either desc or asc + */ + protected String sortType; + /** + * Comma seperated list of the fields for sorting + */ + protected String sortBy; + + protected long queryTimeMS = System.currentTimeMillis(); + + + /** + * Default constructor. This will set all the attributes to default value. + */ + public PList() { + } + + /** + * Initialize with existing list + * + * @param objectList + */ + public PList(@SuppressWarnings("rawtypes") List objectList) { + int size = 0; + if (objectList != null) { + size = objectList.size(); + } + + startIndex = 0; + pageSize = size; + totalCount = size; + resultSize = size; + sortType = null; + sortBy = null; + } + + abstract public int getListSize(); + + abstract public List<?> getList(); + + /** + * This method sets the value to the member attribute <b>startIndex</b>. You + * cannot set null to the attribute. + * + * @param startIndex + * Value to set member attribute <b>startIndex</b> + */ + public void setStartIndex(int startIndex) { + this.startIndex = startIndex; + } + public int getStartIndex() { return startIndex; } + + + /** + * This method sets the value to the member attribute <b>pageSize</b>. You + * cannot set null to the attribute. + * + * @param pageSize + * Value to set member attribute <b>pageSize</b> + */ + public void setPageSize(int pageSize) { + this.pageSize = pageSize; + } + public int getPageSize() { return pageSize; } + + + /** + * This method sets the value to the member attribute <b>totalCount</b>. You + * cannot set null to the attribute. + * + * @param totalCount + * Value to set member attribute <b>totalCount</b> + */ + public void setTotalCount(long totalCount) { + this.totalCount = totalCount; + } + public long getTotalCount() { return totalCount; } + + + + /** + * This method sets the value to the member attribute <b>resultSize</b>. You + * cannot set null to the attribute. + * + * @param resultSize + * Value to set member attribute <b>resultSize</b> + */ + public void setResultSize(int resultSize) { + this.resultSize = resultSize; + } + + /** + * Returns the value for the member attribute <b>resultSize</b> + * + * @return int - value of member attribute <b>resultSize</b>. + */ + public int getResultSize() { + return getListSize(); + } + + /** + * This method sets the value to the member attribute <b>sortType</b>. You + * cannot set null to the attribute. + * + * @param sortType + * Value to set member attribute <b>sortType</b> + */ + public void setSortType(String sortType) { + this.sortType = sortType; + } + public String getSortType() { return sortType; } + + + + /** + * This method sets the value to the member attribute <b>sortBy</b>. You + * cannot set null to the attribute. + * + * @param sortBy + * Value to set member attribute <b>sortBy</b> + */ + public void setSortBy(String sortBy) { + this.sortBy = sortBy; + } + public String getSortBy() { return sortBy; } + + + + + + + + /* + * (non-Javadoc) + * + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + return "PList [startIndex=" + startIndex + ", pageSize=" + + pageSize + ", totalCount=" + totalCount + + ", resultSize=" + resultSize + ", sortType=" + + sortType + ", sortBy=" + sortBy + ", queryTimeMS=" + + queryTimeMS + "]"; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/9578f94c/agents-common/src/main/java/org/apache/ranger/plugin/store/RangerPolicyPaginatedList.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/RangerPolicyPaginatedList.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/RangerPolicyPaginatedList.java new file mode 100644 index 0000000..9573886 --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/RangerPolicyPaginatedList.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.store; + +import org.apache.ranger.plugin.model.RangerPolicy; +import org.apache.ranger.plugin.model.RangerServiceDef; + +import java.util.ArrayList; +import java.util.List; + +public class RangerPolicyPaginatedList extends PList { + private static final long serialVersionUID = 1L; + + List<RangerPolicy> policies = new ArrayList<RangerPolicy>(); + + public RangerPolicyPaginatedList() { + super(); + } + + public RangerPolicyPaginatedList(List<RangerPolicy> objList) { + super(objList); + this.policies = objList; + } + + public List<RangerPolicy> getPolicies() { + return policies; + } + + public void setPolicies(List<RangerPolicy> policies) { + this.policies = policies; + } + + @Override + public int getListSize() { + if (policies != null) { + return policies.size(); + } + return 0; + } + + @Override + public List<?> getList() { + return policies; + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/9578f94c/agents-common/src/main/java/org/apache/ranger/plugin/store/RangerServiceDefPaginatedList.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/RangerServiceDefPaginatedList.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/RangerServiceDefPaginatedList.java new file mode 100644 index 0000000..7366ac8 --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/RangerServiceDefPaginatedList.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.store; + +import org.apache.ranger.plugin.model.RangerServiceDef; + +import java.util.ArrayList; +import java.util.List; + +public class RangerServiceDefPaginatedList extends PList { + private static final long serialVersionUID = 1L; + + List<RangerServiceDef> serviceDefs = new ArrayList<RangerServiceDef>(); + + public RangerServiceDefPaginatedList() { + super(); + } + + public RangerServiceDefPaginatedList(List<RangerServiceDef> objList) { + super(objList); + this.serviceDefs = objList; + } + + public List<RangerServiceDef> getServiceDefs() { + return serviceDefs; + } + + public void setServiceDefs(List<RangerServiceDef> serviceDefs) { + this.serviceDefs = serviceDefs; + } + + @Override + public int getListSize() { + if (serviceDefs != null) { + return serviceDefs.size(); + } + return 0; + } + + @Override + public List<?> getList() { + return serviceDefs; + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/9578f94c/agents-common/src/main/java/org/apache/ranger/plugin/store/RangerServicePaginatedList.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/RangerServicePaginatedList.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/RangerServicePaginatedList.java new file mode 100644 index 0000000..6194bb8 --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/RangerServicePaginatedList.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.store; + +import org.apache.ranger.plugin.model.RangerService; +import org.apache.ranger.plugin.model.RangerServiceDef; + +import java.util.ArrayList; +import java.util.List; + +public class RangerServicePaginatedList extends PList { + private static final long serialVersionUID = 1L; + + List<RangerService> services = new ArrayList<RangerService>(); + + public RangerServicePaginatedList() { + super(); + } + + public RangerServicePaginatedList(List<RangerService> objList) { + super(objList); + this.services = objList; + } + + public List<RangerService> getServices() { + return services; + } + + public void setServices(List<RangerService> services) { + this.services = services; + } + + @Override + public int getListSize() { + if (services != null) { + return services.size(); + } + return 0; + } + + @Override + public List<?> getList() { + return services; + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/9578f94c/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java index b998e93..0915cf5 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java @@ -35,6 +35,10 @@ public interface ServiceStore { RangerServiceDef updateServiceDef(RangerServiceDef serviceDef) throws Exception; void deleteServiceDef(Long id) throws Exception; + void deleteServiceDef(Long id, Boolean forceDelete) throws Exception; + + void updateTagServiceDefForAccessTypes() throws Exception; + RangerServiceDef getServiceDef(Long id) throws Exception; @@ -42,6 +46,7 @@ public interface ServiceStore { List<RangerServiceDef> getServiceDefs(SearchFilter filter) throws Exception; + RangerServiceDefPaginatedList getPaginatedServiceDefs(SearchFilter filter) throws Exception; RangerService createService(RangerService service) throws Exception; @@ -55,6 +60,7 @@ public interface ServiceStore { List<RangerService> getServices(SearchFilter filter) throws Exception; + RangerServicePaginatedList getPaginatedServices(SearchFilter filter) throws Exception; RangerPolicy createPolicy(RangerPolicy policy) throws Exception; @@ -66,11 +72,25 @@ public interface ServiceStore { List<RangerPolicy> getPolicies(SearchFilter filter) throws Exception; + RangerPolicyPaginatedList getPaginatedPolicies(SearchFilter filter) throws Exception; + List<RangerPolicy> getPoliciesByResourceSignature(String serviceName, String policySignature, Boolean isPolicyEnabled) throws Exception; List<RangerPolicy> getServicePolicies(Long serviceId, SearchFilter filter) throws Exception; + RangerPolicyPaginatedList getPaginatedServicePolicies(Long serviceId, SearchFilter filter) throws Exception; + List<RangerPolicy> getServicePolicies(String serviceName, SearchFilter filter) throws Exception; + RangerPolicyPaginatedList getPaginatedServicePolicies(String serviceName, SearchFilter filter) throws Exception; + ServicePolicies getServicePoliciesIfUpdated(String serviceName, Long lastKnownVersion) throws Exception; -} + + RangerPolicy getPolicyFromEventTime(String eventTimeStr, Long policyId); + + RangerPolicy getPolicyForVersionNumber(Long policyId, Integer versionNo); + + String getPolicyForVersionNumber(Long policyId); + + + } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/9578f94c/agents-common/src/main/java/org/apache/ranger/plugin/store/TagPredicateUtil.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/TagPredicateUtil.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/TagPredicateUtil.java new file mode 100644 index 0000000..b880179 --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/TagPredicateUtil.java @@ -0,0 +1,212 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.store; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.Predicate; +import org.apache.commons.collections.PredicateUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.ranger.plugin.model.RangerBaseModelObject; +import org.apache.ranger.plugin.model.RangerResource; +import org.apache.ranger.plugin.model.RangerTagDef; +import org.apache.ranger.plugin.util.SearchFilter; + +import java.util.ArrayList; +import java.util.List; + +public class TagPredicateUtil extends AbstractPredicateUtil { + private TagStore tagStore = null; + public TagPredicateUtil(TagStore tagStore) { + super(); + this.tagStore = tagStore; + } + + @Override + public void addPredicates(SearchFilter filter, List<Predicate> predicates) { + super.addPredicates(filter, predicates); + + addPredicateForTagDefId(filter.getParam(SearchFilter.TAG_DEF_ID), predicates); + addPredicateForTagDefName(filter.getParam(SearchFilter.TAG_DEF_NAME), predicates); + + addPredicateForTagResourceServiceName(filter.getParam(SearchFilter.TAG_RESOURCE_SERVICE_NAME), predicates); + addPredicateForTagResourceServiceType(filter.getParam(SearchFilter.TAG_RESOURCE_SERVICE_TYPE), predicates); + + addPredicateForTagResourceId(filter.getParam(SearchFilter.TAG_RESOURCE_ID), predicates); + } + + private Predicate addPredicateForTagDefId(final String id, List<Predicate> predicates) { + if (StringUtils.isEmpty(id)) { + return null; + } + + Predicate ret = new Predicate() { + @Override + public boolean evaluate(Object object) { + + boolean ret = false; + + if (object == null) { + return ret; + } + + if (object instanceof RangerTagDef) { + RangerTagDef tagDef = (RangerTagDef) object; + + ret = StringUtils.equals(id, tagDef.getId().toString()); + } + + return ret; + } + }; + + if (predicates != null) { + predicates.add(ret); + } + + return ret; + } + + private Predicate addPredicateForTagDefName(final String name, List<Predicate> predicates) { + if (name == null || StringUtils.isEmpty(name)) { + return null; + } + + Predicate ret = new Predicate() { + @Override + public boolean evaluate(Object object) { + + boolean ret = false; + + if (object == null) { + return ret; + } + + if (object instanceof RangerTagDef) { + RangerTagDef tagDef = (RangerTagDef) object; + + ret = StringUtils.equals(name, tagDef.getName()); + } + + return ret; + } + }; + + if (predicates != null) { + predicates.add(ret); + } + + return ret; + } + + private Predicate addPredicateForTagResourceServiceName(final String name, List<Predicate> predicates) { + if (name == null || StringUtils.isEmpty(name)) { + return null; + } + + Predicate ret = new Predicate() { + @Override + public boolean evaluate(Object object) { + + boolean ret = false; + + if (object == null) { + return ret; + } + + if (object instanceof RangerResource) { + RangerResource rangerResource = (RangerResource) object; + + ret = StringUtils.equals(name, rangerResource.getTagServiceName()); + } + + return ret; + } + }; + + if (predicates != null) { + predicates.add(ret); + } + + return ret; + } + + private Predicate addPredicateForTagResourceServiceType(final String type, List<Predicate> predicates) { + if (type == null || StringUtils.isEmpty(type)) { + return null; + } + + Predicate ret = new Predicate() { + @Override + public boolean evaluate(Object object) { + + boolean ret = false; + + if (object == null) { + return ret; + } + + if (object instanceof RangerResource) { + RangerResource rangerResource = (RangerResource) object; + + ret = StringUtils.equals(type, rangerResource.getServiceType()); + } + + return ret; + } + }; + + if (predicates != null) { + predicates.add(ret); + } + + return ret; + } + private Predicate addPredicateForTagResourceId(final String id, List<Predicate> predicates) { + if (StringUtils.isEmpty(id)) { + return null; + } + + Predicate ret = new Predicate() { + @Override + public boolean evaluate(Object object) { + + boolean ret = false; + + if (object == null) { + return ret; + } + + if (object instanceof RangerResource) { + RangerResource rangerResource = (RangerResource) object; + + ret = StringUtils.equals(id, rangerResource.getId().toString()); + } + + return ret; + } + }; + + if (predicates != null) { + predicates.add(ret); + } + + return ret; + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/9578f94c/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java new file mode 100644 index 0000000..87e2b6f --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.store; + +import org.apache.ranger.plugin.model.RangerResource; +import org.apache.ranger.plugin.model.RangerTagDef; +import org.apache.ranger.plugin.util.SearchFilter; + +import java.util.List; + +/** + * Interface to backing store for the top-level TAG model objects + */ + +public interface TagStore { + void init() throws Exception; + + RangerTagDef createTagDef(RangerTagDef tagDef) throws Exception; + + RangerTagDef updateTagDef(RangerTagDef TagDef) throws Exception; + + void deleteTagDef(String name) throws Exception; + + RangerTagDef getTagDef(String name) throws Exception; + + RangerTagDef getTagDefById(Long id) throws Exception; + + List<RangerTagDef> getTagDefs(SearchFilter filter) throws Exception; + + + RangerResource createResource(RangerResource resource) throws Exception; + + RangerResource updateResource(RangerResource resource) throws Exception; + + void deleteResource(Long id) throws Exception; + + RangerResource getResource(Long id) throws Exception; + + List<RangerResource> getResources(String tagServiceName, String serviceType) throws Exception; + + List<RangerResource> getResources(SearchFilter filter) throws Exception; +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/9578f94c/agents-common/src/main/java/org/apache/ranger/plugin/store/file/BaseFileStore.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/file/BaseFileStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/file/BaseFileStore.java deleted file mode 100644 index 6ce2589..0000000 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/file/BaseFileStore.java +++ /dev/null @@ -1,395 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.ranger.plugin.store.file; - -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.OutputStreamWriter; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.UUID; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.PathFilter; -import org.apache.ranger.plugin.model.RangerBaseModelObject; -import org.apache.ranger.plugin.model.RangerPolicy; -import org.apache.ranger.plugin.model.RangerService; -import org.apache.ranger.plugin.model.RangerServiceDef; - -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; - -public class BaseFileStore { - private static final Log LOG = LogFactory.getLog(BaseFileStore.class); - - private Gson gsonBuilder = null; - private String dataDir = null; - - protected static final String FILE_PREFIX_SERVICE_DEF = "ranger-servicedef-"; - protected static final String FILE_PREFIX_SERVICE = "ranger-service-"; - protected static final String FILE_PREFIX_POLICY = "ranger-policy-"; - protected static final String FILE_SUFFIX_JSON = ".json"; - - protected void initStore(String dataDir) { - this.dataDir = dataDir; - - try { - gsonBuilder = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z").setPrettyPrinting().create(); - } catch(Throwable excp) { - LOG.fatal("BaseFileStore.init(): failed to create GsonBuilder object", excp); - } - } - - protected String getDataDir() { - return dataDir; - } - - protected String getServiceDefFile(Long id) { - String filePath = dataDir + Path.SEPARATOR + FILE_PREFIX_SERVICE_DEF + id + FILE_SUFFIX_JSON; - - return filePath; - } - - protected String getServiceFile(Long id) { - String filePath = dataDir + Path.SEPARATOR + FILE_PREFIX_SERVICE + id + FILE_SUFFIX_JSON; - - return filePath; - } - - protected String getPolicyFile(Long serviceId, Long policyId) { - String filePath = dataDir + Path.SEPARATOR + FILE_PREFIX_POLICY + serviceId + "-" + policyId + FILE_SUFFIX_JSON; - - return filePath; - } - - protected <T> T loadFromResource(String resource, Class<T> cls) throws Exception { - if(LOG.isDebugEnabled()) { - LOG.debug("==> BaseFileStore.loadFromResource(" + resource + ")"); - } - - InputStream inStream = this.getClass().getResourceAsStream(resource); - - T ret = loadFromStream(inStream, cls); - - if(LOG.isDebugEnabled()) { - LOG.debug("<== BaseFileStore.loadFromResource(" + resource + "): " + ret); - } - - return ret; - } - - protected <T> T loadFromStream(InputStream inStream, Class<T> cls) throws Exception { - if(LOG.isDebugEnabled()) { - LOG.debug("==> BaseFileStore.loadFromStream()"); - } - - InputStreamReader reader = new InputStreamReader(inStream); - - T ret = gsonBuilder.fromJson(reader, cls); - - if(LOG.isDebugEnabled()) { - LOG.debug("<== BaseFileStore.loadFromStream(): " + ret); - } - - return ret; - } - - protected <T> T loadFromFile(Path filePath, Class<T> cls) throws Exception { - if(LOG.isDebugEnabled()) { - LOG.debug("==> BaseFileStore.loadFromFile(" + filePath + ")"); - } - - T ret = null; - InputStreamReader reader = null; - - try { - FileSystem fileSystem = getFileSystem(filePath); - FSDataInputStream inStream = fileSystem.open(filePath); - - ret = loadFromStream(inStream, cls); - } finally { - close(reader); - } - - if(LOG.isDebugEnabled()) { - LOG.debug("<== BaseFileStore.loadFromFile(" + filePath + "): " + ret); - } - - return ret; - } - - protected <T> List<T> loadFromDir(Path dirPath, final String filePrefix, Class<T> cls) throws Exception { - if(LOG.isDebugEnabled()) { - LOG.debug("==> BaseFileStore.loadFromDir()"); - } - - List<T> ret = new ArrayList<T>(); - - try { - FileSystem fileSystem = getFileSystem(dirPath); - - if(fileSystem.exists(dirPath) && fileSystem.isDirectory(dirPath)) { - PathFilter filter = new PathFilter() { - @Override - public boolean accept(Path path) { - return path.getName().startsWith(filePrefix) && - path.getName().endsWith(FILE_SUFFIX_JSON); - } - }; - - FileStatus[] sdFiles = fileSystem.listStatus(dirPath, filter); - - if(sdFiles != null) { - for(FileStatus sdFile : sdFiles) { - T obj = loadFromFile(sdFile.getPath(), cls); - - if(obj != null) { - ret.add(obj); - } - } - } - } else { - LOG.error(dirPath + ": does not exists or not a directory"); - } - } catch(IOException excp) { - LOG.warn("error loading service-def in directory " + dirPath, excp); - } - - if(LOG.isDebugEnabled()) { - LOG.debug("<== BaseFileStore.loadFromDir(): count=" + (ret == null ? 0 : ret.size())); - } - - return ret; - } - - protected <T> T saveToFile(T obj, Path filePath, boolean overWrite) throws Exception { - if(LOG.isDebugEnabled()) { - LOG.debug("==> BaseFileStore.saveToFile(" + filePath + ")"); - } - - OutputStreamWriter writer = null; - - try { - FileSystem fileSystem = getFileSystem(filePath); - FSDataOutputStream outStream = fileSystem.create(filePath, overWrite); - - writer = new OutputStreamWriter(outStream); - - gsonBuilder.toJson(obj, writer); - } finally { - close(writer); - } - - if(LOG.isDebugEnabled()) { - LOG.debug("<== BaseFileStore.saveToFile(" + filePath + "): " + obj); - } - - return obj; - } - - protected boolean deleteFile(Path filePath) throws Exception { - LOG.debug("==> BaseFileStore.deleteFile(" + filePath + ")"); - - FileSystem fileSystem = getFileSystem(filePath); - - boolean ret = false; - - if(fileSystem.exists(filePath)) { - ret = fileSystem.delete(filePath, false); - } else { - ret = true; // nothing to delete - } - - LOG.debug("<== BaseFileStore.deleteFile(" + filePath + "): " + ret); - - return ret; - } - - protected boolean renamePath(Path oldPath, Path newPath) throws Exception { - if(LOG.isDebugEnabled()) { - LOG.debug("==> BaseFileStore.renamePath(" + oldPath + "," + newPath + ")"); - } - - FileSystem fileSystem = getFileSystem(oldPath); - - boolean ret = false; - - if(fileSystem.exists(oldPath)) { - if(! fileSystem.exists(newPath)) { - ret = fileSystem.rename(oldPath, newPath); - } else { - LOG.warn("target of rename '" + newPath + "' already exists"); - } - } - - if(LOG.isDebugEnabled()) { - LOG.debug("<== BaseFileStore.renamePath(" + oldPath + "," + newPath + "): " + ret); - } - - return ret; - } - - protected RangerServiceDef saveToFile(RangerServiceDef serviceDef, boolean overWrite) throws Exception { - if(LOG.isDebugEnabled()) { - LOG.debug("==> BaseFileStore.saveToFile(" + serviceDef + "," + overWrite + ")"); - } - - Path filePath = new Path(getServiceDefFile(serviceDef.getId())); - - RangerServiceDef ret = saveToFile(serviceDef, filePath, overWrite); - - if(LOG.isDebugEnabled()) { - LOG.debug("<== BaseFileStore.saveToFile(" + serviceDef + "," + overWrite + "): "); - } - - return ret; - } - - protected RangerService saveToFile(RangerService service, boolean overWrite) throws Exception { - Path filePath = new Path(getServiceFile(service.getId())); - - RangerService ret = saveToFile(service, filePath, overWrite); - - return ret; - } - - protected RangerPolicy saveToFile(RangerPolicy policy, long serviceId, boolean overWrite) throws Exception { - Path filePath = new Path(getPolicyFile(serviceId, policy.getId())); - - RangerPolicy ret = saveToFile(policy, filePath, overWrite); - - return ret; - } - - protected long getMaxId(List<? extends RangerBaseModelObject> objs) { - long ret = -1; - - if(objs != null) { - for(RangerBaseModelObject obj : objs) { - if(obj.getId() > ret) { - ret = obj.getId(); - } - } - } - - return ret; - } - protected FileSystem getFileSystem(Path filePath) throws Exception { - Configuration conf = new Configuration(); - FileSystem fileSystem = filePath.getFileSystem(conf); - - return fileSystem; - } - - protected void close(FileSystem fs) { - if(fs != null) { - try { - fs.close(); - } catch(IOException excp) { - // ignore - } - } - } - - protected void close(InputStreamReader reader) { - if(reader != null) { - try { - reader.close(); - } catch(IOException excp) { - // ignore - } - } - } - - protected void close(OutputStreamWriter writer) { - if(writer != null) { - try { - writer.close(); - } catch(IOException excp) { - // ignore - } - } - } - - protected void preCreate(RangerBaseModelObject obj) { - obj.setId(new Long(0)); - obj.setGuid(UUID.randomUUID().toString()); - obj.setCreateTime(new Date()); - obj.setUpdateTime(obj.getCreateTime()); - obj.setVersion(new Long(1)); - } - - protected void preCreate(RangerService service) { - preCreate((RangerBaseModelObject)service); - - service.setPolicyVersion(new Long(0)); - service.setPolicyUpdateTime(service.getCreateTime()); - } - - protected void postCreate(RangerBaseModelObject obj) { - // TODO: - } - - protected void preUpdate(RangerBaseModelObject obj) { - if(obj.getId() == null) { - obj.setId(new Long(0)); - } - - if(obj.getGuid() == null) { - obj.setGuid(UUID.randomUUID().toString()); - } - - if(obj.getCreateTime() == null) { - obj.setCreateTime(new Date()); - } - - Long version = obj.getVersion(); - - if(version == null) { - version = new Long(1); - } else { - version = new Long(version.longValue() + 1); - } - - obj.setVersion(version); - obj.setUpdateTime(new Date()); - } - - protected void postUpdate(RangerBaseModelObject obj) { - // TODO: - } - - protected void preDelete(RangerBaseModelObject obj) { - // TODO: - } - - protected void postDelete(RangerBaseModelObject obj) { - // TODO: - } -} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/9578f94c/agents-common/src/main/java/org/apache/ranger/plugin/store/file/FileStoreUtil.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/file/FileStoreUtil.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/file/FileStoreUtil.java new file mode 100644 index 0000000..50de24b --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/file/FileStoreUtil.java @@ -0,0 +1,317 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.store.file; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.UUID; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.PathFilter; +import org.apache.ranger.plugin.model.RangerBaseModelObject; +import org.apache.ranger.plugin.model.RangerPolicy; +import org.apache.ranger.plugin.model.RangerService; +import org.apache.ranger.plugin.model.RangerServiceDef; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; + +public class FileStoreUtil { + private static final Log LOG = LogFactory.getLog(FileStoreUtil.class); + + private Gson gsonBuilder = null; + private String dataDir = null; + + private static final String FILE_SUFFIX_JSON = ".json"; + + public void initStore(String dataDir) { + this.dataDir = dataDir; + + try { + gsonBuilder = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z").setPrettyPrinting().create(); + } catch(Throwable excp) { + LOG.fatal("FileStoreUtil.init(): failed to create GsonBuilder object", excp); + } + } + + public String getDataDir() { + return dataDir; + } + + public String getDataFile(String filePrefix, Long id) { + String filePath = dataDir + Path.SEPARATOR + filePrefix + id + FILE_SUFFIX_JSON; + + return filePath; + } + + public String getDataFile(String filePrefix, Long parentId, Long objectId) { + String filePath = dataDir + Path.SEPARATOR + filePrefix + parentId + "-" + objectId + FILE_SUFFIX_JSON; + + return filePath; + } + + public <T> T loadFromResource(String resource, Class<T> cls) throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> FileStoreUtil.loadFromResource(" + resource + ")"); + } + + InputStream inStream = this.getClass().getResourceAsStream(resource); + + T ret = loadFromStream(inStream, cls); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== FileStoreUtil.loadFromResource(" + resource + "): " + ret); + } + + return ret; + } + + public <T> T loadFromStream(InputStream inStream, Class<T> cls) throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> FileStoreUtil.loadFromStream()"); + } + + InputStreamReader reader = new InputStreamReader(inStream); + + T ret = gsonBuilder.fromJson(reader, cls); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== FileStoreUtil.loadFromStream(): " + ret); + } + + return ret; + } + + public <T> T loadFromFile(Path filePath, Class<T> cls) throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> FileStoreUtil.loadFromFile(" + filePath + ")"); + } + + T ret = null; + InputStreamReader reader = null; + + try { + FileSystem fileSystem = getFileSystem(filePath); + FSDataInputStream inStream = fileSystem.open(filePath); + + ret = loadFromStream(inStream, cls); + } finally { + close(reader); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== FileStoreUtil.loadFromFile(" + filePath + "): " + ret); + } + + return ret; + } + + public <T> List<T> loadFromDir(Path dirPath, final String filePrefix, Class<T> cls) throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> FileStoreUtil.loadFromDir()"); + } + + List<T> ret = new ArrayList<T>(); + + try { + FileSystem fileSystem = getFileSystem(dirPath); + + if(fileSystem.exists(dirPath) && fileSystem.isDirectory(dirPath)) { + PathFilter filter = new PathFilter() { + @Override + public boolean accept(Path path) { + return path.getName().startsWith(filePrefix) && + path.getName().endsWith(FILE_SUFFIX_JSON); + } + }; + + FileStatus[] sdFiles = fileSystem.listStatus(dirPath, filter); + + if(sdFiles != null) { + for(FileStatus sdFile : sdFiles) { + T obj = loadFromFile(sdFile.getPath(), cls); + + if(obj != null) { + ret.add(obj); + } + } + } + } else { + LOG.error(dirPath + ": does not exists or not a directory"); + } + } catch(IOException excp) { + LOG.warn("error loading service-def in directory " + dirPath, excp); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== FileStoreUtil.loadFromDir(): count=" + (ret == null ? 0 : ret.size())); + } + + return ret; + } + + public <T> T saveToFile(T obj, Path filePath, boolean overWrite) throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> FileStoreUtil.saveToFile(" + filePath + ")"); + } + + OutputStreamWriter writer = null; + + try { + FileSystem fileSystem = getFileSystem(filePath); + FSDataOutputStream outStream = fileSystem.create(filePath, overWrite); + + writer = new OutputStreamWriter(outStream); + + gsonBuilder.toJson(obj, writer); + } finally { + close(writer); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== FileStoreUtil.saveToFile(" + filePath + "): " + obj); + } + + return obj; + } + + public boolean deleteFile(Path filePath) throws Exception { + LOG.debug("==> FileStoreUtil.deleteFile(" + filePath + ")"); + + FileSystem fileSystem = getFileSystem(filePath); + + boolean ret = false; + + if(fileSystem.exists(filePath)) { + ret = fileSystem.delete(filePath, false); + } else { + ret = true; // nothing to delete + } + + LOG.debug("<== FileStoreUtil.deleteFile(" + filePath + "): " + ret); + + return ret; + } + + public boolean renamePath(Path oldPath, Path newPath) throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> FileStoreUtil.renamePath(" + oldPath + "," + newPath + ")"); + } + + FileSystem fileSystem = getFileSystem(oldPath); + + boolean ret = false; + + if(fileSystem.exists(oldPath)) { + if(! fileSystem.exists(newPath)) { + ret = fileSystem.rename(oldPath, newPath); + } else { + LOG.warn("target of rename '" + newPath + "' already exists"); + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== FileStoreUtil.renamePath(" + oldPath + "," + newPath + "): " + ret); + } + + return ret; + } + + public RangerServiceDef saveToFile(RangerServiceDef serviceDef, String filePrefix, boolean overWrite) throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> FileStoreUtil.saveToFile(" + serviceDef + "," + overWrite + ")"); + } + + Path filePath = new Path(getDataFile(filePrefix, serviceDef.getId())); + + RangerServiceDef ret = saveToFile(serviceDef, filePath, overWrite); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== FileStoreUtil.saveToFile(" + serviceDef + "," + overWrite + "): "); + } + + return ret; + } + + public RangerService saveToFile(RangerService service, String filePrefix,boolean overWrite) throws Exception { + Path filePath = new Path(getDataFile(filePrefix, service.getId())); + + RangerService ret = saveToFile(service, filePath, overWrite); + + return ret; + } + + public RangerPolicy saveToFile(RangerPolicy policy, String filePrefix, long serviceId, boolean overWrite) throws Exception { + Path filePath = new Path(getDataFile(filePrefix, serviceId, policy.getId())); + + RangerPolicy ret = saveToFile(policy, filePath, overWrite); + + return ret; + } + + public FileSystem getFileSystem(Path filePath) throws Exception { + Configuration conf = new Configuration(); + FileSystem fileSystem = filePath.getFileSystem(conf); + + return fileSystem; + } + + protected void close(FileSystem fs) { + if(fs != null) { + try { + fs.close(); + } catch(IOException excp) { + // ignore + } + } + } + + protected void close(InputStreamReader reader) { + if(reader != null) { + try { + reader.close(); + } catch(IOException excp) { + // ignore + } + } + } + + protected void close(OutputStreamWriter writer) { + if(writer != null) { + try { + writer.close(); + } catch(IOException excp) { + // ignore + } + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/9578f94c/agents-common/src/main/java/org/apache/ranger/plugin/store/file/ServiceFileStore.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/file/ServiceFileStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/file/ServiceFileStore.java index 2e469cd..8a34c80 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/file/ServiceFileStore.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/file/ServiceFileStore.java @@ -36,32 +36,39 @@ import org.apache.ranger.plugin.model.RangerBaseModelObject; import org.apache.ranger.plugin.model.RangerPolicy; import org.apache.ranger.plugin.model.RangerService; import org.apache.ranger.plugin.model.RangerServiceDef; +import org.apache.ranger.plugin.store.AbstractServiceStore; import org.apache.ranger.plugin.store.EmbeddedServiceDefsUtil; import org.apache.ranger.plugin.store.ServicePredicateUtil; import org.apache.ranger.plugin.store.ServiceStore; +import org.apache.ranger.plugin.store.file.FileStoreUtil; import org.apache.ranger.plugin.util.SearchFilter; import org.apache.ranger.plugin.util.ServicePolicies; - -public class ServiceFileStore extends BaseFileStore implements ServiceStore { +public class ServiceFileStore extends AbstractServiceStore { private static final Log LOG = LogFactory.getLog(ServiceFileStore.class); public static final String PROPERTY_SERVICE_FILE_STORE_DIR = "ranger.service.store.file.dir"; + protected static final String FILE_PREFIX_SERVICE_DEF = "ranger-servicedef-"; + protected static final String FILE_PREFIX_SERVICE = "ranger-service-"; + protected static final String FILE_PREFIX_POLICY = "ranger-policy-"; + private String dataDir = null; private long nextServiceDefId = 0; private long nextServiceId = 0; private long nextPolicyId = 0; private ServicePredicateUtil predicateUtil = null; + private FileStoreUtil fileStoreUtil = null; public ServiceFileStore() { if(LOG.isDebugEnabled()) { LOG.debug("==> ServiceFileStore.ServiceFileStore()"); } - dataDir = RangerConfiguration.getInstance().get(PROPERTY_SERVICE_FILE_STORE_DIR, "file:///etc/ranger/data"); + this.dataDir = RangerConfiguration.getInstance().get(PROPERTY_SERVICE_FILE_STORE_DIR, "file:///etc/ranger/data"); predicateUtil = new ServicePredicateUtil(this); + fileStoreUtil = new FileStoreUtil(); if(LOG.isDebugEnabled()) { LOG.debug("<== ServiceFileStore.ServiceFileStore()"); @@ -75,6 +82,8 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { this.dataDir = dataDir; predicateUtil = new ServicePredicateUtil(this); + fileStoreUtil = new FileStoreUtil(); + fileStoreUtil.initStore(dataDir); if(LOG.isDebugEnabled()) { LOG.debug("<== ServiceFileStore.ServiceFileStore()"); @@ -87,7 +96,7 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { LOG.debug("==> ServiceFileStore.init()"); } - super.initStore(dataDir); + fileStoreUtil.initStore(dataDir); EmbeddedServiceDefsUtil.instance().init(this); @@ -115,7 +124,7 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { serviceDef.setId(nextServiceDefId++); - ret = saveToFile(serviceDef, false); + ret = fileStoreUtil.saveToFile(serviceDef, FILE_PREFIX_SERVICE_DEF, false); postCreate(ret); } catch(Exception excp) { @@ -161,7 +170,7 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { preUpdate(existing); - ret = saveToFile(existing, true); + ret = fileStoreUtil.saveToFile(existing, FILE_PREFIX_SERVICE_DEF, true); postUpdate(ret); } catch(Exception excp) { @@ -194,9 +203,9 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { try { preDelete(existing); - Path filePath = new Path(getServiceDefFile(id)); + Path filePath = new Path(fileStoreUtil.getDataFile(FILE_PREFIX_SERVICE_DEF, id)); - deleteFile(filePath); + fileStoreUtil.deleteFile(filePath); postDelete(existing); } catch(Exception excp) { @@ -299,7 +308,7 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { service.setId(nextServiceId++); - ret = saveToFile(service, false); + ret = fileStoreUtil.saveToFile(service, FILE_PREFIX_SERVICE, false); postCreate(service); } catch(Exception excp) { @@ -344,7 +353,7 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { preUpdate(existing); - ret = saveToFile(existing, true); + ret = fileStoreUtil.saveToFile(existing, FILE_PREFIX_SERVICE, true); postUpdate(ret); @@ -375,13 +384,13 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { } try { - Path filePath = new Path(getServiceFile(id)); + Path filePath = new Path(fileStoreUtil.getDataFile(FILE_PREFIX_SERVICE, id)); preDelete(existing); handleServiceDelete(existing); - deleteFile(filePath); + fileStoreUtil.deleteFile(filePath); postDelete(existing); } catch(Exception excp) { @@ -402,9 +411,9 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { RangerService ret = null; try { - Path filePath = new Path(getServiceFile(id)); + Path filePath = new Path(fileStoreUtil.getDataFile(FILE_PREFIX_SERVICE, id)); - ret = loadFromFile(filePath, RangerService.class); + ret = fileStoreUtil.loadFromFile(filePath, RangerService.class); } catch(Exception excp) { LOG.error("ServiceFileStore.getService(" + id + "): failed to read service", excp); } @@ -489,7 +498,7 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { policy.setId(nextPolicyId++); - ret = saveToFile(policy, service.getId(), false); + ret = fileStoreUtil.saveToFile(policy, FILE_PREFIX_POLICY, service.getId(), false); handlePolicyUpdate(service); @@ -544,7 +553,7 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { preUpdate(existing); - ret = saveToFile(existing, service.getId(), true); + ret = fileStoreUtil.saveToFile(existing, FILE_PREFIX_POLICY, service.getId(), true); handlePolicyUpdate(service); @@ -581,9 +590,9 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { try { preDelete(existing); - Path filePath = new Path(getPolicyFile(service.getId(), existing.getId())); + Path filePath = new Path(fileStoreUtil.getDataFile(FILE_PREFIX_POLICY, service.getId(), existing.getId())); - deleteFile(filePath); + fileStoreUtil.deleteFile(filePath); handlePolicyUpdate(service); @@ -768,7 +777,7 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { preUpdate(policy); - saveToFile(policy, service.getId(), true); + fileStoreUtil.saveToFile(policy, FILE_PREFIX_POLICY, service.getId(), true); postUpdate(policy); } @@ -787,9 +796,9 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { preDelete(policy); - Path filePath = new Path(getPolicyFile(service.getId(), policy.getId())); + Path filePath = new Path(fileStoreUtil.getDataFile(FILE_PREFIX_POLICY, service.getId(), policy.getId())); - deleteFile(filePath); + fileStoreUtil.deleteFile(filePath); postDelete(policy); } @@ -812,7 +821,7 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { service.setPolicyVersion(policyVersion); service.setPolicyUpdateTime(new Date()); - saveToFile(service, true); + fileStoreUtil.saveToFile(service, FILE_PREFIX_SERVICE, true); } private RangerPolicy findPolicyByName(String serviceName, String policyName) throws Exception { @@ -853,7 +862,7 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { try { // load service definitions from file system - List<RangerServiceDef> sds = loadFromDir(new Path(getDataDir()), FILE_PREFIX_SERVICE_DEF, RangerServiceDef.class); + List<RangerServiceDef> sds = fileStoreUtil.loadFromDir(new Path(fileStoreUtil.getDataDir()), FILE_PREFIX_SERVICE_DEF, RangerServiceDef.class); if(sds != null) { for(RangerServiceDef sd : sds) { @@ -900,7 +909,7 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { List<RangerService> ret = null; try { - ret = loadFromDir(new Path(getDataDir()), FILE_PREFIX_SERVICE, RangerService.class); + ret = fileStoreUtil.loadFromDir(new Path(fileStoreUtil.getDataDir()), FILE_PREFIX_SERVICE, RangerService.class); nextServiceId = getMaxId(ret) + 1; } catch(Exception excp) { @@ -926,7 +935,7 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { List<RangerPolicy> ret = null; try { - ret = loadFromDir(new Path(getDataDir()), FILE_PREFIX_POLICY, RangerPolicy.class); + ret = fileStoreUtil.loadFromDir(new Path(fileStoreUtil.getDataDir()), FILE_PREFIX_POLICY, RangerPolicy.class); nextPolicyId = getMaxId(ret) + 1; } catch(Exception excp) {
