http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/83cb21e0/agents-common/src/main/java/org/apache/ranger/plugin/store/RangerServiceResourceSignature.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/RangerServiceResourceSignature.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/RangerServiceResourceSignature.java new file mode 100644 index 0000000..3e32a36 --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/RangerServiceResourceSignature.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.store; + +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.ranger.plugin.model.RangerPolicy; +import org.apache.ranger.plugin.model.RangerServiceResource; + +import java.util.*; + +public class RangerServiceResourceSignature { + private final String _string; + private final String _hash; + private final RangerServiceResource serviceResource; + + public RangerServiceResourceSignature(RangerServiceResource serviceResource) { + this.serviceResource = serviceResource; + _string = ServiceResourceSpecSerializer.toString(serviceResource); + _hash = DigestUtils.md5Hex(_string); + } + + String asString() { + return _string; + } + + public String getSignature() { + return _hash; + } + + static class ServiceResourceSpecSerializer { + + static final int _SignatureVersion = 1; + + static public String toString(final RangerServiceResource serviceResource) { + // invalid/empty serviceResource gets a deterministic signature as if it had an + // empty resource string + Map<String, RangerPolicy.RangerPolicyResource> serviceResourceSpec = serviceResource.getResourceSpec(); + Map<String, ResourceSerializer> resources = new TreeMap<String, ResourceSerializer>(); + for (Map.Entry<String, RangerPolicy.RangerPolicyResource> entry : serviceResourceSpec.entrySet()) { + String resourceName = entry.getKey(); + ResourceSerializer resourceView = new ResourceSerializer(entry.getValue()); + resources.put(resourceName, resourceView); + } + String resourcesAsString = resources.toString(); + return String.format("{version=%d,service=%s,resource=%s}", _SignatureVersion, serviceResource.getServiceName(), resourcesAsString); + } + + static class ResourceSerializer { + final RangerPolicy.RangerPolicyResource _policyResource; + + ResourceSerializer(RangerPolicy.RangerPolicyResource policyResource) { + _policyResource = policyResource; + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("{"); + if (_policyResource != null) { + builder.append("values="); + if (_policyResource.getValues() != null) { + List<String> values = new ArrayList<String>(_policyResource.getValues()); + Collections.sort(values); + builder.append(values); + } + } + builder.append(",excludes="); + if (_policyResource.getIsExcludes() == null) { // null is same as false + builder.append(Boolean.FALSE); + } else { + builder.append(_policyResource.getIsExcludes()); + } + builder.append(",recursive="); + if (_policyResource.getIsRecursive() == null) { // null is the same as false + builder.append(Boolean.FALSE); + } else { + builder.append(_policyResource.getIsRecursive()); + } + builder.append("}"); + return builder.toString(); + } + } + } +} +
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/83cb21e0/agents-common/src/main/java/org/apache/ranger/plugin/store/TagPredicateUtil.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/TagPredicateUtil.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/TagPredicateUtil.java index a7cee87..950969e 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/TagPredicateUtil.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/TagPredicateUtil.java @@ -21,7 +21,9 @@ package org.apache.ranger.plugin.store; import org.apache.commons.collections.Predicate; import org.apache.commons.lang.StringUtils; -import org.apache.ranger.plugin.model.RangerTaggedResource; +import org.apache.ranger.plugin.model.RangerServiceResource; +import org.apache.ranger.plugin.model.RangerTag; +import org.apache.ranger.plugin.model.RangerTagResourceMap; import org.apache.ranger.plugin.model.RangerTagDef; import org.apache.ranger.plugin.util.SearchFilter; @@ -36,12 +38,21 @@ public class TagPredicateUtil extends AbstractPredicateUtil { super.addPredicates(filter, predicates); addPredicateForTagDefId(filter.getParam(SearchFilter.TAG_DEF_ID), predicates); + addPredicateForTagDefExternalId(filter.getParam(SearchFilter.TAG_DEF_EXTERNAL_ID), predicates); addPredicateForTagDefName(filter.getParam(SearchFilter.TAG_DEF_NAME), predicates); - addPredicateForTagResourceServiceName(filter.getParam(SearchFilter.TAG_RESOURCE_SERVICE_NAME), predicates); - addPredicateForTagResourceTimestamp(filter.getParamAsLong(SearchFilter.TAG_RESOURCE_TIMESTAMP), predicates); + addPredicateForTagId(filter.getParam(SearchFilter.TAG_ID), predicates); + addPredicateForTagExternalId(filter.getParam(SearchFilter.TAG_EXTERNAL_ID), predicates); + addPredicateForTagName(filter.getParam(SearchFilter.TAG_NAME), predicates); - addPredicateForTagResourceId(filter.getParam(SearchFilter.TAG_RESOURCE_ID), predicates); + addPredicateForResourceId(filter.getParam(SearchFilter.TAG_RESOURCE_ID), predicates); + addPredicateForResourceExternalId(filter.getParam(SearchFilter.TAG_RESOURCE_EXTERNAL_ID), predicates); + addPredicateForServiceResourceServiceName(filter.getParam(SearchFilter.TAG_RESOURCE_SERVICE_NAME), predicates); + addPredicateForResourceSignature(filter.getParam(SearchFilter.TAG_RESOURCE_SIGNATURE), predicates); + + addPredicateForTagResourceMapId(filter.getParam(SearchFilter.TAG_MAP_ID), predicates); + addPredicateForTagResourceMapResourceId(filter.getParam(SearchFilter.TAG_MAP_RESOURCE_ID), predicates); + addPredicateForTagResourceMapTagId(filter.getParam(SearchFilter.TAG_MAP_TAG_ID), predicates); } private Predicate addPredicateForTagDefId(final String id, List<Predicate> predicates) { @@ -76,6 +87,38 @@ public class TagPredicateUtil extends AbstractPredicateUtil { return ret; } + private Predicate addPredicateForTagDefExternalId(final String externalId, List<Predicate> predicates) { + if (externalId == null || StringUtils.isEmpty(externalId)) { + return null; + } + + Predicate ret = new Predicate() { + @Override + public boolean evaluate(Object object) { + + boolean ret = false; + + if (object == null) { + return ret; + } + + if (object instanceof RangerTagDef) { + RangerTagDef tagDef = (RangerTagDef) object; + + ret = StringUtils.equals(externalId, tagDef.getGuid()); + } + + return ret; + } + }; + + if (predicates != null) { + predicates.add(ret); + } + + return ret; + } + private Predicate addPredicateForTagDefName(final String name, List<Predicate> predicates) { if (name == null || StringUtils.isEmpty(name)) { return null; @@ -108,8 +151,104 @@ public class TagPredicateUtil extends AbstractPredicateUtil { return ret; } - private Predicate addPredicateForTagResourceServiceName(final String name, List<Predicate> predicates) { - if (name == null || StringUtils.isEmpty(name)) { + private Predicate addPredicateForTagId(final String id, List<Predicate> predicates) { + if (StringUtils.isEmpty(id)) { + return null; + } + + Predicate ret = new Predicate() { + @Override + public boolean evaluate(Object object) { + + boolean ret = false; + + if (object == null) { + return ret; + } + + if (object instanceof RangerTag) { + RangerTag tag = (RangerTag) object; + + ret = StringUtils.equals(id, tag.getId().toString()); + } + + return ret; + } + }; + + if (predicates != null) { + predicates.add(ret); + } + + return ret; + } + + private Predicate addPredicateForTagExternalId(final String externalId, List<Predicate> predicates) { + if (StringUtils.isEmpty(externalId)) { + return null; + } + + Predicate ret = new Predicate() { + @Override + public boolean evaluate(Object object) { + + boolean ret = false; + + if (object == null) { + return ret; + } + + if (object instanceof RangerTag) { + RangerTag tag = (RangerTag) object; + + ret = StringUtils.equals(externalId, tag.getGuid()); + } + + return ret; + } + }; + + if (predicates != null) { + predicates.add(ret); + } + + return ret; + } + + private Predicate addPredicateForTagName(final String name, List<Predicate> predicates) { + if (StringUtils.isEmpty(name)) { + return null; + } + + Predicate ret = new Predicate() { + @Override + public boolean evaluate(Object object) { + + boolean ret = false; + + if (object == null) { + return ret; + } + + if (object instanceof RangerTag) { + RangerTag tag = (RangerTag) object; + + ret = StringUtils.equals(name, tag.getName()); + } + + return ret; + } + }; + + if (predicates != null) { + predicates.add(ret); + } + + return ret; + } + + private Predicate addPredicateForResourceId(final String id, List<Predicate> predicates) { + if (StringUtils.isEmpty(id)) { return null; } @@ -123,10 +262,10 @@ public class TagPredicateUtil extends AbstractPredicateUtil { return ret; } - if (object instanceof RangerTaggedResource) { - RangerTaggedResource rangerResource = (RangerTaggedResource) object; + if (object instanceof RangerServiceResource) { + RangerServiceResource resource = (RangerServiceResource) object; - ret = StringUtils.equals(name, rangerResource.getKey().getServiceName()); + ret = StringUtils.equals(id, resource.getId().toString()); } return ret; @@ -139,7 +278,8 @@ public class TagPredicateUtil extends AbstractPredicateUtil { return ret; } - private Predicate addPredicateForTagResourceId(final String id, List<Predicate> predicates) { + + private Predicate addPredicateForResourceExternalId(final String id, List<Predicate> predicates) { if (StringUtils.isEmpty(id)) { return null; } @@ -154,10 +294,73 @@ public class TagPredicateUtil extends AbstractPredicateUtil { return ret; } - if (object instanceof RangerTaggedResource) { - RangerTaggedResource rangerResource = (RangerTaggedResource) object; + if (object instanceof RangerServiceResource) { + RangerServiceResource resource = (RangerServiceResource) object; + + ret = StringUtils.equals(id, resource.getGuid()); + } + + return ret; + } + }; + + if (predicates != null) { + predicates.add(ret); + } + + return ret; + } + + private Predicate addPredicateForServiceResourceServiceName(final String serviceName, List<Predicate> predicates) { + if (serviceName == null || StringUtils.isEmpty(serviceName)) { + return null; + } + + Predicate ret = new Predicate() { + @Override + public boolean evaluate(Object object) { + + boolean ret = false; + + if (object == null) { + return ret; + } + + if (object instanceof RangerServiceResource) { + RangerServiceResource resource = (RangerServiceResource) object; + ret = StringUtils.equals(resource.getServiceName(), serviceName); + } + + return ret; + } + }; + + if (predicates != null) { + predicates.add(ret); + } + + return ret; + } + + private Predicate addPredicateForResourceSignature(final String signature, List<Predicate> predicates) { + if (StringUtils.isEmpty(signature)) { + return null; + } + + Predicate ret = new Predicate() { + @Override + public boolean evaluate(Object object) { + + boolean ret = false; + + if (object == null) { + return ret; + } + + if (object instanceof RangerServiceResource) { + RangerServiceResource resource = (RangerServiceResource) object; - ret = StringUtils.equals(id, rangerResource.getId().toString()); + ret = StringUtils.equals(signature, resource.getResourceSignature()); } return ret; @@ -170,13 +373,40 @@ public class TagPredicateUtil extends AbstractPredicateUtil { return ret; } - private Predicate addPredicateForTagResourceTimestamp(final Long lastTimestamp, List<Predicate> predicates) { - final int uploadInterval = 1*1000; - // Assumption: it may take maximum of one second for a taggedResource to be persisted after the timestamp - // was generated for it. The round-trip time is already taken into consideration by client. + private Predicate addPredicateForTagResourceMapId(final String id, List<Predicate> predicates) { + if (StringUtils.isEmpty(id)) { + return null; + } + + Predicate ret = new Predicate() { + @Override + public boolean evaluate(Object object) { + + boolean ret = false; + + if (object == null) { + return ret; + } + + if (object instanceof RangerTagResourceMap) { + RangerTagResourceMap tagResourceMap = (RangerTagResourceMap) object; + ret = StringUtils.equals(id, tagResourceMap.getId().toString()); + } + + return ret; + } + }; + + if (predicates != null) { + predicates.add(ret); + } - if (lastTimestamp == null) { + return ret; + } + + private Predicate addPredicateForTagResourceMapResourceId(final String resourceId, List<Predicate> predicates) { + if (StringUtils.isEmpty(resourceId)) { return null; } @@ -190,10 +420,40 @@ public class TagPredicateUtil extends AbstractPredicateUtil { return ret; } - if (object instanceof RangerTaggedResource) { - RangerTaggedResource rangerResource = (RangerTaggedResource) object; + if (object instanceof RangerTagResourceMap) { + RangerTagResourceMap tagResourceMap = (RangerTagResourceMap) object; + ret = StringUtils.equals(resourceId, tagResourceMap.getResourceId().toString()); + } + + return ret; + } + }; + + if (predicates != null) { + predicates.add(ret); + } - ret = rangerResource.getUpdateTime().getTime() >= (lastTimestamp - uploadInterval); + return ret; + } + + private Predicate addPredicateForTagResourceMapTagId(final String tagId, List<Predicate> predicates) { + if (StringUtils.isEmpty(tagId)) { + return null; + } + + Predicate ret = new Predicate() { + @Override + public boolean evaluate(Object object) { + + boolean ret = false; + + if (object == null) { + return ret; + } + + if (object instanceof RangerTagResourceMap) { + RangerTagResourceMap tagResourceMap = (RangerTagResourceMap) object; + ret = StringUtils.equals(tagId, tagResourceMap.getTagId().toString()); } return ret; @@ -205,4 +465,5 @@ public class TagPredicateUtil extends AbstractPredicateUtil { } return ret; - }} + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/83cb21e0/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java index 3e61f92..aaa70c6 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java @@ -19,13 +19,12 @@ package org.apache.ranger.plugin.store; -import org.apache.ranger.plugin.model.RangerTaggedResourceKey; -import org.apache.ranger.plugin.model.RangerTaggedResource; -import org.apache.ranger.plugin.model.RangerTagDef; +import org.apache.ranger.plugin.model.*; import org.apache.ranger.plugin.util.SearchFilter; -import org.apache.ranger.plugin.util.TagServiceResources; +import org.apache.ranger.plugin.util.ServiceTags; import java.util.List; +import java.util.Map; /** * Interface to backing store for the top-level TAG model objects @@ -52,23 +51,60 @@ public interface TagStore { PList<RangerTagDef> getPaginatedTagDefs(SearchFilter filter) throws Exception; - RangerTaggedResource createTaggedResource(RangerTaggedResource resource, boolean createOrUpdate) throws Exception; + RangerTag createTag(RangerTag tag) throws Exception; - RangerTaggedResource updateTaggedResource(RangerTaggedResource resource) throws Exception; + RangerTag updateTag(RangerTag tag) throws Exception; - void deleteResource(Long id) throws Exception; + void deleteTagById(Long id) throws Exception; - RangerTaggedResource getResource(Long id) throws Exception; + RangerTag getTagById(Long id) throws Exception; - TagServiceResources getResources(String serviceName, Long lastTimestamp) throws Exception; + List<RangerTag> getTagsByName(String name) throws Exception; - List<RangerTaggedResource> getResources(SearchFilter filter) throws Exception; + List<RangerTag> getTagsByExternalId(String externalId) throws Exception; - PList<RangerTaggedResource> getPaginatedResources(SearchFilter filter) throws Exception; + List<RangerTag> getTags(SearchFilter filter) throws Exception; + + RangerServiceResource createServiceResource(RangerServiceResource resource) throws Exception; + + RangerServiceResource updateServiceResource(RangerServiceResource resource) throws Exception; + + void deleteServiceResourceById(Long id) throws Exception; + + List<RangerServiceResource> getServiceResourcesByExternalId(String externalId) throws Exception; + + RangerServiceResource getServiceResourceById(Long id) throws Exception; + + List<RangerServiceResource> getServiceResourcesByServiceAndResourceSpec(String serviceName, Map<String, RangerPolicy.RangerPolicyResource> resourceSpec) throws Exception; + + List<RangerServiceResource> getServiceResources(SearchFilter filter) throws Exception; + + RangerTagResourceMap createTagResourceMap(RangerTagResourceMap tagResourceMap) throws Exception; + + void deleteTagResourceMapById(Long id) throws Exception; + + List<RangerTagResourceMap> getTagResourceMap(String externalResourceId, String externalTagId) throws Exception; + + RangerTagResourceMap getTagResourceMapById(Long id) throws Exception; + + List<RangerTagResourceMap> getTagResourceMapsByTagId(Long tagId) throws Exception; + + List<RangerTagResourceMap> getTagResourceMapsByResourceId(Long resourceId) throws Exception; + + List<RangerTagResourceMap> getTagResourceMaps(SearchFilter filter) throws Exception; + + ServiceTags getServiceTagsIfUpdated(String serviceName, Long lastKnownVersion) throws Exception; + + PList<RangerTagResourceMap> getPaginatedTagResourceMaps(SearchFilter filter) throws Exception; List<String> getTags(String serviceName) throws Exception; List<String> lookupTags(String serviceName, String tagNamePattern) throws Exception; - RangerTaggedResource getResource(RangerTaggedResourceKey key) throws Exception; + List<RangerTag> getTagsForServiceResource(Long resourceId) throws Exception; + + List<RangerTag> getTagsForServiceResourceByExtId(String resourceExtId) throws Exception; + + List<RangerTagDef> getTagDefsByExternalId(String extId) throws Exception; + } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/83cb21e0/agents-common/src/main/java/org/apache/ranger/plugin/store/TagValidator.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/TagValidator.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/TagValidator.java new file mode 100644 index 0000000..ada5dd2 --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/TagValidator.java @@ -0,0 +1,260 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.store; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.ranger.plugin.model.*; + +import java.util.*; + +public class TagValidator { + private TagStore tagStore; + + public TagValidator() {} + + public void setTagStore(TagStore tagStore) { + this.tagStore = tagStore; + } + + public void preCreateTag(final RangerTag tag) throws Exception { + if (StringUtils.isBlank(tag.getName())) { + throw new Exception("Tag has no name"); + } + } + + public void preUpdateTagById(final Long id, final RangerTag tag) throws Exception { + if (StringUtils.isBlank(tag.getName())) { + throw new Exception("Tag has no name"); + } + + if (id == null) { + throw new Exception("Invalid/null id"); + } + + RangerTag exist = tagStore.getTagById(id); + + if (exist == null) { + throw new Exception("Attempt to update nonexistant tag, id=" + id); + } + tag.setId(exist.getId()); + } + + public void preUpdateTagByExternalId(String externalId, final RangerTag tag) throws Exception { + if (StringUtils.isBlank(tag.getName())) { + throw new Exception("Tag has no name"); + } + + List<RangerTag> exist = tagStore.getTagsByExternalId(externalId); + if (CollectionUtils.isEmpty(exist) || CollectionUtils.size(exist) != 1) { + throw new Exception("Attempt to update nonexistent or multiple tags, externalId=" + externalId); + } + + RangerTag onlyTag = exist.get(0); + + tag.setId(onlyTag.getId()); + tag.setGuid(externalId); + + } + + public void preUpdateTagByName(String name, final RangerTag tag) throws Exception { + if (StringUtils.isNotBlank(tag.getName())) { + throw new Exception("tag has no name"); + } + + List<RangerTag> exist = tagStore.getTagsByName(name); + if (CollectionUtils.isEmpty(exist) || CollectionUtils.size(exist) != 1) { + throw new Exception("Attempt to update nonexistent or multiple tags, name=" + name); + } + + RangerTag onlyTag = exist.get(0); + + tag.setId(onlyTag.getId()); + tag.setName(name); + + } + + public RangerTag preDeleteTagById(Long id) throws Exception { + RangerTag exist; + exist = tagStore.getTagById(id); + if (exist == null) { + throw new Exception("Attempt to delete nonexistent tag, id=" + id); + } + + List<RangerTagResourceMap> associations = tagStore.getTagResourceMapsByTagId(exist.getId()); + if (CollectionUtils.isNotEmpty(associations)) { + throw new Exception("Attempt to delete tag which is associated with a service-resource, id=" + id); + } + return exist; + } + + public RangerTag preDeleteTagByExternalId(String externalId) throws Exception { + List<RangerTag> exist; + exist = tagStore.getTagsByExternalId(externalId); + if (CollectionUtils.isEmpty(exist) || CollectionUtils.size(exist) != 1) { + throw new Exception("Attempt to delete nonexistent or multiple tags, externalId=" + externalId); + } + + RangerTag ret = exist.get(0); + List<RangerTagResourceMap> associations = tagStore.getTagResourceMapsByTagId(ret.getId()); + if (CollectionUtils.isNotEmpty(associations)) { + throw new Exception("Attempt to delete tag which is associated with a service-resource, externalId=" + externalId); + } + return ret; + } + + public RangerTag preDeleteTagByName(String name) throws Exception { + List<RangerTag> exist; + exist = tagStore.getTagsByName(name); + if (CollectionUtils.isEmpty(exist) || CollectionUtils.size(exist) != 1) { + throw new Exception("Attempt to delete nonexistent or multiple tags, name=" + name); + } + RangerTag ret = exist.get(0); + List<RangerTagResourceMap> associations = tagStore.getTagResourceMapsByTagId(ret.getId()); + if (CollectionUtils.isNotEmpty(associations)) { + throw new Exception("Attempt to delete tag which is associated with a service-resource, name=" + name); + } + return ret; + + } + + public void preCreateServiceResource(RangerServiceResource resource) throws Exception { + if (StringUtils.isBlank(resource.getServiceName()) + || resource.getResourceSpec() == null + || CollectionUtils.size(resource.getResourceSpec()) == 0) { + throw new Exception("No serviceName or resourceSpec in RangerServiceResource"); + } + + List<RangerServiceResource> exist; + exist = tagStore.getServiceResourcesByServiceAndResourceSpec(resource.getServiceName(), resource.getResourceSpec()); + if (CollectionUtils.isNotEmpty(exist)) { + throw new Exception("Attempt to create existing resource, serviceName=" + resource.getServiceName()); + } + RangerServiceResourceSignature serializer = new RangerServiceResourceSignature(resource); + resource.setResourceSignature(serializer.getSignature()); + } + + public void preUpdateServiceResourceById(Long id, RangerServiceResource resource) throws Exception { + if (StringUtils.isBlank(resource.getServiceName()) + || resource.getResourceSpec() == null + || CollectionUtils.size(resource.getResourceSpec()) == 0) { + throw new Exception("No serviceName or resourceSpec in RangerServiceResource"); + } + + if (id == null) { + throw new Exception("Invalid/null id"); + } + + RangerServiceResource exist = tagStore.getServiceResourceById(id); + if (exist == null) { + throw new Exception("Attempt to update nonexistent resource, id=" + id); + } + resource.setId(exist.getId()); + + RangerServiceResourceSignature serializer = new RangerServiceResourceSignature(resource); + resource.setResourceSignature(serializer.getSignature()); + + } + + public void preUpdateServiceResourceByExternalId(String externalId, RangerServiceResource resource) throws Exception { + if (StringUtils.isBlank(resource.getServiceName()) + || resource.getResourceSpec() == null + || CollectionUtils.size(resource.getResourceSpec()) == 0) { + throw new Exception("No serviceName or resourceSpec in RangerServiceResource"); + } + + List<RangerServiceResource> exist; + exist = tagStore.getServiceResourcesByExternalId(externalId); + if (CollectionUtils.isEmpty(exist) || CollectionUtils.size(exist) != 1) { + throw new Exception("Attempt to update nonexistent or multiple resources, externalId=" + externalId); + } + + RangerServiceResource onlyResource = exist.get(0); + + resource.setId(onlyResource.getId()); + resource.setGuid(externalId); + + RangerServiceResourceSignature serializer = new RangerServiceResourceSignature(resource); + resource.setResourceSignature(serializer.getSignature()); + } + + public RangerServiceResource preDeleteServiceResourceById(Long id) throws Exception { + RangerServiceResource exist; + exist = tagStore.getServiceResourceById(id); + if (exist == null) { + throw new Exception("Attempt to delete nonexistent resource, id=" + id); + } + List<RangerTagResourceMap> associations = tagStore.getTagResourceMapsByResourceId(exist.getId()); + if (CollectionUtils.isNotEmpty(associations)) { + throw new Exception("Attempt to delete serviceResource which is associated with a tag, id=" + id); + } + return exist; + } + + public RangerServiceResource preDeleteServiceResourceByExternalId(String externalId) throws Exception { + List<RangerServiceResource> exist; + exist = tagStore.getServiceResourcesByExternalId(externalId); + if (CollectionUtils.isEmpty(exist) || CollectionUtils.size(exist) != 1) { + throw new Exception("Attempt to delete nonexistent or multiple resources, externalId=" + externalId); + } + RangerServiceResource ret = exist.get(0); + List<RangerTagResourceMap> associations = tagStore.getTagResourceMapsByResourceId(ret.getId()); + if (CollectionUtils.isNotEmpty(associations)) { + throw new Exception("Attempt to delete serviceResource which is associated with a tag, externalId=" + externalId); + } + return ret; + } + + public RangerTagResourceMap preCreateTagResourceMap(String externalResourceId, String externalTagId) throws Exception { + if (StringUtils.isBlank(externalResourceId) || StringUtils.isBlank(externalTagId)) { + throw new Exception("Both externalResourceId and internalResourceId need to be non-empty"); + } + + List<RangerTagResourceMap> exist; + exist = tagStore.getTagResourceMap(externalResourceId, externalTagId); + if (CollectionUtils.isNotEmpty(exist)) { + throw new Exception("Attempt to create existing association between resourceId=" + externalResourceId + " and tagId=" + externalTagId); + } + List<RangerServiceResource> existingServiceResources = tagStore.getServiceResourcesByExternalId(externalResourceId); + List<RangerTag> existingTags = tagStore.getTagsByExternalId(externalTagId); + + if (CollectionUtils.isNotEmpty(existingServiceResources) && CollectionUtils.size(existingServiceResources) == 1) { + if (CollectionUtils.isNotEmpty(existingTags) && CollectionUtils.size(existingTags) == 1) { + RangerTagResourceMap newTagResourceMap = new RangerTagResourceMap(); + newTagResourceMap.setResourceId(existingServiceResources.get(0).getId()); + newTagResourceMap.setTagId(existingTags.get(0).getId()); + return newTagResourceMap; + } else { + throw new Exception("No unique tag found for externalId=" + externalTagId); + } + } else { + throw new Exception("No unique resource found for externalId=" + externalResourceId); + } + } + + public RangerTagResourceMap preDeleteTagResourceMap(String externalResourceId, String externalTagId) throws Exception { + List<RangerTagResourceMap> exist; + exist = tagStore.getTagResourceMap(externalResourceId, externalTagId); + if (CollectionUtils.isEmpty(exist) || CollectionUtils.size(exist) != 1) { + throw new Exception("Attempt to create nonexistent association between resourceId=" + externalResourceId + " and tagId=" + externalTagId); + } + return exist.get(0); + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/83cb21e0/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java index 6538060..8a63f72 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java @@ -20,7 +20,6 @@ package org.apache.ranger.plugin.store.file; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.ObjectUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; @@ -28,12 +27,10 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; import org.apache.ranger.plugin.model.*; -import org.apache.ranger.plugin.policyresourcematcher.RangerDefaultPolicyResourceMatcher; import org.apache.ranger.plugin.store.AbstractTagStore; import org.apache.ranger.plugin.store.TagPredicateUtil; import org.apache.ranger.plugin.store.TagStore; import org.apache.ranger.plugin.util.SearchFilter; -import org.apache.ranger.plugin.util.TagServiceResources; import java.util.*; import java.util.regex.Matcher; @@ -44,11 +41,15 @@ public class TagFileStore extends AbstractTagStore { public static final String PROPERTY_TAG_FILE_STORE_DIR = "ranger.tag.store.file.dir"; protected static final String FILE_PREFIX_TAG_DEF = "ranger-tagdef-"; - protected static final String FILE_PREFIX_TAG_RESOURCE = "ranger-tag-resource-"; + protected static final String FILE_PREFIX_TAG = "ranger-tag-"; + protected static final String FILE_PREFIX_RESOURCE = "ranger-serviceresource-"; + protected static final String FILE_PREFIX_TAG_RESOURCE_MAP = "ranger-tagresourcemap-"; private String tagDataDir = null; private long nextTagDefId = 0; - private long nextTagResourceId = 0; + private long nextTagId = 0; + private long nextServiceResourceId = 0; + private long nextTagResourceMapId = 0; private TagPredicateUtil predicateUtil = null; @@ -300,287 +301,259 @@ public class TagFileStore extends AbstractTagStore { } @Override - public RangerTaggedResource createTaggedResource(RangerTaggedResource resource, boolean createOrUpdate) throws Exception { - if (LOG.isDebugEnabled()) { - LOG.debug("==> TagFileStore.createResource(" + resource + ")"); - } + public RangerTag createTag(RangerTag tag) throws Exception + { + RangerTag ret; - RangerTaggedResource ret = null; - RangerTaggedResource existing = null; - boolean updateResource = false; + try { + preCreate(tag); - existing = getResource(resource.getKey()); + tag.setId(nextTagId); - if (existing != null) { - if (!createOrUpdate) { - throw new Exception("resource(s) with same specification already exists"); - } else { - updateResource = true; - } - } + ret = fileStoreUtil.saveToFile(tag, new Path(fileStoreUtil.getDataFile(FILE_PREFIX_TAG, nextTagId++)), false); - if (! updateResource) { - if (resource.getId() != null) { - existing = getResource(resource.getId()); - } + postCreate(ret); + } catch (Exception excp) { + LOG.warn("TagFileStore.createTag(): failed to save tag '" + tag.getName() + "'", excp); - if (existing != null) { - if (! createOrUpdate) { - throw new Exception(resource.getId() + ": resource already exists (id=" + existing.getId() + ")"); - } else { - updateResource = true; - } - } + throw new Exception("failed to save tag '" + tag.getName() + "'", excp); } - try { - if (updateResource) { - ret = updateTaggedResource(resource); - } else { - preCreate(resource); + if (LOG.isDebugEnabled()) { + LOG.debug("<== TagFileStore.createTag(" + tag + ")"); + } - resource.setId(nextTagResourceId); + return ret; + } - ret = fileStoreUtil.saveToFile(resource, new Path(fileStoreUtil.getDataFile(FILE_PREFIX_TAG_RESOURCE, nextTagResourceId++)), false); + @Override + public RangerTag updateTag(RangerTag tag) throws Exception + { + RangerTag ret; - postCreate(ret); - } + try { + preUpdate(tag); + + ret = fileStoreUtil.saveToFile(tag, new Path(fileStoreUtil.getDataFile(FILE_PREFIX_TAG, tag.getId())), true); + + postUpdate(tag); } catch (Exception excp) { - LOG.warn("TagFileStore.createResource(): failed to save resource '" + resource.getId() + "'", excp); + LOG.warn("TagFileStore.updateTag(): failed to save tag '" + tag.getName() + "'", excp); - throw new Exception("failed to save resource '" + resource.getId() + "'", excp); + throw new Exception("failed to save tag '" + tag.getName() + "'", excp); } if (LOG.isDebugEnabled()) { - LOG.debug("<== TagFileStore.createResource(" + resource + ")"); + LOG.debug("<== TagFileStore.updateTag(" + tag + ")"); } return ret; } @Override - public RangerTaggedResource updateTaggedResource(RangerTaggedResource resource) throws Exception { + public void deleteTagById(Long id) throws Exception { if (LOG.isDebugEnabled()) { - LOG.debug("==> TagFileStore.updateResource(" + resource + ")"); + LOG.debug("==> TagFileStore.deleteTag(" + id + ")"); } - RangerTaggedResource existing = getResource(resource.getId()); - if (existing == null) { - throw new Exception(resource.getId() + ": resource does not exist (id=" + resource.getId() + ")"); - } + try { + RangerTag tag = getTagById(id); - RangerTaggedResource ret = null; + Path filePath = new Path(fileStoreUtil.getDataFile(FILE_PREFIX_TAG, tag.getId())); - try { - preUpdate(existing); + preDelete(tag); - existing.getKey().setResourceSpec(resource.getKey().getResourceSpec()); - existing.getKey().setServiceName(resource.getKey().getServiceName()); - existing.setTags(resource.getTags()); + fileStoreUtil.deleteFile(filePath); - ret = fileStoreUtil.saveToFile(existing, new Path(fileStoreUtil.getDataFile(FILE_PREFIX_TAG_RESOURCE, existing.getId())), true); + postDelete(tag); - postUpdate(existing); } catch (Exception excp) { - LOG.warn("TagFileStore.updateTagDef(): failed to save resource '" + resource.getId() + "'", excp); - - throw new Exception("failed to save tag-def '" + resource.getId() + "'", excp); + throw new Exception("failed to delete tag with ID=" + id, excp); } - if (LOG.isDebugEnabled()) { - LOG.debug("<== TagFileStore.updateResource(" + resource + ")"); + LOG.debug("<== TagFileStore.deleteTag(" + id + ")"); } - return ret; } @Override - public void deleteResource(Long id) throws Exception { + public List<RangerTag> getTags(SearchFilter filter) throws Exception { + if (LOG.isDebugEnabled()) { - LOG.debug("==> TagFileStore.deleteResource(" + id + ")"); + LOG.debug("==> TagFileStore.getTags()"); } - RangerTaggedResource existing = getResource(id); + List<RangerTag> ret = getAllTags(); - if (existing == null) { - throw new Exception("no resource exists with ID=" + id); + if (CollectionUtils.isNotEmpty(ret) && filter != null && !filter.isEmpty()) { + CollectionUtils.filter(ret, predicateUtil.getPredicate(filter)); } + if (LOG.isDebugEnabled()) { + LOG.debug("<== TagFileStore.getTags(): count=" + (ret == null ? 0 : ret.size())); + } + + return ret; + } + + + @Override + public RangerServiceResource createServiceResource(RangerServiceResource resource) throws Exception { + RangerServiceResource ret; + try { - Path filePath = new Path(fileStoreUtil.getDataFile(FILE_PREFIX_TAG_RESOURCE, existing.getId())); + preCreate(resource); - preDelete(existing); + resource.setId(nextServiceResourceId); - fileStoreUtil.deleteFile(filePath); + ret = fileStoreUtil.saveToFile(resource, new Path(fileStoreUtil.getDataFile(FILE_PREFIX_RESOURCE, nextServiceResourceId++)), false); - postDelete(existing); + postCreate(ret); } catch (Exception excp) { - throw new Exception("failed to delete resource with ID=" + id, excp); + LOG.warn("TagFileStore.createServiceResource(): failed to save resource '" + resource.getId() + "'", excp); + + throw new Exception("failed to save service-resource '" + resource.getId() + "'", excp); } if (LOG.isDebugEnabled()) { - LOG.debug("<== TagFileStore.deleteResource(" + id + ")"); + LOG.debug("<== TagFileStore.createServiceResource(" + resource + ")"); } + + return ret; } @Override - public RangerTaggedResource getResource(Long id) throws Exception { + public RangerServiceResource updateServiceResource(RangerServiceResource resource) throws Exception { if (LOG.isDebugEnabled()) { - LOG.debug("==> TagFileStore.getResource(" + id + ")"); + LOG.debug("==> TagFileStore.updateServiceResource(" + resource + ")"); } - RangerTaggedResource ret; + RangerServiceResource ret; - if (id != null) { - SearchFilter filter = new SearchFilter(SearchFilter.TAG_RESOURCE_ID, id.toString()); + try { + preUpdate(resource); - List<RangerTaggedResource> resources = getResources(filter); + ret = fileStoreUtil.saveToFile(resource, new Path(fileStoreUtil.getDataFile(FILE_PREFIX_RESOURCE, resource.getId())), true); - ret = CollectionUtils.isEmpty(resources) ? null : resources.get(0); - } else { - ret = null; + postUpdate(resource); + } catch (Exception excp) { + LOG.warn("TagFileStore.updateServiceResource(): failed to save resource '" + resource.getId() + "'", excp); + + throw new Exception("failed to save service-resource '" + resource.getId() + "'", excp); } + if (LOG.isDebugEnabled()) { - LOG.debug("<== TagFileStore.getResource(" + id + ")"); + LOG.debug("<== TagFileStore.updateServiceResource(" + resource + ")"); } + return ret; + } @Override - public RangerTaggedResource getResource(RangerTaggedResourceKey key) throws Exception { - + public void deleteServiceResourceById(Long id) throws Exception { if (LOG.isDebugEnabled()) { - LOG.debug("==> TagFileStore.getResource( " + key.getServiceName() + " )"); - } - - if (this.svcStore == null) { - LOG.error("TagFileStore.getResource() - TagFileStore object does not have reference to a valid ServiceStore."); - throw new Exception("TagFileStore.getResources() - TagFileStore object does not have reference to a valid ServiceStore."); - } - - if (key == null) { - LOG.error("TagFileStore.getResource() - parameter 'key' is null"); - throw new Exception("TagFileStore.getResources() - parameter 'key' is null."); + LOG.debug("==> TagFileStore.deleteServiceResource(" + id + ")"); } - RangerService service = null; try { - service = svcStore.getServiceByName(key.getServiceName()); - } catch(Exception excp) { - LOG.error("TagFileStore.getResource - failed to get service " + key.getServiceName()); - throw new Exception("Invalid service: " + key.getServiceName()); - } + RangerServiceResource resource = getServiceResourceById(id); - RangerServiceDef serviceDef = null; + Path filePath = new Path(fileStoreUtil.getDataFile(FILE_PREFIX_RESOURCE, resource.getId())); - try { - serviceDef = svcStore.getServiceDefByName(service.getType()); - } catch (Exception exception) { - LOG.error("TagFileStore.getResource - failed to get serviceDef for " + service.getType()); - throw new Exception("Invalid component-type: " + service.getType()); - } + preDelete(resource); - List<RangerTaggedResource> resources = null; + fileStoreUtil.deleteFile(filePath); - if (MapUtils.isNotEmpty(key.getResourceSpec())) { + postDelete(resource); - TagServiceResources tagServiceResources = getResources(key.getServiceName(), 0L); - resources = tagServiceResources.getTaggedResources(); + } catch (Exception excp) { + throw new Exception("failed to delete service-resource with ID=" + id, excp); + } - List<RangerTaggedResource> notMatchedResources = new ArrayList<>(); + if (LOG.isDebugEnabled()) { + LOG.debug("<== TagFileStore.deleteServiceResource(" + id + ")"); + } + } - if (CollectionUtils.isNotEmpty(resources)) { - for (RangerTaggedResource resource : resources) { + @Override + public List<RangerServiceResource> getServiceResources(SearchFilter filter) throws Exception { + if (LOG.isDebugEnabled()) { + LOG.debug("==> TagFileStore.getServiceResources()"); + } - RangerDefaultPolicyResourceMatcher policyResourceMatcher = - new RangerDefaultPolicyResourceMatcher(); + List<RangerServiceResource> ret = getAllResources(); - policyResourceMatcher.setPolicyResources(resource.getKey().getResourceSpec()); + if (CollectionUtils.isNotEmpty(ret) && filter != null && !filter.isEmpty()) { + CollectionUtils.filter(ret, predicateUtil.getPredicate(filter)); + } - policyResourceMatcher.setServiceDef(serviceDef); + if (LOG.isDebugEnabled()) { + LOG.debug("<== TagFileStore.getServicesResources(): count=" + (ret == null ? 0 : ret.size())); + } - policyResourceMatcher.init(); + return ret; + } - boolean isMatch = policyResourceMatcher.isExactMatch(key.getResourceSpec()); + @Override + public RangerTagResourceMap createTagResourceMap(RangerTagResourceMap tagResourceMap) throws Exception { - if (! isMatch) { - notMatchedResources.add(resource); - break; - } + RangerTagResourceMap ret; - } + preCreate(tagResourceMap); - resources.removeAll(notMatchedResources); - } - } else { - resources = null; - } + tagResourceMap.setId(nextTagResourceMapId); - RangerTaggedResource ret = (resources == null || CollectionUtils.isEmpty(resources)) ? null : resources.get(0); + ret = fileStoreUtil.saveToFile(tagResourceMap, new Path(fileStoreUtil.getDataFile(FILE_PREFIX_TAG_RESOURCE_MAP, nextTagResourceMapId++)), false); - if (LOG.isDebugEnabled()) { - LOG.debug("==> TagFileStore.getResource( " + key.getServiceName() + " )" + ret); - } + postCreate(ret); return ret; } @Override - public TagServiceResources getResources(String serviceName, Long lastTimeStamp) throws Exception { + public void deleteTagResourceMapById(Long id) throws Exception { if (LOG.isDebugEnabled()) { - LOG.debug("==> TagFileStore.getResources(" + serviceName + ", " + lastTimeStamp + ")"); + LOG.debug("==> TagFileStore.deleteTagResourceMapById(" + id + ")"); } - List<RangerTaggedResource> taggedResources; - SearchFilter filter = new SearchFilter(); + try { + RangerTagResourceMap tagResourceMap = getTagResourceMapById(id); - if (StringUtils.isNotBlank(serviceName)) { - filter.setParam(SearchFilter.TAG_RESOURCE_SERVICE_NAME, serviceName); - } + Path filePath = new Path(fileStoreUtil.getDataFile(FILE_PREFIX_TAG_RESOURCE_MAP, tagResourceMap.getId())); - if (lastTimeStamp != null) { - filter.setParam(SearchFilter.TAG_RESOURCE_TIMESTAMP, Long.toString(lastTimeStamp.longValue())); - } + preDelete(tagResourceMap); - taggedResources = getResources(filter); + fileStoreUtil.deleteFile(filePath); - if (LOG.isDebugEnabled()) { - LOG.debug("<== TagFileStore.getResources(" + serviceName + ", " + lastTimeStamp + ")"); + postDelete(tagResourceMap); + } catch (Exception excp) { + throw new Exception("failed to delete tagResourceMap with ID=" + id, excp); } - TagServiceResources ret = new TagServiceResources(); - ret.setTaggedResources(taggedResources); - // TBD - ret.setLastUpdateTime(new Date()); - ret.setVersion(1L); - - return ret; - + if (LOG.isDebugEnabled()) { + LOG.debug("<== TagFileStore.deleteTagResourceMapById(" + id + ")"); + } } @Override - public List<RangerTaggedResource> getResources(SearchFilter filter) throws Exception { + public List<RangerTagResourceMap> getTagResourceMaps(SearchFilter filter) throws Exception { if (LOG.isDebugEnabled()) { - LOG.debug("==> TagFileStore.getResources()"); + LOG.debug("==> TagFileStore.getTagResourceMaps()"); } - List<RangerTaggedResource> ret = getAllTaggedResources(); + List<RangerTagResourceMap> ret = getAllTaggedResources(); if (CollectionUtils.isNotEmpty(ret) && filter != null && !filter.isEmpty()) { CollectionUtils.filter(ret, predicateUtil.getPredicate(filter)); - - //Comparator<RangerBaseModelObject> comparator = getSorter(filter); - - //if(comparator != null) { - //Collections.sort(ret, comparator); - //} } if (LOG.isDebugEnabled()) { - LOG.debug("<== TagFileStore.getResources(): count=" + (ret == null ? 0 : ret.size())); + LOG.debug("<== TagFileStore.getTagResourceMaps(): count=" + (ret == null ? 0 : ret.size())); } return ret; + } private List<RangerTagDef> getAllTagDefs() throws Exception { @@ -620,32 +593,103 @@ public class TagFileStore extends AbstractTagStore { LOG.debug("<== TagFileStore.getAllTagDefs(): count=" + ret.size()); } - //Collections.sort(ret, idComparator); + return ret; + } - //for (RangerTagDef sd : ret) { - //Collections.sort(sd.getResources(), resourceLevelComparator); - //} + private List<RangerTag> getAllTags() throws Exception { + if (LOG.isDebugEnabled()) { + LOG.debug("==> TagFileStore.getAllTags()"); + } + + List<RangerTag> ret = new ArrayList<RangerTag>(); + + try { + List<RangerTag> sds = fileStoreUtil.loadFromDir(new Path(fileStoreUtil.getDataDir()), FILE_PREFIX_TAG, RangerTag.class); + + if (CollectionUtils.isNotEmpty(sds)) { + for (RangerTag sd : sds) { + if (sd != null) { + // if the Tag is already found, remove the earlier one + for (int i = 0; i < ret.size(); i++) { + RangerTag currSd = ret.get(i); + + if (StringUtils.equals(currSd.getName(), sd.getName()) || + ObjectUtils.equals(currSd.getId(), sd.getId())) { + ret.remove(i); + } + } + + ret.add(sd); + } + } + } + nextTagId = getMaxId(ret) + 1; + } catch (Exception excp) { + LOG.error("TagFileStore.getAllTags(): failed to read Tags", excp); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== TagFileStore.getAllTags(): count=" + ret.size()); + } + + return ret; + } + + private List<RangerServiceResource> getAllResources() throws Exception { + if (LOG.isDebugEnabled()) { + LOG.debug("==> TagFileStore.getAllResources()"); + } + + List<RangerServiceResource> ret = new ArrayList<RangerServiceResource>(); + + try { + List<RangerServiceResource> sds = fileStoreUtil.loadFromDir(new Path(fileStoreUtil.getDataDir()), FILE_PREFIX_RESOURCE, RangerServiceResource.class); + + if (CollectionUtils.isNotEmpty(sds)) { + for (RangerServiceResource sd : sds) { + if (sd != null) { + // if the resource is already found, remove the earlier one + for (int i = 0; i < ret.size(); i++) { + RangerServiceResource currSd = ret.get(i); + + if (ObjectUtils.equals(currSd.getId(), sd.getId())) { + ret.remove(i); + } + } + + ret.add(sd); + } + } + } + nextServiceResourceId = getMaxId(ret) + 1; + } catch (Exception excp) { + LOG.error("TagFileStore.getAllResources(): failed to read Resources", excp); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== TagFileStore.getAllResourcess(): count=" + ret.size()); + } return ret; } - private List<RangerTaggedResource> getAllTaggedResources() throws Exception { + private List<RangerTagResourceMap> getAllTaggedResources() throws Exception { if (LOG.isDebugEnabled()) { LOG.debug("==> TagFileStore.getAllTaggedResources()"); } - List<RangerTaggedResource> ret = new ArrayList<RangerTaggedResource>(); + List<RangerTagResourceMap> ret = new ArrayList<RangerTagResourceMap>(); try { // load resource definitions from file system - List<RangerTaggedResource> resources = fileStoreUtil.loadFromDir(new Path(fileStoreUtil.getDataDir()), FILE_PREFIX_TAG_RESOURCE, RangerTaggedResource.class); + List<RangerTagResourceMap> resources = fileStoreUtil.loadFromDir(new Path(fileStoreUtil.getDataDir()), FILE_PREFIX_TAG_RESOURCE_MAP, RangerTagResourceMap.class); if (CollectionUtils.isNotEmpty(resources)) { - for (RangerTaggedResource resource : resources) { + for (RangerTagResourceMap resource : resources) { if (resource != null) { - // if the RangerTaggedResource is already found, remove the earlier definition + // if the RangerTagResourceMap is already found, remove the earlier definition for (int i = 0; i < ret.size(); i++) { - RangerTaggedResource currResource = ret.get(i); + RangerTagResourceMap currResource = ret.get(i); if (ObjectUtils.equals(currResource.getId(), resource.getId())) { ret.remove(i); @@ -656,7 +700,7 @@ public class TagFileStore extends AbstractTagStore { } } } - nextTagResourceId = getMaxId(ret) + 1; + nextTagResourceMapId = getMaxId(ret) + 1; } catch (Exception excp) { LOG.error("TagFileStore.getAllTaggedResources(): failed to read tagged resources", excp); } @@ -665,13 +709,6 @@ public class TagFileStore extends AbstractTagStore { LOG.debug("<== TagFileStore.getAllTaggedResources(): count=" + ret.size()); } - - //Collections.sort(ret, idComparator); - - //for (RangerTagDef sd : ret) { - //Collections.sort(sd.getResources(), resourceLevelComparator); - //} - return ret; } @@ -682,28 +719,16 @@ public class TagFileStore extends AbstractTagStore { LOG.debug("==> TagFileStore.getTags(" + serviceName + ")"); } - SortedSet<String> tagNameSet = new TreeSet<String>(); - - TagServiceResources tagServiceResources = getResources(serviceName, 0L); - List<RangerTaggedResource> resources = tagServiceResources.getTaggedResources(); + // Ignore serviceName + List<RangerTag> allTags = getAllTags(); - if (CollectionUtils.isNotEmpty(resources)) { - for (RangerTaggedResource resource : resources) { - List<RangerTaggedResource.RangerResourceTag> tags = resource.getTags(); - - if (CollectionUtils.isNotEmpty(tags)) { - for (RangerTaggedResource.RangerResourceTag tag : tags) { - tagNameSet.add(tag.getName()); - } - } - } - } + List<String> ret = new ArrayList<String>(); - if (LOG.isDebugEnabled()) { - LOG.debug("<== TagFileStore.getTags(" + serviceName + ")"); + for (RangerTag tag : allTags) { + ret.add(tag.getName()); } - return new ArrayList<String>(tagNameSet); + return ret; } @Override http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/83cb21e0/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRESTUtils.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRESTUtils.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRESTUtils.java index d0eda13..e162c5c 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRESTUtils.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRESTUtils.java @@ -40,16 +40,12 @@ public class RangerRESTUtils { public static final String REST_URL_SERVICE_GRANT_ACCESS = "/service/plugins/services/grant/"; public static final String REST_URL_SERVICE_REVOKE_ACCESS = "/service/plugins/services/revoke/"; - public static final String REST_URL_GET_UPDATED_TAGGED_RESOURCES = "/service/tag-def/v1/resources/updated-resources"; - public static final String REST_URL_SET_TAGGED_RESOURCE = "/service/tag-def/v1/resources/set-resource"; - public static final String REST_URL_SET_TAGGED_RESOURCES = "/service/tag-def/v1/resources/set-resources"; - public static final String REST_URL_UPDATE_TAGGED_RESOURCE = "/service/tag-def/v1/resources/update-resource"; - public static final String TAG_SERVICE_NAME_PARAM = "tagservicename"; - public static final String COMPONENT_TYPE_PARAM = "componenttype"; - public static final String TAG_TIMESTAMP_PARAM = "tagtimestamp"; - public static final String TAG_PATTERN_PARAM = "tagpattern"; - - public static final String REST_URL_LOOKUP_TAG_NAMES = "/service/tag-def/v1/resources/lookup-tags"; + public static final String REST_URL_GET_SERVICE_TAGS_IF_UPDATED = "/service/tags/download"; + public static final String SERVICE_NAME_PARAM = "serviceName"; + public static final String LAST_KNOWN_TAG_VERSION_PARAM = "tagVersion"; + public static final String PATTERN_PARAM = "pattern"; + + public static final String REST_URL_LOOKUP_TAG_NAMES = "/service/tags/lookup"; public static final String REST_EXPECTED_MIME_TYPE = "application/json" ; public static final String REST_MIME_TYPE_JSON = "application/json" ; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/83cb21e0/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java index a55a760..9dc4994 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java @@ -49,17 +49,25 @@ public class SearchFilter { public static final String PAGE_SIZE = "pageSize"; public static final String SORT_BY = "sortBy"; public static final String RESOURCE_SIGNATURE = "resourceSignature:"; // search + public static final String POLICY_TYPE = "policyType"; // search public static final String TAG_DEF_ID = "tagDefId"; // search + public static final String TAG_DEF_EXTERNAL_ID = "tagDefExtId"; // search public static final String TAG_DEF_NAME = "tagDefName"; // search - public static final String TAG_RESOURCE_ID = "tagResourceId"; // search - public static final String TAG_RESOURCE_SERVICE_NAME = "tagResourceServiceName"; // search - public static final String TAG_RESOURCE_TIMESTAMP = "tagResourceTimestamp"; // search - public static final String POLICY_TYPE = "policyType"; // search + public static final String TAG_ID = "tagId"; // search + public static final String TAG_EXTERNAL_ID = "tagExternalId"; // search + public static final String TAG_NAME = "tagName"; // search + public static final String TAG_RESOURCE_ID = "resourceId"; // search + public static final String TAG_RESOURCE_EXTERNAL_ID = "externalResourceId"; // search + public static final String TAG_RESOURCE_SERVICE_NAME = "resourceServiceName"; // search + public static final String TAG_RESOURCE_SIGNATURE = "resourceSignature"; // search + public static final String TAG_MAP_ID = "tagResourceMapId"; // search + public static final String TAG_MAP_RESOURCE_ID = "tagResourceMapResourceId"; // search + public static final String TAG_MAP_TAG_ID = "tagResourceMapTagId"; // search private Map<String, String> params = null; private int startIndex = 0; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/83cb21e0/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceTags.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceTags.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceTags.java new file mode 100644 index 0000000..3159b2b --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceTags.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.util; + + +import java.util.Date; +import java.util.List; +import java.util.Map; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.ranger.plugin.model.RangerServiceResource; +import org.apache.ranger.plugin.model.RangerTag; +import org.apache.ranger.plugin.model.RangerTagDef; +import org.codehaus.jackson.annotate.JsonAutoDetect; +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.annotate.JsonAutoDetect.Visibility; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +@JsonAutoDetect(getterVisibility=Visibility.NONE, setterVisibility=Visibility.NONE, fieldVisibility=Visibility.ANY) +@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL ) +@JsonIgnoreProperties(ignoreUnknown=true) +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class ServiceTags implements java.io.Serializable { + private static final long serialVersionUID = 1L; + + private String serviceName; + private Long tagVersion; + private Date tagUpdateTime; + Map<Long, RangerTagDef> tagDefinitions; + Map<Long, RangerTag> tags; + List<RangerServiceResource> serviceResources; + Map<Long, List<Long>> resourceToTagIds; + + /** + * @return the serviceName + */ + public String getServiceName() { + return serviceName; + } + + /** + * @param serviceName the serviceName to set + */ + public void setServiceName(String serviceName) { + this.serviceName = serviceName; + } + + /** + * @return the tagVersion + */ + public Long getTagVersion() { + return tagVersion; + } + + /** + * @param tagVersion the version to set + */ + public void setTagVersion(Long tagVersion) { + this.tagVersion = tagVersion; + } + + /** + * @return the tagUpdateTime + */ + public Date getTagUpdateTime() { + return tagUpdateTime; + } + + /** + * @param tagUpdateTime the tagUpdateTime to set + */ + public void setTagUpdateTime(Date tagUpdateTime) { + this.tagUpdateTime = tagUpdateTime; + } + + public Map<Long, RangerTagDef> getTagDefinitions() { + return tagDefinitions; + } + + public void setTagDefinitions(Map<Long, RangerTagDef> tagDefinitions) { + this.tagDefinitions = tagDefinitions; + } + + public Map<Long, RangerTag> getTags() { + return tags; + } + + public void setTags(Map<Long, RangerTag> tags) { + this.tags = tags; + } + + public List<RangerServiceResource> getServiceResources() { + return serviceResources; + } + + public void setServiceResources(List<RangerServiceResource> serviceResources) { + this.serviceResources = serviceResources; + } + + public Map<Long, List<Long>> getResourceToTagIds() { + return resourceToTagIds; + } + + public void setResourceToTagIds(Map<Long, List<Long>> resourceToTagIds) { + this.resourceToTagIds = resourceToTagIds; + } + + @Override + public String toString( ) { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("ServiceTags={").append("tagVersion=").append(tagVersion).append(", ") + .append("tagUpdateTime={").append(tagUpdateTime).append("} ") + .append("}"); + + return sb; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/83cb21e0/agents-common/src/main/java/org/apache/ranger/plugin/util/TagServiceResources.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/TagServiceResources.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/TagServiceResources.java deleted file mode 100644 index 9eef939..0000000 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/TagServiceResources.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.ranger.plugin.util; - - -import java.util.Date; -import java.util.List; - -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlRootElement; - -import org.apache.ranger.plugin.model.RangerTaggedResource; -import org.codehaus.jackson.annotate.JsonAutoDetect; -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.annotate.JsonAutoDetect.Visibility; -import org.codehaus.jackson.map.annotate.JsonSerialize; - -import com.google.common.base.Objects; - -@JsonAutoDetect(getterVisibility=Visibility.NONE, setterVisibility=Visibility.NONE, fieldVisibility=Visibility.ANY) -@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL ) -@JsonIgnoreProperties(ignoreUnknown=true) -@XmlRootElement -@XmlAccessorType(XmlAccessType.FIELD) -public class TagServiceResources implements java.io.Serializable { - private static final long serialVersionUID = 1L; - - private Long version; // RFU - private Date lastUpdateTime; // RFU - private List<RangerTaggedResource> taggedResources; - - public Long getVersion() { - return version; - } - /** - * @param version the version to set - */ - public void setVersion(Long version) { - this.version = version; - } - /** - * @return the lastUpdateTime - */ - public Date getLastUpdateTime() { - return lastUpdateTime; - } - /** - * @param lastUpdateTime the lastUpdateTime to set - */ - public void setLastUpdateTime(Date lastUpdateTime) { - this.lastUpdateTime = lastUpdateTime; - } - /** - * @return the tagged resources - */ - public List<RangerTaggedResource> getTaggedResources() { - return taggedResources; - } - /** - * @param taggedResources the taggedResources to set - */ - public void setTaggedResources(List<RangerTaggedResource> taggedResources) { - this.taggedResources = taggedResources; - } - - @Override - public String toString() { - return Objects.toStringHelper(this.getClass()) - .add("version", version) - .add("lastUpdateTime", lastUpdateTime) - .add("taggedResources", taggedResources) - .toString(); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/83cb21e0/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java ---------------------------------------------------------------------- diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java b/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java index 269a27f..969e7d7 100644 --- a/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java +++ b/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java @@ -31,7 +31,7 @@ import org.apache.ranger.audit.provider.AuditProviderFactory; import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; import org.apache.ranger.plugin.audit.RangerDefaultAuditHandler; import org.apache.ranger.plugin.model.RangerPolicy; -import org.apache.ranger.plugin.model.RangerTaggedResource; +import org.apache.ranger.plugin.model.RangerTag; import org.apache.ranger.plugin.model.RangerServiceDef; import org.apache.ranger.plugin.policyengine.TestPolicyEngine.PolicyEngineTestCase.TestData; import org.apache.ranger.plugin.util.RangerRequestedResources; @@ -241,21 +241,19 @@ public class TestPolicyEngine { newRequest.setSessionId(test.request.getSessionId()); Map<String, Object> context = test.request.getContext(); - if (test.request.getContext().containsKey(RangerPolicyEngine.KEY_CONTEXT_TAGS)) { - String tagsJsonString = (String) context.get(RangerPolicyEngine.KEY_CONTEXT_TAGS); - context.remove(RangerPolicyEngine.KEY_CONTEXT_TAGS); - - if (!StringUtils.isEmpty(tagsJsonString)) { - try { - Type listType = new TypeToken<List<RangerTaggedResource.RangerResourceTag>>() { - }.getType(); - List<RangerTaggedResource.RangerResourceTag> tagList = gsonBuilder.fromJson(tagsJsonString, listType); - - context.put(RangerPolicyEngine.KEY_CONTEXT_TAGS, tagList); - } catch (Exception e) { - System.err.println("TestPolicyEngine.runTests(): error parsing TAGS JSON string in file " + testName + ", tagsJsonString=" + - tagsJsonString + ", exception=" + e); - } + String tagsJsonString = (String) context.get(RangerPolicyEngine.KEY_CONTEXT_TAGS); + context.remove(RangerPolicyEngine.KEY_CONTEXT_TAGS); + + if(!StringUtils.isEmpty(tagsJsonString)) { + try { + Type listType = new TypeToken<List<RangerTag>>() { + }.getType(); + List<RangerTag> tagList = gsonBuilder.fromJson(tagsJsonString, listType); + + context.put(RangerPolicyEngine.KEY_CONTEXT_TAGS, tagList); + } catch (Exception e) { + System.err.println("TestPolicyEngine.runTests(): error parsing TAGS JSON string in file " + testName + ", tagsJsonString=" + + tagsJsonString + ", exception=" + e); } } else if (test.request.getContext().containsKey(RangerRequestedResources.KEY_CONTEXT_REQUESTED_RESOURCES)) { String resourcesJsonString = (String) context.get(RangerRequestedResources.KEY_CONTEXT_REQUESTED_RESOURCES); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/83cb21e0/agents-common/src/test/java/org/apache/ranger/plugin/store/TestTagStore.java ---------------------------------------------------------------------- diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/store/TestTagStore.java b/agents-common/src/test/java/org/apache/ranger/plugin/store/TestTagStore.java index e36f318..797b083 100644 --- a/agents-common/src/test/java/org/apache/ranger/plugin/store/TestTagStore.java +++ b/agents-common/src/test/java/org/apache/ranger/plugin/store/TestTagStore.java @@ -22,17 +22,26 @@ package org.apache.ranger.plugin.store; import static org.junit.Assert.*; import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; import java.io.OutputStreamWriter; import java.util.*; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import org.apache.commons.collections.CollectionUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.*; import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; import org.apache.ranger.plugin.model.*; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource; +import org.apache.ranger.plugin.policyengine.RangerAccessRequest; +import org.apache.ranger.plugin.policyengine.RangerAccessResource; +import org.apache.ranger.plugin.store.file.ServiceFileStore; import org.apache.ranger.plugin.store.file.TagFileStore; import org.apache.ranger.plugin.store.rest.ServiceRESTStore; import org.apache.ranger.plugin.util.SearchFilter; +import org.apache.ranger.plugin.util.ServiceTags; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -40,47 +49,42 @@ import org.junit.Test; public class TestTagStore { static TagStore tagStore = null; - static SearchFilter filter = null; + static TagValidator validator = null; + + static SearchFilter filter = null; static Path filePath = new Path("file:///etc/ranger/data/ranger-admin-test-site.xml"); static Configuration config = new Configuration(); - static final String sdName = "tagDef-unit-test-TestTagStore"; + static final String serviceDefJsonFile = "/admin/service-defs/test-hive-servicedef.json"; static final String serviceName = "tag-unit-test-TestTagStore"; - static final String policyName = "tag-1"; + + static final String crcSuffix = ".crc"; + static final String jsonSuffix = ".json"; + + static Gson gsonBuilder = null; + static RangerServiceDef serviceDef = null; + static RangerService service = null; @BeforeClass public static void setupTest() throws Exception { - /* - tearDownAfterClass(); + tearDownAfterClass(crcSuffix); + tearDownAfterClass(jsonSuffix); FileSystem fs = filePath.getFileSystem(config); - FSDataOutputStream outStream = fs.create(filePath, true); + FSDataOutputStream outStream = fs.create(filePath, true); OutputStreamWriter writer = null; - writer = new OutputStreamWriter(outStream); writer.write("<configuration>\n" + " <property>\n" + - " <name>ranger.service.store.rest.url</name>\n" + - " <value>http://node-1.example.com:6080</value>\n" + - " </property>\n" + - " <property>\n" + - " <name>ranger.service.store.rest.basicauth.username</name>\n" + - " <value>admin</value>\n" + - " </property>\n" + - " <property>\n" + - " <name>ranger.service.store.rest.basicauth.password</name>\n" + - " <value>admin</value>\n" + - " </property>\n" + - " <property>\n" + - " <name>ranger.plugin.tag.policy.rest.url</name>\n" + - " <value>http://node-1.example.com:6080</value>\n" + + " <name>ranger.tag.store.file.dir</name>\n" + + " <value>file:///etc/ranger/data</value>\n" + " </property>\n" + " <property>\n" + - " <name>ranger.tag.store.file.dir</name>\n" + + " <name>ranger.service.store.file.dir</name>\n" + " <value>file:///etc/ranger/data</value>\n" + " </property>\n" + "</configuration>\n"); @@ -93,98 +97,235 @@ public class TestTagStore { tagStore = TagFileStore.getInstance(); tagStore.init(); - ServiceStore svcStore = new ServiceRESTStore(); + ServiceStore svcStore; + + svcStore = new ServiceFileStore(); svcStore.init(); tagStore.setServiceStore(svcStore); - */ + + validator = new TagValidator(); + + validator.setTagStore(tagStore); + + gsonBuilder = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z") + .setPrettyPrinting() + .create(); + + InputStream inStream = TestTagStore.class.getResourceAsStream(serviceDefJsonFile); + InputStreamReader reader = new InputStreamReader(inStream); + serviceDef = gsonBuilder.fromJson(reader, RangerServiceDef.class); + service = svcStore.createService(new RangerService(serviceDef.getName(), serviceName, serviceName, null, null)); + reader.close(); + inStream.close(); + } //@AfterClass - public static void tearDownAfterClass() throws Exception { + public static void tearDownAfterClass(String suffix) throws Exception { - /* Path dirPath = new Path("file:///etc/ranger/data"); FileSystem fs = dirPath.getFileSystem(config); try { - if(fs.exists(dirPath) && fs.isDirectory(dirPath)) { - PathFilter filter = new PathFilter() { - @Override - public boolean accept(Path path) { - return path.getName().endsWith(".json") || - path.getName().endsWith(".crc"); - } - }; + if (fs.exists(dirPath) && fs.isDirectory(dirPath)) { RemoteIterator<LocatedFileStatus> files = fs.listFiles(dirPath, false); - if(files != null) { + if (files != null) { while (files.hasNext()) { LocatedFileStatus fileStatus = files.next(); Path path = fileStatus.getPath(); - if (fs.isFile(path) && path.getName().endsWith(".json") || path.getName().endsWith(".crc")) { + if (fs.isFile(path) && path.getName().endsWith(suffix)) { fs.delete(path, true); } } } } - } catch(IOException excp) { + } catch (IOException excp) { } fs.delete(filePath, true); - */ } @Test - public void testTagStore() throws Exception { + public void testTagStore_tag() throws Exception { + + String tagName = "ssn"; + String newTagName = "new-ssn"; + + List<RangerTag> tags = tagStore.getTags(filter); + + int initTagCount = tags == null ? 0 : tags.size(); + + RangerTag tag = new RangerTag(tagName, new HashMap<String, String>()); + tag.setGuid("GUID_TAG_TEST"); + + validator.preCreateTag(tag); + RangerTag createdTag = tagStore.createTag(tag); + + assertNotNull("createTag() failed", createdTag); + assertTrue("createTag() name mismatch", createdTag.getName().equals(tag.getName())); + assertTrue("createTag() GUID mismatch", createdTag.getGuid().equals(tag.getGuid())); + + tags = tagStore.getTags(filter); + + assertEquals("createTag() failed", initTagCount + 1, tags == null ? 0 : tags.size()); + + createdTag.setName(newTagName); + validator.preUpdateTagById(createdTag.getId(), createdTag); + RangerTag updatedTag = tagStore.updateTag(createdTag); + + tag = tagStore.getTagById(updatedTag.getId()); + + assertTrue("updateTag() name mismatch", tag.getName().equals(updatedTag.getName())); + assertTrue("updatedTag() GUID mismatch", tag.getGuid().equals(updatedTag.getGuid())); + + validator.preDeleteTagById(createdTag.getId()); + tagStore.deleteTagById(createdTag.getId()); + + tags = tagStore.getTags(filter); + + assertEquals("deleteTag() failed", initTagCount, tags == null ? 0 : tags.size()); + + // Try deleting it again + try { + validator.preDeleteTagById(createdTag.getId()); + tagStore.deleteTagById(createdTag.getId()); + assertTrue("deleteTag() failed. Deleted tag again successfully? ", false); + } catch (Exception exception) { + assertTrue(true); + } + } + + @Test + public void testTagStore_serviceresource() throws Exception { + + String externalId = "GUID_SERVICERESOURCE_TEST"; + String newExternalId = "NEW_GUID_SERVICERESOURCE_TEST"; + + Map<String, RangerPolicyResource> resourceResources = new HashMap<String, RangerPolicyResource>(); + + RangerPolicyResource resource = new RangerPolicyResource(); + resource.setValue("*"); + resourceResources.put("database", resource); + + List<RangerServiceResource> serviceResources = tagStore.getServiceResources(filter); + + int initServiceResourceCount = serviceResources == null ? 0 : serviceResources.size(); + + RangerServiceResource serviceResource = new RangerServiceResource(); + serviceResource.setServiceName(serviceName); + serviceResource.setResourceSpec(resourceResources); + serviceResource.setGuid(externalId); + + validator.preCreateServiceResource(serviceResource); + RangerServiceResource createdServiceResource = tagStore.createServiceResource(serviceResource); + + assertNotNull("createServiceResource() failed", createdServiceResource); + assertTrue("createServiceResource() GUID mismatch", createdServiceResource.getGuid().equals(createdServiceResource.getGuid())); + + serviceResources = tagStore.getServiceResources(filter); + + assertEquals("createServiceResource() failed", initServiceResourceCount + 1, serviceResources == null ? 0 : serviceResources.size()); + + createdServiceResource.setGuid(newExternalId); + validator.preUpdateServiceResourceById(createdServiceResource.getId(), createdServiceResource); + RangerServiceResource updatedServiceResource = tagStore.updateServiceResource(createdServiceResource); + + serviceResource = tagStore.getServiceResourceById(updatedServiceResource.getId()); + + assertTrue("updatedServiceResource() GUID mismatch", serviceResource.getGuid().equals(updatedServiceResource.getGuid())); + + validator.preDeleteServiceResourceById(updatedServiceResource.getId()); + tagStore.deleteServiceResourceById(updatedServiceResource.getId()); + + serviceResources = tagStore.getServiceResources(filter); + + assertEquals("deleteServiceResource() failed", initServiceResourceCount, serviceResources == null ? 0 : serviceResources.size()); + + // Try deleting it again + try { + validator.preDeleteServiceResourceById(createdServiceResource.getId()); + tagStore.deleteServiceResourceById(createdServiceResource.getId()); + assertTrue("deleteServiceResource() failed. Deleted serviceResource again successfully? ", false); + } catch (Exception exception) { + assertTrue(true); + } + } + + @Test + public void testTagStore_tagResourceMap() throws Exception { + + String tagName = "ssn"; + + String externalResourceId = "GUID_SERVICERESOURCE_TEST"; + String externalTagId = "GUID_TAG_TEST"; + + List<RangerTag> tags = tagStore.getTags(filter); + + int initTagCount = tags == null ? 0 : tags.size(); + + RangerTag tag = new RangerTag(tagName, new HashMap<String, String>()); + tag.setGuid(externalTagId); - /* - List<RangerTaggedResource> taggedResources = tagStore.getResources(filter); - RangerTaggedResource taggedResource = null; + validator.preCreateTag(tag); + RangerTag createdTag = tagStore.createTag(tag); - int initResourceCount = taggedResources == null ? 0 : taggedResources.size(); + assertNotNull("createTag() failed", createdTag); + tags = tagStore.getTags(filter); - RangerTaggedResource rr = new RangerTaggedResource(); - rr.getKey().setComponentType("hive"); - rr.getKey().setTagServiceName("tagdev"); + assertEquals("createTag() failed", initTagCount + 1, tags == null ? 0 : tags.size()); - Map<String, RangerPolicyResource> resourceSpec = new HashMap<>(); + Map<String, RangerPolicyResource> resourceResources = new HashMap<String, RangerPolicyResource>(); - RangerPolicyResource policyResource = new RangerPolicyResource(); - policyResource.setValues(Arrays.asList("default", "hr", "finance")); - resourceSpec.put("database", policyResource); + RangerPolicyResource resource = new RangerPolicyResource(); + resource.setValue("*"); + resourceResources.put("database", resource); - policyResource = new RangerPolicyResource(); - policyResource.setValues(Arrays.asList("table1", "employee", "invoice")); - resourceSpec.put("table", policyResource); + List<RangerServiceResource> serviceResources = tagStore.getServiceResources(filter); - policyResource = new RangerPolicyResource(); - policyResource.setValues(Arrays.asList("column1", "ssn", "vendor")); - resourceSpec.put("column", policyResource); + int initServiceResourceCount = serviceResources == null ? 0 : serviceResources.size(); - rr.getKey().setResourceSpec(resourceSpec); + RangerServiceResource serviceResource = new RangerServiceResource(); + serviceResource.setServiceName(serviceName); + serviceResource.setResourceSpec(resourceResources); - List<RangerTaggedResource.RangerResourceTag> tags = new ArrayList<>(); + serviceResource.setGuid(externalResourceId); + validator.preCreateServiceResource(serviceResource); + RangerServiceResource createdServiceResource = tagStore.createServiceResource(serviceResource); - tags.add(new RangerTaggedResource.RangerResourceTag("PII", null)); - tags.add(new RangerTaggedResource.RangerResourceTag("FINANCE", null)); + assertNotNull("createServiceResource() failed", createdServiceResource); - rr.setTags(tags); + serviceResources = tagStore.getServiceResources(filter); - RangerTaggedResource createdResource = tagStore.createResource(rr, false); + assertEquals("createServiceResource() failed", initServiceResourceCount + 1, serviceResources == null ? 0 : serviceResources.size()); - assertNotNull("createResource() failed", createdResource); + // Now create map - taggedResources = tagStore.getResources(filter); - assertEquals("createResource() failed", initResourceCount + 1, taggedResources == null ? 0 : taggedResources.size()); + RangerTagResourceMap tagResourceMap = validator.preCreateTagResourceMap(externalResourceId, externalTagId); - taggedResource = tagStore.getResource(rr.getKey()); - assertNotNull("createResource() failed", taggedResource); + RangerTagResourceMap createdTagResourceMap = tagStore.createTagResourceMap(tagResourceMap); + + assertNotNull("createTagResourceMap() failed", createdTagResourceMap); + + ServiceTags serviceTags = tagStore.getServiceTagsIfUpdated(serviceName, -1L); + List<RangerServiceResource> resourceList = serviceTags.getServiceResources(); + + assertTrue("No tagged resources found!", CollectionUtils.isNotEmpty(resourceList) && CollectionUtils.size(resourceList) == 1); + + // Delete all created entities + RangerTagResourceMap map = validator.preDeleteTagResourceMap(externalResourceId, externalTagId); + tagStore.deleteTagResourceMapById(map.getId()); + + validator.preDeleteServiceResourceById(createdServiceResource.getId()); + tagStore.deleteServiceResourceById(createdServiceResource.getId()); + + validator.preDeleteTagById(createdTag.getId()); + tagStore.deleteTagById(createdTag.getId()); - rr.getKey().getResourceSpec().remove("column"); - taggedResource = tagStore.getResource(rr.getKey()); - assertNull("createResource() failed", taggedResource); - */ } } + + +
