Repository: incubator-ranger Updated Branches: refs/heads/master 73b22d3eb -> 5ac7c52ad
RANGER-359 Store policy resource signature in database and use stored value to check for duplicates Signed-off-by: Madhan Neethiraj <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/5ac7c52a Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/5ac7c52a Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/5ac7c52a Branch: refs/heads/master Commit: 5ac7c52ad1b9f4d68cd2226e29af412c127d5c53 Parents: 73b22d3 Author: Alok Lal <[email protected]> Authored: Fri May 1 17:47:40 2015 -0700 Committer: Madhan Neethiraj <[email protected]> Committed: Mon May 4 13:50:16 2015 -0700 ---------------------------------------------------------------------- .../model/RangerPolicyResourceSignature.java | 161 +++++++++++++ .../RangerPolicyResourceSignature.java | 162 ------------- .../model/validation/RangerPolicyValidator.java | 53 ++--- .../model/validation/RangerValidator.java | 19 ++ .../plugin/store/AbstractServiceStore.java | 49 +++- .../ranger/plugin/store/ServiceStore.java | 2 + .../plugin/store/file/ServiceFileStore.java | 17 ++ .../plugin/store/rest/ServiceRESTStore.java | 5 + .../ranger/plugin/util/RangerObjectFactory.java | 2 +- .../apache/ranger/plugin/util/SearchFilter.java | 24 ++ .../TestRangerPolicyResourceSignature.java | 234 +++++++++++++++++++ .../TestRangerPolicyResourceSignature.java | 232 ------------------ .../validation/TestRangerPolicyValidator.java | 76 +++--- .../model/validation/TestRangerValidator.java | 22 +- .../model/validation/ValidationTestUtils.java | 2 +- .../org/apache/ranger/biz/ServiceDBStore.java | 40 +++- .../apache/ranger/service/RangerFactory.java | 33 +++ 17 files changed, 653 insertions(+), 480 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/5ac7c52a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerPolicyResourceSignature.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerPolicyResourceSignature.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerPolicyResourceSignature.java new file mode 100644 index 0000000..62fdabc --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerPolicyResourceSignature.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.model; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.TreeMap; + +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource; + +public class RangerPolicyResourceSignature { + + private static final Log LOG = LogFactory.getLog(RangerPolicyResourceSignature.class); + static final RangerPolicyResourceSignature _EmptyResourceSignature = new RangerPolicyResourceSignature((RangerPolicy)null); + + private final String _string; + private final String _hash; + private final RangerPolicy _policy; + + public RangerPolicyResourceSignature(RangerPolicy policy) { + _policy = policy; + String asString = getResourceString(_policy); + if (asString == null) { + _string = ""; + } else { + _string = asString; + } + _hash = DigestUtils.md5Hex(_string); + } + + /** + * Only added for testability. Do not make public + * @param string + */ + RangerPolicyResourceSignature(String string) { + _policy = null; + if (string == null) { + _string = ""; + } else { + _string = string; + } + _hash = DigestUtils.md5Hex(_string); + } + + String asString() { + return _string; + } + + public String getSignature() { + return _hash; + } + + @Override + public int hashCode() { + // we assume no collision + return Objects.hashCode(_hash); + } + + @Override + public boolean equals(Object object) { + if (object == null || !(object instanceof RangerPolicyResourceSignature)) { + return false; + } + RangerPolicyResourceSignature that = (RangerPolicyResourceSignature)object; + return Objects.equals(this._hash, that._hash); + } + + @Override + public String toString() { + return String.format("%s: %s", _hash, _string); + } + + String getResourceString(RangerPolicy policy) { + // invalid/empty policy gets a deterministic signature as if it had an + // empty resource string + if (!isPolicyValidForResourceSignatureComputation(policy)) { + return null; + } + Map<String, RangerPolicyResourceView> resources = new TreeMap<String, RangerPolicyResourceView>(); + for (Map.Entry<String, RangerPolicyResource> entry : policy.getResources().entrySet()) { + String resourceName = entry.getKey(); + RangerPolicyResourceView resourceView = new RangerPolicyResourceView(entry.getValue()); + resources.put(resourceName, resourceView); + } + String result = resources.toString(); + return result; + } + + boolean isPolicyValidForResourceSignatureComputation(RangerPolicy policy) { + boolean valid = false; + if (policy == null) { + LOG.debug("isPolicyValidForResourceSignatureComputation: policy was null!"); + } else if (policy.getResources() == null) { + LOG.debug("isPolicyValidForResourceSignatureComputation: resources collection on policy was null!"); + } else if (policy.getResources().containsKey(null)) { + LOG.debug("isPolicyValidForResourceSignatureComputation: resources collection has resource with null name!"); + } else { + valid = true; + } + return valid; + } + + static class RangerPolicyResourceView { + final RangerPolicyResource _policyResource; + + RangerPolicyResourceView(RangerPolicyResource policyResource) { + _policyResource = policyResource; + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("{"); + if (_policyResource != null) { + builder.append("values="); + if (_policyResource.getValues() != null) { + List<String> values = new ArrayList<String>(_policyResource.getValues()); + Collections.sort(values); + builder.append(values); + } + builder.append(",excludes="); + if (_policyResource.getIsExcludes() == null) { // null is same as false + builder.append(Boolean.FALSE); + } else { + builder.append(_policyResource.getIsExcludes()); + } + builder.append(",recursive="); + if (_policyResource.getIsRecursive() == null) { // null is the same as false + builder.append(Boolean.FALSE); + } else { + builder.append(_policyResource.getIsRecursive()); + } + } + builder.append("}"); + return builder.toString(); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/5ac7c52a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerPolicyResourceSignature.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerPolicyResourceSignature.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerPolicyResourceSignature.java deleted file mode 100644 index 9b21dc9..0000000 --- a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerPolicyResourceSignature.java +++ /dev/null @@ -1,162 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.ranger.plugin.model.validation; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.TreeMap; - -import org.apache.commons.codec.digest.DigestUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.ranger.plugin.model.RangerPolicy; -import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource; - -public class RangerPolicyResourceSignature { - - private static final Log LOG = LogFactory.getLog(RangerPolicyResourceSignature.class); - static final RangerPolicyResourceSignature _EmptyResourceSignature = new RangerPolicyResourceSignature((RangerPolicy)null); - - private final String _string; - private final String _hash; - private final RangerPolicy _policy; - - public RangerPolicyResourceSignature(RangerPolicy policy) { - _policy = policy; - String asString = getResourceString(_policy); - if (asString == null) { - _string = ""; - } else { - _string = asString; - } - _hash = DigestUtils.md5Hex(_string); - } - - /** - * Only added for testability. Do not make public - * @param string - */ - RangerPolicyResourceSignature(String string) { - _policy = null; - if (string == null) { - _string = ""; - } else { - _string = string; - } - _hash = DigestUtils.md5Hex(_string); - } - - public String asString() { - return _string; - } - - public String asHashHex() { - return _hash; - } - - @Override - public int hashCode() { - // we assume no collision - return Objects.hashCode(_hash); - } - - @Override - public boolean equals(Object object) { - if (object == null || !(object instanceof RangerPolicyResourceSignature)) { - return false; - } - RangerPolicyResourceSignature that = (RangerPolicyResourceSignature)object; - return Objects.equals(this._hash, that._hash); - } - - @Override - public String toString() { - return String.format("%s: %s", _hash, _string); - } - - String getResourceString(RangerPolicy policy) { - // invalid/empty policy gets a deterministic signature as if it had an - // empty resource string - if (!isPolicyValidForResourceSignatureComputation(policy)) { - return null; - } - Map<String, RangerPolicyResourceView> resources = new TreeMap<String, RangerPolicyResourceView>(); - for (Map.Entry<String, RangerPolicyResource> entry : policy.getResources().entrySet()) { - String resourceName = entry.getKey(); - RangerPolicyResourceView resourceView = new RangerPolicyResourceView(entry.getValue()); - resources.put(resourceName, resourceView); - } - String result = resources.toString(); - return result; - } - - boolean isPolicyValidForResourceSignatureComputation(RangerPolicy policy) { - boolean valid = false; - if (policy == null) { - LOG.debug("isPolicyValidForResourceSignatureComputation: policy was null!"); - } else if (policy.getResources() == null) { - LOG.debug("isPolicyValidForResourceSignatureComputation: resources collection on policy was null!"); - } else if (policy.getResources().containsKey(null)) { - LOG.debug("isPolicyValidForResourceSignatureComputation: resources collection has resource with null name!"); - } else { - valid = true; - } - return valid; - } - - static class RangerPolicyResourceView { - final RangerPolicyResource _policyResource; - - RangerPolicyResourceView(RangerPolicyResource policyResource) { - _policyResource = policyResource; - } - - @Override - public String toString() { - StringBuilder builder = new StringBuilder(); - builder.append("{"); - if (_policyResource != null) { - builder.append("values="); - if (_policyResource.getValues() != null) { - List<String> values = new ArrayList<String>(_policyResource.getValues()); - Collections.sort(values); - builder.append(values); - } - builder.append(",excludes="); - if (_policyResource.getIsExcludes() == null) { // null is same as false - builder.append(Boolean.FALSE); - } else { - builder.append(_policyResource.getIsExcludes()); - } - builder.append(",recursive="); - if (_policyResource.getIsRecursive() == null) { // null is the same as false - builder.append(Boolean.FALSE); - } else { - builder.append(_policyResource.getIsRecursive()); - } - } - builder.append("}"); - return builder.toString(); - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/5ac7c52a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerPolicyValidator.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerPolicyValidator.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerPolicyValidator.java index a76e970..0092aaf 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerPolicyValidator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerPolicyValidator.java @@ -33,6 +33,7 @@ import org.apache.ranger.plugin.model.RangerPolicy; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItem; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemAccess; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource; +import org.apache.ranger.plugin.model.RangerPolicyResourceSignature; import org.apache.ranger.plugin.model.RangerService; import org.apache.ranger.plugin.model.RangerServiceDef; import org.apache.ranger.plugin.model.RangerServiceDef.RangerResourceDef; @@ -240,13 +241,13 @@ public class RangerPolicyValidator extends RangerValidator { boolean valid = true; Map<String, RangerPolicyResource> resourceMap = policy.getResources(); - if (serviceDef != null && resourceMap != null) { // following checks can't be done meaningfully otherwise - valid = isValidResourceNames(policy, failures, serviceDef) && valid; - valid = isValidResourceValues(resourceMap, failures, serviceDef) && valid; - valid = isValidResourceFlags(resourceMap, failures, serviceDef.getResources(), serviceDef.getName(), policy.getName(), isAdmin) && valid; - } - if (StringUtils.isNotBlank(serviceName)) { // resource uniqueness check cannot be done meaningfully otherwise - valid = isPolicyResourceUnique(policy, failures, action, serviceName) && valid; + if (resourceMap != null) { // following checks can't be done meaningfully otherwise + valid = isPolicyResourceUnique(policy, failures) && valid; + if (serviceDef != null) { // following checks can't be done meaningfully otherwise + valid = isValidResourceNames(policy, failures, serviceDef) && valid; + valid = isValidResourceValues(resourceMap, failures, serviceDef) && valid; + valid = isValidResourceFlags(resourceMap, failures, serviceDef.getResources(), serviceDef.getName(), policy.getName(), isAdmin) && valid; + } } if(LOG.isDebugEnabled()) { @@ -255,38 +256,28 @@ public class RangerPolicyValidator extends RangerValidator { return valid; } - boolean isPolicyResourceUnique(RangerPolicy policy, final List<ValidationFailureDetails> failures, Action action, final String serviceName) { + boolean isPolicyResourceUnique(RangerPolicy policy, final List<ValidationFailureDetails> failures) { if(LOG.isDebugEnabled()) { - LOG.debug(String.format("==> RangerPolicyValidator.isPolicyResourceUnique(%s, %s, %s, %s)", policy, failures, action, serviceName)); + LOG.debug(String.format("==> RangerPolicyValidator.isPolicyResourceUnique(%s, %s)", policy, failures)); } - boolean foundDuplicate = false; - RangerPolicyResourceSignature signature = _factory.createPolicyResourceSignature(policy); - List<RangerPolicy> policies = getPolicies(serviceName, null); + boolean valid = true; + RangerPolicyResourceSignature policySignature = _factory.createPolicyResourceSignature(policy); + String signature = policySignature.getSignature(); + List<RangerPolicy> policies = getPoliciesForResourceSignature(signature); if (CollectionUtils.isNotEmpty(policies)) { - Iterator<RangerPolicy> iterator = policies.iterator(); - while (iterator.hasNext() && !foundDuplicate) { - RangerPolicy otherPolicy = iterator.next(); - if (otherPolicy.getId().equals(policy.getId()) && action == Action.UPDATE) { - LOG.debug("isPolicyResourceUnique: Skipping self during update!"); - } else { - RangerPolicyResourceSignature otherSignature = _factory.createPolicyResourceSignature(otherPolicy); - if (signature.equals(otherSignature)) { - foundDuplicate = true; - failures.add(new ValidationFailureDetailsBuilder() - .field("resources") - .isSemanticallyIncorrect() - .becauseOf("found another policy[" + otherPolicy.getName() + "] with matching resources[" + otherPolicy.getResources() + "]!") - .build()); - } - } - } + RangerPolicy otherPolicy = policies.iterator().next(); + valid = false; + failures.add(new ValidationFailureDetailsBuilder() + .field("resources") + .isSemanticallyIncorrect() + .becauseOf("found another policy[" + otherPolicy.getName() + "] with matching resources[" + otherPolicy.getResources() + "]!") + .build()); } - boolean valid = !foundDuplicate; if(LOG.isDebugEnabled()) { - LOG.debug(String.format("<== RangerPolicyValidator.isPolicyResourceUnique(%s, %s, %s, %s): %s", policy, failures, action, serviceName, valid)); + LOG.debug(String.format("<== RangerPolicyValidator.isPolicyResourceUnique(%s, %s): %s", policy, failures, valid)); } return valid; } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/5ac7c52a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerValidator.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerValidator.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerValidator.java index 492949b..bc4c7f1 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerValidator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerValidator.java @@ -268,6 +268,25 @@ public abstract class RangerValidator { } return policies; } + + List<RangerPolicy> getPoliciesForResourceSignature(String hexSignature) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerValidator.getPolicies(" + hexSignature + ")"); + } + + List<RangerPolicy> policies = null; + try { + policies = _store.getPoliciesByResourceSignature(hexSignature); + } catch (Exception e) { + LOG.debug("Encountred exception while retrieving policies from service store!", e); + } + + if(LOG.isDebugEnabled()) { + int count = policies == null ? 0 : policies.size(); + LOG.debug("<== RangerValidator.getPolicies(" + hexSignature + "): count[" + count + "], " + policies); + } + return policies; + } Set<String> getAccessTypes(RangerServiceDef serviceDef) { if(LOG.isDebugEnabled()) { http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/5ac7c52a/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java index ee480fa..2ce08bb 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java @@ -36,10 +36,10 @@ import org.apache.commons.lang.ObjectUtils; import org.apache.commons.lang.StringUtils; import org.apache.ranger.plugin.model.RangerBaseModelObject; import org.apache.ranger.plugin.model.RangerPolicy; -import org.apache.ranger.plugin.model.RangerService; -import org.apache.ranger.plugin.model.RangerServiceDef; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItem; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource; +import org.apache.ranger.plugin.model.RangerService; +import org.apache.ranger.plugin.model.RangerServiceDef; import org.apache.ranger.plugin.model.RangerServiceDef.RangerResourceDef; import org.apache.ranger.plugin.util.SearchFilter; @@ -81,6 +81,7 @@ public abstract class AbstractServiceStore implements ServiceStore { addPredicateForIsRecursive(filter.getParam(SearchFilter.IS_RECURSIVE), predicates); addPredicateForUserName(filter.getParam(SearchFilter.USER), predicates); addPredicateForGroupName(filter.getParam(SearchFilter.GROUP), predicates); + addPredicateForResourceSignature(filter.getParam(SearchFilter.RESOURCE_SIGNATURE), predicates); addPredicateForResources(filter.getParamsWithPrefix(SearchFilter.RESOURCE_PREFIX, true), predicates); Predicate ret = CollectionUtils.isEmpty(predicates) ? null : PredicateUtils.allPredicate(predicates); @@ -683,4 +684,48 @@ public abstract class AbstractServiceStore implements ServiceStore { return ret; } + + private Predicate addPredicateForResourceSignature(final String hexSignature, List<Predicate> predicates) { + + Predicate ret = createPredicateForResourceSignature(hexSignature); + + if(predicates != null && ret != null) { + predicates.add(ret); + } + + return ret; + } + + /** + * NOTE: Null or empty hexSignature, though invalid, is supported in search. + * @param hexSignature + * @return + */ + public Predicate createPredicateForResourceSignature(final String hexSignature) { + + if(StringUtils.isEmpty(hexSignature)) { + return null; + } + + return new Predicate() { + @Override + public boolean evaluate(Object object) { + if(object == null) { + return false; + } + + boolean ret = false; + + if (object instanceof RangerPolicy) { + RangerPolicy policy = (RangerPolicy)object; + + ret = StringUtils.equals(hexSignature, policy.getResourceSignature()); + } else { + ret = true; + } + + return ret; + } + }; + } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/5ac7c52a/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java index e8d970c..708fbd2 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java @@ -66,6 +66,8 @@ public interface ServiceStore { List<RangerPolicy> getPolicies(SearchFilter filter) throws Exception; + List<RangerPolicy> getPoliciesByResourceSignature(String hexSignature) throws Exception; + List<RangerPolicy> getServicePolicies(Long serviceId, SearchFilter filter) throws Exception; List<RangerPolicy> getServicePolicies(String serviceName, SearchFilter filter) throws Exception; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/5ac7c52a/agents-common/src/main/java/org/apache/ranger/plugin/store/file/ServiceFileStore.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/file/ServiceFileStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/file/ServiceFileStore.java index b90de22..00b7521 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/file/ServiceFileStore.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/file/ServiceFileStore.java @@ -615,6 +615,23 @@ public class ServiceFileStore extends BaseFileStore { } @Override + public List<RangerPolicy> getPoliciesByResourceSignature(String hexSignature) throws Exception { + if (LOG.isDebugEnabled()) { + LOG.debug("==> ServiceFileStore.getPolicies()"); + } + + List<RangerPolicy> ret = getAllPolicies(); + + CollectionUtils.filter(ret, createPredicateForResourceSignature(hexSignature)); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== ServiceFileStore.getPolicies(): count=" + (ret == null ? 0 : ret.size())); + } + + return ret; + } + + @Override public List<RangerPolicy> getPolicies(SearchFilter filter) throws Exception { if(LOG.isDebugEnabled()) { LOG.debug("==> ServiceFileStore.getPolicies()"); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/5ac7c52a/agents-common/src/main/java/org/apache/ranger/plugin/store/rest/ServiceRESTStore.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/rest/ServiceRESTStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/rest/ServiceRESTStore.java index ca8024f..5c742f9 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/rest/ServiceRESTStore.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/rest/ServiceRESTStore.java @@ -612,4 +612,9 @@ public class ServiceRESTStore implements ServiceStore { return ret; } + + @Override + public List<RangerPolicy> getPoliciesByResourceSignature(String hexSignature) throws Exception { + throw new UnsupportedOperationException("Querying policies by resource signature is not supported!"); + } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/5ac7c52a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerObjectFactory.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerObjectFactory.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerObjectFactory.java index 0faa20d..72f0fd5 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerObjectFactory.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerObjectFactory.java @@ -20,7 +20,7 @@ package org.apache.ranger.plugin.util; import org.apache.ranger.plugin.model.RangerPolicy; -import org.apache.ranger.plugin.model.validation.RangerPolicyResourceSignature; +import org.apache.ranger.plugin.model.RangerPolicyResourceSignature; public class RangerObjectFactory { public RangerPolicyResourceSignature createPolicyResourceSignature(RangerPolicy policy) { http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/5ac7c52a/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java index dac8a8e..17738be 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java @@ -46,6 +46,7 @@ public class SearchFilter { public static final String START_INDEX = "startIndex"; public static final String PAGE_SIZE = "pageSize"; public static final String SORT_BY = "sortBy"; + public static final String RESOURCE_SIGNATURE = "resourceSignature:"; // search private Map<String, String> params = null; private int startIndex = 0; @@ -175,4 +176,27 @@ public class SearchFilter { public int hashCode() { return Objects.hash(params); } + + @Override + public String toString( ) { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("SearchFilter={"); + + sb.append("getCount={").append(getCount).append("} "); + sb.append("maxRows={").append(maxRows).append("} "); + sb.append("params={").append(params).append("} "); + sb.append("sortBy={").append(sortBy).append("} "); + sb.append("sortType={").append(sortType).append("} "); + sb.append("startIndex={").append(startIndex).append("} "); + sb.append("}"); + + return sb; + } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/5ac7c52a/agents-common/src/test/java/org/apache/ranger/plugin/model/TestRangerPolicyResourceSignature.java ---------------------------------------------------------------------- diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/model/TestRangerPolicyResourceSignature.java b/agents-common/src/test/java/org/apache/ranger/plugin/model/TestRangerPolicyResourceSignature.java new file mode 100644 index 0000000..46e924f --- /dev/null +++ b/agents-common/src/test/java/org/apache/ranger/plugin/model/TestRangerPolicyResourceSignature.java @@ -0,0 +1,234 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.model; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.ranger.plugin.model.RangerPolicy; +import org.apache.ranger.plugin.model.RangerPolicyResourceSignature; +import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource; +import org.apache.ranger.plugin.model.RangerPolicyResourceSignature.RangerPolicyResourceView; +import org.apache.ranger.plugin.model.validation.ValidationTestUtils; +import org.junit.Test; + +public class TestRangerPolicyResourceSignature { + + @Test + public void test_RangerPolicyResourceView_toString() { + // null resource + RangerPolicyResource policyResource = null; + RangerPolicyResourceView policyResourceView = new RangerPolicyResourceView(policyResource); + assertEquals("{}", policyResourceView.toString()); + + // non-null policy resource with null values/recursive flag + policyResource = createPolicyResource(null, null, null); + policyResourceView = new RangerPolicyResourceView(policyResource); + assertEquals("{values=,excludes=false,recursive=false}", policyResourceView.toString()); + + // valid values in non-asending order + policyResource = createPolicyResource(new String[]{"b", "a", "d", "c"}, true, false); + policyResourceView = new RangerPolicyResourceView(policyResource); + assertEquals("{values=[a, b, c, d],excludes=false,recursive=true}", policyResourceView.toString()); + + // recursive flag is false and different variation of values to show lexicographic ordering + policyResource = createPolicyResource(new String[]{"9", "A", "e", "_"}, false, true); + policyResourceView = new RangerPolicyResourceView(policyResource); + assertEquals("{values=[9, A, _, e],excludes=true,recursive=false}", policyResourceView.toString()); + } + + RangerPolicyResource createPolicyResource(String[] values, Boolean recursive, Boolean excludes) { + + RangerPolicyResource resource = mock(RangerPolicyResource.class); + if (values == null) { + when(resource.getValues()).thenReturn(null); + } else { + when(resource.getValues()).thenReturn(Arrays.asList(values)); + } + when(resource.getIsRecursive()).thenReturn(recursive); + when(resource.getIsExcludes()).thenReturn(excludes); + + return resource; + } + + @Test + public void test_isPolicyValidForResourceSignatureComputation() { + // null policy is invalid + RangerPolicyResourceSignature utils = new RangerPolicyResourceSignature((String)null); + RangerPolicy rangerPolicy = null; + assertFalse("policy==null", utils.isPolicyValidForResourceSignatureComputation(rangerPolicy)); + + // null resource map is invalid + rangerPolicy = mock(RangerPolicy.class); + when(rangerPolicy.getResources()).thenReturn(null); + assertFalse("policy.getResources()==null", utils.isPolicyValidForResourceSignatureComputation(rangerPolicy)); + + // empty resources map is ok! + Map<String, RangerPolicyResource> policyResources = new HashMap<String, RangerPolicyResource>(); + when(rangerPolicy.getResources()).thenReturn(policyResources); + assertTrue("policy.getResources().isEmpty()", utils.isPolicyValidForResourceSignatureComputation(rangerPolicy)); + + // but having a resource map with null key is not ok! + RangerPolicyResource aPolicyResource = mock(RangerPolicyResource.class); + policyResources.put(null, aPolicyResource); + assertFalse("policy.getResources().contains(null)", utils.isPolicyValidForResourceSignatureComputation(rangerPolicy)); + } + + @Test + public void test_RangerPolicyResourceSignature() { + // String rep of a null policy is an empty string! and its hash is sha of empty string! + RangerPolicyResourceSignature signature = new RangerPolicyResourceSignature((String)null); + assertEquals("", signature.asString()); + assertEquals(DigestUtils.md5Hex(""), signature.getSignature()); + } + + /* + * Format of data expected by the utility function which uses this is: + * { "resource-name", "values" "isExcludes", "isRecursive" } + */ + Object[][] first = new Object[][] { + { "table", new String[] { "tbl3", "tbl1", "tbl2"}, true, false}, + { "db", new String[] { "db1", "db2"}, false, null}, + { "col", new String[] { "col2", "col1", "col3"}, null, true}, + }; + + Object[][] first_recursive_null_or_false = new Object[][] { + { "table", new String[] { "tbl3", "tbl1", "tbl2"}, true, null}, // recursive flag is false in first + { "db", new String[] { "db1", "db2"}, false, null}, + { "col", new String[] { "col2", "col1", "col3"}, null, true}, + }; + + Object[][] first_recursive_flag_different = new Object[][] { + { "table", new String[] { "tbl3", "tbl1", "tbl2"}, true, false}, + { "db", new String[] { "db1", "db2"}, false, null}, + { "col", new String[] { "col2", "col1", "col3"}, null, false}, // recursive flag is true in first + }; + + Object[][] first_excludes_null_or_false = new Object[][] { + { "table", new String[] { "tbl3", "tbl1", "tbl2"}, true, false}, + { "db", new String[] { "db1", "db2"}, false, null}, // excludes flag is null in first + { "col", new String[] { "col2", "col1", "col3"}, false, true}, + }; + + Object[][] first_excludes_flag_different = new Object[][] { + { "table", new String[] { "tbl3", "tbl1", "tbl2"}, true, false}, + { "db", new String[] { "db1", "db2"}, false, null}, + { "col", new String[] { "col2", "col1", "col3"}, true, true}, // excludes flag is false in first + }; + + Object[][] data_second = new Object[][] { + { "db", new String[] { "db2", "db1"}, false, null}, + { "table", new String[] { "tbl2", "tbl3", "tbl1"}, true, false}, + { "col", new String[] { "col1", "col3", "col2"}, null, true}, + }; + + @Test + public void test_getResourceSignature_happyPath() { + // null policy returns signature of empty resource + RangerPolicy policy = null; + RangerPolicyResourceSignature sig = new RangerPolicyResourceSignature(policy); + assertEquals(null, sig.getResourceString(policy)); + + policy = mock(RangerPolicy.class); + Map<String, RangerPolicyResource> policyResources = _utils.createPolicyResourceMap(first); + when(policy.getResources()).thenReturn(policyResources); + String expected = "{" + + "col={values=[col1, col2, col3],excludes=false,recursive=true}, " + + "db={values=[db1, db2],excludes=false,recursive=false}, " + + "table={values=[tbl1, tbl2, tbl3],excludes=true,recursive=false}" + + "}"; + assertEquals(expected, sig.getResourceString(policy)); + + // order of values should not matter + policyResources = _utils.createPolicyResourceMap(data_second); + when(policy.getResources()).thenReturn(policyResources); + assertEquals(expected, sig.getResourceString(policy)); + } + + + @Test + public void test_nullRecursiveFlagIsSameAsFlase() { + // create two policies with resources that differ only in the recursive flag such that flags are null in one and false in another + RangerPolicy policy1 = createPolicy(first); + RangerPolicy policy2 = createPolicy(first_recursive_null_or_false); + RangerPolicyResourceSignature signature = new RangerPolicyResourceSignature((String)null); + assertEquals("null is same as false", signature.getResourceString(policy1), signature.getResourceString(policy2)); + } + + @Test + public void test_onlyDifferByRecursiveFlag() { + // create two policies with resources that differ only in the recursive flag, i.e. null/false in one and true in another + RangerPolicy policy1 = createPolicy(first); + RangerPolicy policy2 = createPolicy(first_recursive_flag_different); + RangerPolicyResourceSignature signature = new RangerPolicyResourceSignature((String)null); + assertFalse("Resources differ only by recursive flag true vs false/null", signature.getResourceString(policy1).equals(signature.getResourceString(policy2))); + } + + @Test + public void test_nullExcludesFlagIsSameAsFlase() { + // create two policies with resources that differ only in the excludes flag such that flags are null in one and false in another + RangerPolicy policy1 = createPolicy(first); + RangerPolicy policy2 = createPolicy(first_excludes_null_or_false); + RangerPolicyResourceSignature signature = new RangerPolicyResourceSignature((String)null); + assertEquals("null is same as false", signature.getResourceString(policy1), signature.getResourceString(policy2)); + } + + @Test + public void test_onlyDifferByExcludesFlag() { + // create two policies with resources that differ only in the excludes flag, i.e. null/false in one and true in another + RangerPolicy policy1 = createPolicy(first); + RangerPolicy policy2 = createPolicy(first_excludes_flag_different); + RangerPolicyResourceSignature signature = new RangerPolicyResourceSignature((String)null); + assertFalse("Resources differ only by recursive flag true vs false/null", signature.getResourceString(policy1).equals(signature.getResourceString(policy2))); + } + + RangerPolicy createPolicy(Object[][] data) { + RangerPolicy policy = mock(RangerPolicy.class); + Map<String, RangerPolicyResource> resources = _utils.createPolicyResourceMap(data); + when(policy.getResources()).thenReturn(resources); + return policy; + } + + @Test + public void test_integration() { + // setup two policies with resources that are structurally different but semantically the same. + RangerPolicy aPolicy = mock(RangerPolicy.class); + Map<String, RangerPolicyResource> resources = _utils.createPolicyResourceMap(first); + when(aPolicy.getResources()).thenReturn(resources); + RangerPolicyResourceSignature signature = new RangerPolicyResourceSignature(aPolicy); + + RangerPolicy anotherPolicy = mock(RangerPolicy.class); + resources = _utils.createPolicyResourceMap(data_second); + when(anotherPolicy.getResources()).thenReturn(resources); + RangerPolicyResourceSignature anotherSignature = new RangerPolicyResourceSignature(anotherPolicy); + assertTrue(signature.equals(anotherSignature)); + assertTrue(anotherSignature.equals(signature)); + } + + ValidationTestUtils _utils = new ValidationTestUtils(); +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/5ac7c52a/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/TestRangerPolicyResourceSignature.java ---------------------------------------------------------------------- diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/TestRangerPolicyResourceSignature.java b/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/TestRangerPolicyResourceSignature.java deleted file mode 100644 index a8b03ce..0000000 --- a/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/TestRangerPolicyResourceSignature.java +++ /dev/null @@ -1,232 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.ranger.plugin.model.validation; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; - -import org.apache.commons.codec.digest.DigestUtils; -import org.apache.ranger.plugin.model.RangerPolicy; -import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource; -import org.apache.ranger.plugin.model.validation.RangerPolicyResourceSignature.RangerPolicyResourceView; -import org.junit.Test; - -public class TestRangerPolicyResourceSignature { - - @Test - public void test_RangerPolicyResourceView_toString() { - // null resource - RangerPolicyResource policyResource = null; - RangerPolicyResourceView policyResourceView = new RangerPolicyResourceView(policyResource); - assertEquals("{}", policyResourceView.toString()); - - // non-null policy resource with null values/recursive flag - policyResource = createPolicyResource(null, null, null); - policyResourceView = new RangerPolicyResourceView(policyResource); - assertEquals("{values=,excludes=false,recursive=false}", policyResourceView.toString()); - - // valid values in non-asending order - policyResource = createPolicyResource(new String[]{"b", "a", "d", "c"}, true, false); - policyResourceView = new RangerPolicyResourceView(policyResource); - assertEquals("{values=[a, b, c, d],excludes=false,recursive=true}", policyResourceView.toString()); - - // recursive flag is false and different variation of values to show lexicographic ordering - policyResource = createPolicyResource(new String[]{"9", "A", "e", "_"}, false, true); - policyResourceView = new RangerPolicyResourceView(policyResource); - assertEquals("{values=[9, A, _, e],excludes=true,recursive=false}", policyResourceView.toString()); - } - - RangerPolicyResource createPolicyResource(String[] values, Boolean recursive, Boolean excludes) { - - RangerPolicyResource resource = mock(RangerPolicyResource.class); - if (values == null) { - when(resource.getValues()).thenReturn(null); - } else { - when(resource.getValues()).thenReturn(Arrays.asList(values)); - } - when(resource.getIsRecursive()).thenReturn(recursive); - when(resource.getIsExcludes()).thenReturn(excludes); - - return resource; - } - - @Test - public void test_isPolicyValidForResourceSignatureComputation() { - // null policy is invalid - RangerPolicyResourceSignature utils = new RangerPolicyResourceSignature((String)null); - RangerPolicy rangerPolicy = null; - assertFalse("policy==null", utils.isPolicyValidForResourceSignatureComputation(rangerPolicy)); - - // null resource map is invalid - rangerPolicy = mock(RangerPolicy.class); - when(rangerPolicy.getResources()).thenReturn(null); - assertFalse("policy.getResources()==null", utils.isPolicyValidForResourceSignatureComputation(rangerPolicy)); - - // empty resources map is ok! - Map<String, RangerPolicyResource> policyResources = new HashMap<String, RangerPolicyResource>(); - when(rangerPolicy.getResources()).thenReturn(policyResources); - assertTrue("policy.getResources().isEmpty()", utils.isPolicyValidForResourceSignatureComputation(rangerPolicy)); - - // but having a resource map with null key is not ok! - RangerPolicyResource aPolicyResource = mock(RangerPolicyResource.class); - policyResources.put(null, aPolicyResource); - assertFalse("policy.getResources().contains(null)", utils.isPolicyValidForResourceSignatureComputation(rangerPolicy)); - } - - @Test - public void test_RangerPolicyResourceSignature() { - // String rep of a null policy is an empty string! and its hash is sha of empty string! - RangerPolicyResourceSignature signature = new RangerPolicyResourceSignature((String)null); - assertEquals("", signature.asString()); - assertEquals(DigestUtils.md5Hex(""), signature.asHashHex()); - } - - /* - * Format of data expected by the utility function which uses this is: - * { "resource-name", "values" "isExcludes", "isRecursive" } - */ - Object[][] first = new Object[][] { - { "table", new String[] { "tbl3", "tbl1", "tbl2"}, true, false}, - { "db", new String[] { "db1", "db2"}, false, null}, - { "col", new String[] { "col2", "col1", "col3"}, null, true}, - }; - - Object[][] first_recursive_null_or_false = new Object[][] { - { "table", new String[] { "tbl3", "tbl1", "tbl2"}, true, null}, // recursive flag is false in first - { "db", new String[] { "db1", "db2"}, false, null}, - { "col", new String[] { "col2", "col1", "col3"}, null, true}, - }; - - Object[][] first_recursive_flag_different = new Object[][] { - { "table", new String[] { "tbl3", "tbl1", "tbl2"}, true, false}, - { "db", new String[] { "db1", "db2"}, false, null}, - { "col", new String[] { "col2", "col1", "col3"}, null, false}, // recursive flag is true in first - }; - - Object[][] first_excludes_null_or_false = new Object[][] { - { "table", new String[] { "tbl3", "tbl1", "tbl2"}, true, false}, - { "db", new String[] { "db1", "db2"}, false, null}, // excludes flag is null in first - { "col", new String[] { "col2", "col1", "col3"}, false, true}, - }; - - Object[][] first_excludes_flag_different = new Object[][] { - { "table", new String[] { "tbl3", "tbl1", "tbl2"}, true, false}, - { "db", new String[] { "db1", "db2"}, false, null}, - { "col", new String[] { "col2", "col1", "col3"}, true, true}, // excludes flag is false in first - }; - - Object[][] data_second = new Object[][] { - { "db", new String[] { "db2", "db1"}, false, null}, - { "table", new String[] { "tbl2", "tbl3", "tbl1"}, true, false}, - { "col", new String[] { "col1", "col3", "col2"}, null, true}, - }; - - @Test - public void test_getResourceSignature_happyPath() { - // null policy returns signature of empty resource - RangerPolicy policy = null; - RangerPolicyResourceSignature sig = new RangerPolicyResourceSignature(policy); - assertEquals(null, sig.getResourceString(policy)); - - policy = mock(RangerPolicy.class); - Map<String, RangerPolicyResource> policyResources = _utils.createPolicyResourceMap(first); - when(policy.getResources()).thenReturn(policyResources); - String expected = "{" + - "col={values=[col1, col2, col3],excludes=false,recursive=true}, " + - "db={values=[db1, db2],excludes=false,recursive=false}, " + - "table={values=[tbl1, tbl2, tbl3],excludes=true,recursive=false}" + - "}"; - assertEquals(expected, sig.getResourceString(policy)); - - // order of values should not matter - policyResources = _utils.createPolicyResourceMap(data_second); - when(policy.getResources()).thenReturn(policyResources); - assertEquals(expected, sig.getResourceString(policy)); - } - - - @Test - public void test_nullRecursiveFlagIsSameAsFlase() { - // create two policies with resources that differ only in the recursive flag such that flags are null in one and false in another - RangerPolicy policy1 = createPolicy(first); - RangerPolicy policy2 = createPolicy(first_recursive_null_or_false); - RangerPolicyResourceSignature signature = new RangerPolicyResourceSignature((String)null); - assertEquals("null is same as false", signature.getResourceString(policy1), signature.getResourceString(policy2)); - } - - @Test - public void test_onlyDifferByRecursiveFlag() { - // create two policies with resources that differ only in the recursive flag, i.e. null/false in one and true in another - RangerPolicy policy1 = createPolicy(first); - RangerPolicy policy2 = createPolicy(first_recursive_flag_different); - RangerPolicyResourceSignature signature = new RangerPolicyResourceSignature((String)null); - assertFalse("Resources differ only by recursive flag true vs false/null", signature.getResourceString(policy1).equals(signature.getResourceString(policy2))); - } - - @Test - public void test_nullExcludesFlagIsSameAsFlase() { - // create two policies with resources that differ only in the excludes flag such that flags are null in one and false in another - RangerPolicy policy1 = createPolicy(first); - RangerPolicy policy2 = createPolicy(first_excludes_null_or_false); - RangerPolicyResourceSignature signature = new RangerPolicyResourceSignature((String)null); - assertEquals("null is same as false", signature.getResourceString(policy1), signature.getResourceString(policy2)); - } - - @Test - public void test_onlyDifferByExcludesFlag() { - // create two policies with resources that differ only in the excludes flag, i.e. null/false in one and true in another - RangerPolicy policy1 = createPolicy(first); - RangerPolicy policy2 = createPolicy(first_excludes_flag_different); - RangerPolicyResourceSignature signature = new RangerPolicyResourceSignature((String)null); - assertFalse("Resources differ only by recursive flag true vs false/null", signature.getResourceString(policy1).equals(signature.getResourceString(policy2))); - } - - RangerPolicy createPolicy(Object[][] data) { - RangerPolicy policy = mock(RangerPolicy.class); - Map<String, RangerPolicyResource> resources = _utils.createPolicyResourceMap(data); - when(policy.getResources()).thenReturn(resources); - return policy; - } - - @Test - public void test_integration() { - // setup two policies with resources that are structurally different but semantically the same. - RangerPolicy aPolicy = mock(RangerPolicy.class); - Map<String, RangerPolicyResource> resources = _utils.createPolicyResourceMap(first); - when(aPolicy.getResources()).thenReturn(resources); - RangerPolicyResourceSignature signature = new RangerPolicyResourceSignature(aPolicy); - - RangerPolicy anotherPolicy = mock(RangerPolicy.class); - resources = _utils.createPolicyResourceMap(data_second); - when(anotherPolicy.getResources()).thenReturn(resources); - RangerPolicyResourceSignature anotherSignature = new RangerPolicyResourceSignature(anotherPolicy); - assertTrue(signature.equals(anotherSignature)); - assertTrue(anotherSignature.equals(signature)); - } - - ValidationTestUtils _utils = new ValidationTestUtils(); -} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/5ac7c52a/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/TestRangerPolicyValidator.java ---------------------------------------------------------------------- diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/TestRangerPolicyValidator.java b/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/TestRangerPolicyValidator.java index f02b96e..1a4f366 100644 --- a/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/TestRangerPolicyValidator.java +++ b/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/TestRangerPolicyValidator.java @@ -38,6 +38,7 @@ import org.apache.ranger.plugin.model.RangerPolicy; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItem; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemAccess; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource; +import org.apache.ranger.plugin.model.RangerPolicyResourceSignature; import org.apache.ranger.plugin.model.RangerService; import org.apache.ranger.plugin.model.RangerServiceDef; import org.apache.ranger.plugin.model.RangerServiceDef.RangerResourceDef; @@ -195,6 +196,8 @@ public class TestRangerPolicyValidator { when(_store.getPolicies(updateFilter)).thenReturn(existingPolicies); // valid policy can have empty set of policy items if audit is turned on // null value for audit is treated as audit on. + // for now we want to turn any resource related checking off + when(_policy.getResources()).thenReturn(null); for (Action action : cu) { for (Boolean auditEnabled : new Boolean[] { null, true } ) { for (boolean isAdmin : new boolean[] { true, false }) { @@ -243,12 +246,13 @@ public class TestRangerPolicyValidator { Map<String, RangerPolicyResource> resourceMap = _utils.createPolicyResourceMap(policyResourceMap_good); when(_policy.getResources()).thenReturn(resourceMap); // let's add some other policies in the store for this service that have a different signature - SearchFilter resourceDuplicationFilter = new SearchFilter(); - resourceDuplicationFilter.setParam(SearchFilter.SERVICE_NAME, "service-name"); - when(_factory.createPolicyResourceSignature(_policy)).thenReturn(new RangerPolicyResourceSignature("policy")); - when(_factory.createPolicyResourceSignature(existingPolicy)).thenReturn(new RangerPolicyResourceSignature("policy-name-2")); + // setup the signatures on the policies + RangerPolicyResourceSignature policySignature = mock(RangerPolicyResourceSignature.class); + when(_factory.createPolicyResourceSignature(_policy)).thenReturn(policySignature); + // setup the store to indicate that no other policy exists with matching signature + when(policySignature.getSignature()).thenReturn("hash-1"); + when(_store.getPoliciesByResourceSignature("hash-1")).thenReturn(null); // we are reusing the same policies collection here -- which is fine - when(_store.getPolicies(resourceDuplicationFilter)).thenReturn(existingPolicies); for (Action action : cu) { if (action == Action.CREATE) { when(_policy.getId()).thenReturn(7L); @@ -299,6 +303,7 @@ public class TestRangerPolicyValidator { _policy = mock(RangerPolicy.class); for (String name : new String[] { null, " " }) { when(_policy.getName()).thenReturn(name); + when(_policy.getResources()).thenReturn(null); checkFailure_isValid(action, "missing", "name"); } @@ -437,6 +442,11 @@ public class TestRangerPolicyValidator { // one mandatory is missing (tbl) and one unknown resource is specified (extra), and values of option resource don't conform to validation pattern (col) Map<String, RangerPolicyResource> policyResources = _utils.createPolicyResourceMap(policyResourceMap_bad); when(_policy.getResources()).thenReturn(policyResources); + // ensure thta policy is kosher when it comes to resource signature + RangerPolicyResourceSignature signature = mock(RangerPolicyResourceSignature.class); + when(_factory.createPolicyResourceSignature(_policy)).thenReturn(signature); + when(signature.getSignature()).thenReturn("hash-1"); + when(_store.getPoliciesByResourceSignature("hash-1")).thenReturn(null); // store does not have any policies for that signature hash for (Action action : cu) { for (boolean isAdmin : new boolean[] { true, false }) { _failures.clear(); assertFalse(_validator.isValid(_policy, action, isAdmin, _failures)); @@ -448,13 +458,8 @@ public class TestRangerPolicyValidator { } } - // create the right resource def but let it clash with another policy with matching resource-def - policyResources = _utils.createPolicyResourceMap(policyResourceMap_good); - when(_policy.getResources()).thenReturn(policyResources); - filter = new SearchFilter(); filter.setParam(SearchFilter.SERVICE_NAME, "service-name"); - when(_store.getPolicies(filter)).thenReturn(existingPolicies); - // we are doctoring the factory to always return the same signature - when(_factory.createPolicyResourceSignature(anyPolicy())).thenReturn(new RangerPolicyResourceSignature("blah")); + // Check if error around resource signature clash are reported. have Store return policies for same signature + when(_store.getPoliciesByResourceSignature("hash-1")).thenReturn(existingPolicies); for (Action action : cu) { for (boolean isAdmin : new boolean[] { true, false }) { _failures.clear(); assertFalse(_validator.isValid(_policy, action, isAdmin, _failures)); @@ -665,38 +670,23 @@ public class TestRangerPolicyValidator { @Test public final void test_isPolicyResourceUnique() throws Exception { - RangerPolicy[] policies = new RangerPolicy[3]; - RangerPolicyResourceSignature[] signatures = new RangerPolicyResourceSignature[3]; - for (int i = 0; i < 3; i++) { - RangerPolicy policy = mock(RangerPolicy.class); - when(policy.getId()).thenReturn((long)i); - policies[i] = policy; - signatures[i] = new RangerPolicyResourceSignature("policy" + i); - when(_factory.createPolicyResourceSignature(policies[i])).thenReturn(signatures[i]); - } - - SearchFilter searchFilter = new SearchFilter(); - String serviceName = "aService"; - searchFilter.setParam(SearchFilter.SERVICE_NAME, serviceName); - - List<RangerPolicy> existingPolicies = Arrays.asList(new RangerPolicy[] { policies[1], policies[2]} ); - // all existing policies have distinct signatures - for (Action action : cu) { - when(_store.getPolicies(searchFilter)).thenReturn(existingPolicies); - assertTrue("No duplication: " + action, _validator.isPolicyResourceUnique(policies[0], _failures, action, serviceName)); - } - - // Failure if signature matches an existing policy - // We change the signature of 3rd policy to be same as that of 1st so duplication check will fail - for (Action action : cu) { - when(_factory.createPolicyResourceSignature(policies[2])).thenReturn(new RangerPolicyResourceSignature("policy0")); - when(_store.getPolicies(searchFilter)).thenReturn(existingPolicies); - assertFalse("Duplication:" + action, _validator.isPolicyResourceUnique(policies[0], _failures, action, serviceName)); - } + // if store does not contain any matching policies then check should succeed + RangerPolicyResourceSignature signature = mock(RangerPolicyResourceSignature.class); + String hash = "hash-1"; + when(signature.getSignature()).thenReturn(hash); + when(_factory.createPolicyResourceSignature(_policy)).thenReturn(signature); + List<RangerPolicy> policies = null; + when(_store.getPoliciesByResourceSignature(hash)).thenReturn(policies); + assertTrue(_validator.isPolicyResourceUnique(_policy, _failures)); + policies = new ArrayList<RangerPolicy>(); + assertTrue(_validator.isPolicyResourceUnique(_policy, _failures)); - // update should exclude itself! - let's change id of 3rd policy to be the same as the 1st one. - when(policies[2].getId()).thenReturn((long)0); - assertTrue("No duplication if updating policy", _validator.isPolicyResourceUnique(policies[0], _failures, Action.UPDATE, serviceName)); + // if store does have any policy then test should fail with appropriate error message. + RangerPolicy policy1 = mock(RangerPolicy.class); policies.add(policy1); + RangerPolicy policy2 = mock(RangerPolicy.class); policies.add(policy2); + when(_store.getPoliciesByResourceSignature(hash)).thenReturn(policies); + assertFalse(_validator.isPolicyResourceUnique(_policy, _failures)); + _utils.checkFailureForSemanticError(_failures, "resources"); } @Test http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/5ac7c52a/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/TestRangerValidator.java ---------------------------------------------------------------------- diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/TestRangerValidator.java b/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/TestRangerValidator.java index 81ecfdd..46f488e 100644 --- a/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/TestRangerValidator.java +++ b/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/TestRangerValidator.java @@ -46,7 +46,6 @@ import org.apache.ranger.plugin.model.RangerServiceDef.RangerAccessTypeDef; import org.apache.ranger.plugin.model.RangerServiceDef.RangerEnumDef; import org.apache.ranger.plugin.model.RangerServiceDef.RangerResourceDef; import org.apache.ranger.plugin.model.RangerServiceDef.RangerServiceConfigDef; -import org.apache.ranger.plugin.model.validation.RangerValidator; import org.apache.ranger.plugin.model.validation.RangerValidator.Action; import org.apache.ranger.plugin.store.ServiceStore; import org.apache.ranger.plugin.util.SearchFilter; @@ -193,6 +192,27 @@ public class TestRangerValidator { assertNull(_validator.getPolicy(2L)); assertTrue(_validator.getPolicy(3L) != null); } + + @Test + public final void test_getPoliciesForResourceSignature() throws Exception { + // return null if store returns null or throws an exception + String hexSignature = "aSignature"; + when(_store.getPoliciesByResourceSignature(hexSignature)).thenReturn(null); + assertNull(_validator.getPoliciesForResourceSignature(hexSignature)); + when(_store.getPoliciesByResourceSignature(hexSignature)).thenThrow(new Exception()); + assertNull(_validator.getPoliciesForResourceSignature(hexSignature)); + + // what ever store returns should come back + hexSignature = "anotherSignature"; + List<RangerPolicy> policies = new ArrayList<RangerPolicy>(); + RangerPolicy policy1 = mock(RangerPolicy.class); + policies.add(policy1); + RangerPolicy policy2 = mock(RangerPolicy.class); + policies.add(policy2); + when(_store.getPoliciesByResourceSignature(hexSignature)).thenReturn(policies); + List<RangerPolicy> result = _validator.getPoliciesForResourceSignature(hexSignature); + assertTrue(result.contains(policy1) && result.contains(policy2)); + } @Test public void test_getService_byId() throws Exception { http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/5ac7c52a/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/ValidationTestUtils.java ---------------------------------------------------------------------- diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/ValidationTestUtils.java b/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/ValidationTestUtils.java index 432c9d4..bc4b2c5 100644 --- a/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/ValidationTestUtils.java +++ b/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/ValidationTestUtils.java @@ -332,7 +332,7 @@ public class ValidationTestUtils { return defs; } - Map<String, RangerPolicyResource> createPolicyResourceMap(Object[][] input) { + public Map<String, RangerPolicyResource> createPolicyResourceMap(Object[][] input) { if (input == null) { return null; } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/5ac7c52a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java index 12aa31c..5542f72 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java @@ -87,9 +87,10 @@ import org.apache.ranger.plugin.model.RangerPolicy; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItem; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemAccess; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemCondition; +import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource; +import org.apache.ranger.plugin.model.RangerPolicyResourceSignature; import org.apache.ranger.plugin.model.RangerService; import org.apache.ranger.plugin.model.RangerServiceDef; -import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource; import org.apache.ranger.plugin.model.RangerServiceDef.RangerAccessTypeDef; import org.apache.ranger.plugin.model.RangerServiceDef.RangerContextEnricherDef; import org.apache.ranger.plugin.model.RangerServiceDef.RangerEnumDef; @@ -97,13 +98,15 @@ import org.apache.ranger.plugin.model.RangerServiceDef.RangerEnumElementDef; import org.apache.ranger.plugin.model.RangerServiceDef.RangerPolicyConditionDef; import org.apache.ranger.plugin.model.RangerServiceDef.RangerResourceDef; import org.apache.ranger.plugin.model.RangerServiceDef.RangerServiceConfigDef; -import org.apache.ranger.plugin.store.EmbeddedServiceDefsUtil; import org.apache.ranger.plugin.store.AbstractServiceStore; +import org.apache.ranger.plugin.store.EmbeddedServiceDefsUtil; +import org.apache.ranger.plugin.util.SearchFilter; import org.apache.ranger.plugin.util.ServicePolicies; -import org.apache.ranger.service.RangerPolicyWithAssignedIdService; import org.apache.ranger.service.RangerAuditFields; import org.apache.ranger.service.RangerDataHistService; +import org.apache.ranger.service.RangerFactory; import org.apache.ranger.service.RangerPolicyService; +import org.apache.ranger.service.RangerPolicyWithAssignedIdService; import org.apache.ranger.service.RangerServiceDefService; import org.apache.ranger.service.RangerServiceService; import org.apache.ranger.service.RangerServiceWithAssignedIdService; @@ -120,7 +123,6 @@ import org.springframework.transaction.PlatformTransactionManager; import org.springframework.transaction.TransactionStatus; import org.springframework.transaction.support.TransactionCallback; import org.springframework.transaction.support.TransactionTemplate; -import org.apache.ranger.plugin.util.SearchFilter; @Component @@ -170,6 +172,9 @@ public class ServiceDBStore extends AbstractServiceStore { @Autowired RangerServiceWithAssignedIdService svcServiceWithAssignedId; + @Autowired + RangerFactory factory; + private static volatile boolean legacyServiceDefsInitDone = false; private Boolean populateExistingBaseFields = false; @@ -1194,6 +1199,18 @@ public class ServiceDBStore extends AbstractServiceStore { } @Override + public List<RangerPolicy> getPoliciesByResourceSignature(String hexSignature) throws Exception { + List<XXPolicy> xxPolicies = daoMgr.getXXPolicy().findByResourceSignature(hexSignature); + List<RangerPolicy> policies = new ArrayList<RangerPolicy>(xxPolicies.size()); + for (XXPolicy xxPolicy : xxPolicies) { + RangerPolicy policy = policyService.getPopulatedViewObject(xxPolicy); + policies.add(policy); + } + + return policies; + } + + @Override public RangerService getService(Long id) throws Exception { if(LOG.isDebugEnabled()) { LOG.debug("==> ServiceDBStore.getService()"); @@ -1270,6 +1287,9 @@ public class ServiceDBStore extends AbstractServiceStore { List<RangerPolicyItem> policyItems = policy.getPolicyItems(); policy.setVersion(new Long(1)); + RangerPolicyResourceSignature signature = factory.createPolicyResourceSignature(policy); + String hexSignature = signature.getSignature(); + policy.setResourceSignature(hexSignature); if(populateExistingBaseFields) { assignedIdPolicyService.setPopulateExistingBaseFields(true); @@ -1433,15 +1453,21 @@ public class ServiceDBStore extends AbstractServiceStore { public RangerPolicyList getPaginatedPolicies(SearchFilter filter) throws Exception { if (LOG.isDebugEnabled()) { - LOG.debug("==> ServiceDBStore.getPaginatedPolicies()"); + LOG.debug("==> ServiceDBStore.getPaginatedPolicies(+ " + filter + ")"); } RangerPolicyList policyList = policyService.searchRangerPolicies(filter); - + + if (LOG.isDebugEnabled()) { + LOG.debug("before filter: count=" + policyList.getListSize()); + } applyFilter(policyList.getPolicies(), filter); + if (LOG.isDebugEnabled()) { + LOG.debug("after filter: count=" + policyList.getListSize()); + } if (LOG.isDebugEnabled()) { - LOG.debug("<== ServiceDBStore.getPaginatedPolicies()"); + LOG.debug("<== ServiceDBStore.getPaginatedPolicies(" + filter + "): count=" + policyList.getListSize()); } return policyList; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/5ac7c52a/security-admin/src/main/java/org/apache/ranger/service/RangerFactory.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/service/RangerFactory.java b/security-admin/src/main/java/org/apache/ranger/service/RangerFactory.java new file mode 100644 index 0000000..7834262 --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/service/RangerFactory.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.service; + +import org.apache.ranger.plugin.model.RangerPolicy; +import org.apache.ranger.plugin.model.RangerPolicyResourceSignature; +import org.springframework.context.annotation.Scope; +import org.springframework.stereotype.Service; + +@Service +@Scope("singleton") +public class RangerFactory { + public RangerPolicyResourceSignature createPolicyResourceSignature(RangerPolicy policy) { + return new RangerPolicyResourceSignature(policy); + } +}
