Addressed review comments for Review Request 35827, Review Request 35992, tested for merck demo. Tested tag-service lookupResource.
Signed-off-by: Madhan Neethiraj <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/c930d563 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/c930d563 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/c930d563 Branch: refs/heads/tag-policy Commit: c930d563372d898d7ca4c79a252b4b150486b4d3 Parents: e63a48e Author: Abhay Kulkarni <[email protected]> Authored: Sat Jun 27 23:02:34 2015 -0700 Committer: Madhan Neethiraj <[email protected]> Committed: Fri Jul 3 19:49:46 2015 -0700 ---------------------------------------------------------------------- .../ranger/admin/client/RangerAdminClient.java | 19 ++ .../admin/client/RangerAdminRESTClient.java | 206 +++++++++++++------ .../common/RangerConfigPropertyRepository.java | 50 +++++ .../RangerAdminTagRetriever.java | 57 ++--- .../RangerTagFileStoreRetriever.java | 24 ++- .../contextenricher/RangerTagProvider.java | 4 +- .../contextenricher/RangerTagRetriever.java | 2 +- .../RangerDefaultPolicyResourceMatcher.java | 22 +- .../RangerPolicyResourceMatcher.java | 2 +- .../ranger/plugin/service/RangerBasePlugin.java | 14 +- .../ranger/plugin/store/AbstractTagStore.java | 7 +- .../ranger/plugin/store/TagPredicateUtil.java | 38 +++- .../apache/ranger/plugin/store/TagStore.java | 12 +- .../ranger/plugin/store/file/TagFileStore.java | 133 +++++++----- .../RangerPluginConfigPropertyRepository.java | 48 +++++ .../ranger/plugin/util/RangerRESTUtils.java | 9 +- .../apache/ranger/plugin/util/SearchFilter.java | 13 ++ .../ranger/plugin/util/TagServiceResources.java | 93 +++++++++ .../ranger/services/tag/RangerServiceTag.java | 121 +++++------ .../service-defs/ranger-servicedef-tag.json | 7 +- .../plugin/policyengine/TestPolicyEngine.java | 47 ++++- .../ranger/plugin/store/TestTagStore.java | 21 +- .../test_policyengine_tag_hdfs.json | 2 +- .../test_policyengine_tag_hive.json | 2 +- .../client/RangerAdminJersey2RESTClient.java | 40 +++- .../org/apache/ranger/biz/ServiceDBStore.java | 3 + .../java/org/apache/ranger/biz/ServiceMgr.java | 11 +- .../RangerAdminConfigPropertyRepository.java | 45 ++++ .../java/org/apache/ranger/rest/TagREST.java | 162 +++++++++++---- .../apache/ranger/rest/TagRESTConstants.java | 23 ++- .../resources/conf.dist/ranger-admin-site.xml | 1 + .../conf.dist/security-applicationContext.xml | 6 + 32 files changed, 923 insertions(+), 321 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c930d563/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminClient.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminClient.java b/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminClient.java index 618a44e..b239b8e 100644 --- a/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminClient.java +++ b/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminClient.java @@ -20,8 +20,13 @@ package org.apache.ranger.admin.client; +import org.apache.ranger.plugin.model.RangerTaggedResource; +import org.apache.ranger.plugin.model.RangerTaggedResourceKey; import org.apache.ranger.plugin.util.GrantRevokeRequest; import org.apache.ranger.plugin.util.ServicePolicies; +import org.apache.ranger.plugin.util.TagServiceResources; + +import java.util.List; public interface RangerAdminClient { @@ -32,4 +37,18 @@ public interface RangerAdminClient { void grantAccess(GrantRevokeRequest request) throws Exception; void revokeAccess(GrantRevokeRequest request) throws Exception; + + TagServiceResources getTaggedResources(String tagServiceName, String componentType, Long lastTimestamp) throws Exception; + + List<String> getTagNames(String tagServiceName, String componentType, String tagNamePattern) throws Exception; + + TagServiceResources getAllTaggedResources() throws Exception; + + List<RangerTaggedResource> setResources(List<RangerTaggedResourceKey> keys, List<RangerTaggedResource.RangerResourceTag> tags) throws Exception; + + RangerTaggedResource setResource(RangerTaggedResourceKey key, List<RangerTaggedResource.RangerResourceTag> tags) throws Exception; + + RangerTaggedResource updateResourceTags(RangerTaggedResourceKey key, List<RangerTaggedResource.RangerResourceTag> tagsToAdd, + List<RangerTaggedResource.RangerResourceTag> tagsToDelete) throws Exception; + } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c930d563/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java b/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java index 62f2c1c..4f07489 100644 --- a/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java +++ b/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java @@ -29,20 +29,15 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.security.AccessControlException; import org.apache.ranger.admin.client.datatype.RESTResponse; -import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; +import org.apache.ranger.common.RangerConfigPropertyRepository; import org.apache.ranger.plugin.model.RangerTaggedResource; -import org.apache.ranger.plugin.util.GrantRevokeRequest; -import org.apache.ranger.plugin.util.RangerRESTClient; -import org.apache.ranger.plugin.util.RangerRESTUtils; -import org.apache.ranger.plugin.util.ServicePolicies; +import org.apache.ranger.plugin.model.RangerTaggedResourceKey; +import org.apache.ranger.plugin.util.*; import java.lang.reflect.ParameterizedType; import java.lang.reflect.Type; import java.util.List; -import java.util.Map; -import java.util.Set; - public class RangerAdminRESTClient implements RangerAdminClient { private static final Log LOG = LogFactory.getLog(RangerAdminRESTClient.class); @@ -56,13 +51,44 @@ public class RangerAdminRESTClient implements RangerAdminClient { public RangerAdminRESTClient() { } + public static <T> GenericType<List<T>> getGenericType(final T clazz) { + + ParameterizedType parameterizedGenericType = new ParameterizedType() { + public Type[] getActualTypeArguments() { + return new Type[] { clazz.getClass() }; + } + + public Type getRawType() { + return List.class; + } + + public Type getOwnerType() { + return List.class; + } + }; + + return new GenericType<List<T>>(parameterizedGenericType) {}; + } + @Override public void init(String serviceName, String appId, String propertyPrefix) { this.serviceName = serviceName; this.pluginId = restUtils.getPluginId(serviceName, appId); - String url = RangerConfiguration.getInstance().get(propertyPrefix + ".policy.rest.url"); - String sslConfigFileName = RangerConfiguration.getInstance().get(propertyPrefix + ".policy.rest.ssl.config.file"); + String url = RangerConfigPropertyRepository.getProperty(propertyPrefix + ".policy.rest.url"); + String sslConfigFileName = RangerConfigPropertyRepository.getProperty(propertyPrefix + ".policy.rest.ssl.config.file"); + + if (url == null) { + // Use externalurl + if(LOG.isDebugEnabled()) { + LOG.info("RangerAdminRESTClient.init() : null url found for property " + propertyPrefix + ".policy.rest.url, using value of ranger.externalurl property instead."); + url = RangerConfigPropertyRepository.getProperty("ranger.externalurl"); + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug("RangerAdminRESTClient.init() : url=" + url + ", sslConfigFileName=" + sslConfigFileName + ")"); + } init(url, sslConfigFileName); } @@ -170,78 +196,49 @@ public class RangerAdminRESTClient implements RangerAdminClient { return ret; } - public void init(String serviceName, Map<String, String> configs) { - this.serviceName = serviceName; - // Get all configuration parameter to connect to DGI from configs - String url = configs.get("URL"); - String sslConfigFileName = configs.get("SSL_CONFIG_FILE_NAME"); - String userName = configs.get("username"); - String password = configs.get("password"); - - init(url, sslConfigFileName); - if (restClient != null) { - restClient.setBasicAuthInfo(userName, password); - } - } - - - public List<RangerTaggedResource> getTaggedResources(String componentType) throws Exception { + @Override + public TagServiceResources getTaggedResources(String tagServiceName, String componentType, Long lastTimestamp) throws Exception { if(LOG.isDebugEnabled()) { - LOG.debug("==> RangerAdminRESTClient.getTaggedResources(" + serviceName + ", " + componentType + "): "); + LOG.debug("==> RangerAdminRESTClient.getTaggedResources(" + tagServiceName + ", " + componentType + ", " + lastTimestamp + "): "); } - ParameterizedType parameterizedGenericType = new ParameterizedType() { - public Type[] getActualTypeArguments() { - return new Type[] { new RangerTaggedResource().getClass() }; - } - - public Type getRawType() { - return List.class; - } - - public Type getOwnerType() { - return List.class; - } - }; - - GenericType<List<RangerTaggedResource>> genericType = new GenericType<List<RangerTaggedResource>>( - parameterizedGenericType) { - }; - - List<RangerTaggedResource> ret; + TagServiceResources ret; - WebResource webResource = createWebResource(RangerRESTUtils.REST_URL_GET_TAGGED_RESOURCES) - .queryParam(RangerRESTUtils.TAG_SERVICE_NAME_PARAM, serviceName) - .queryParam(RangerRESTUtils.COMPONENT_TYPE_PARAM, componentType); + WebResource webResource = createWebResource(RangerRESTUtils.REST_URL_GET_UPDATED_TAGGED_RESOURCES) + .queryParam(RangerRESTUtils.TAG_SERVICE_NAME_PARAM, tagServiceName) + .queryParam(RangerRESTUtils.COMPONENT_TYPE_PARAM, componentType) + .queryParam(RangerRESTUtils.TAG_TIMESTAMP_PARAM, Long.toString(lastTimestamp.longValue())); ClientResponse response = webResource.accept(RangerRESTUtils.REST_MIME_TYPE_JSON).get(ClientResponse.class); if(response != null && response.getStatus() == 200) { - ret = response.getEntity(genericType); + ret = response.getEntity(TagServiceResources.class); } else { RESTResponse resp = RESTResponse.fromClientResponse(response); LOG.error("Error getting taggedResources. request=" + webResource.toString() - + ", response=" + resp.toString() + ", serviceName=" + serviceName + ", componentType=" + componentType); + + ", response=" + resp.toString() + ", serviceName=" + tagServiceName + ", componentType=" + componentType + + ", " + "lastTimestamp=" + lastTimestamp); throw new Exception(resp.getMessage()); } if(LOG.isDebugEnabled()) { - LOG.debug("<== RangerAdminRESTClient.getTaggedResources(" + serviceName + ", " + componentType + "): " + ret); + LOG.debug("<==> RangerAdminRESTClient.getTaggedResources(" + tagServiceName + ", " + componentType + ", " + lastTimestamp + "): "); } return ret; } - public Set<String> getTagNames(String componentType, String tagNamePattern) throws Exception { - // TODO + @Override + public List<String> getTagNames(String tagServiceName, String componentType, String tagNamePattern) throws Exception { if(LOG.isDebugEnabled()) { - LOG.debug("==> RangerAdminRESTClient.getTagNames(" + serviceName + ", " + componentType + LOG.debug("==> RangerAdminRESTClient.getTagNames(" + tagServiceName + ", " + componentType + ", " + tagNamePattern + "): "); } - Set<String> ret = null; + List<String> ret = null; + String emptyString = ""; WebResource webResource = createWebResource(RangerRESTUtils.REST_URL_LOOKUP_TAG_NAMES) - .queryParam(RangerRESTUtils.TAG_SERVICE_NAME_PARAM, serviceName) + .queryParam(RangerRESTUtils.TAG_SERVICE_NAME_PARAM, tagServiceName) .queryParam(RangerRESTUtils.TAG_PATTERN_PARAM, tagNamePattern); if (StringUtils.isNotBlank(componentType)) { @@ -251,11 +248,12 @@ public class RangerAdminRESTClient implements RangerAdminClient { ClientResponse response = webResource.accept(RangerRESTUtils.REST_MIME_TYPE_JSON).get(ClientResponse.class); if(response != null && response.getStatus() == 200) { - ret = (Set<String>)response.getEntity(Set.class); + ret = response.getEntity(getGenericType(emptyString)); } else { RESTResponse resp = RESTResponse.fromClientResponse(response); LOG.error("Error getting taggedResources. request=" + webResource.toString() - + ", response=" + resp.toString() + ", serviceName=" + serviceName + ", componentType=" + componentType); + + ", response=" + resp.toString() + ", serviceName=" + tagServiceName + ", componentType=" + componentType + + ", " + "tagNamePattern=" + tagNamePattern); throw new Exception(resp.getMessage()); } @@ -266,4 +264,94 @@ public class RangerAdminRESTClient implements RangerAdminClient { return ret; } + @Override + public TagServiceResources getAllTaggedResources() throws Exception { + String emptyString = ""; + return getTaggedResources(emptyString, emptyString, 0L); + } + + @Override + public List<RangerTaggedResource> setResources(List<RangerTaggedResourceKey> keys, List<RangerTaggedResource.RangerResourceTag> tags) throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerAdminRESTClient.setResources()"); + } + + List<RangerTaggedResource> ret = null; + + WebResource webResource = createWebResource(RangerRESTUtils.REST_URL_SET_TAGGED_RESOURCES); + webResource.entity(keys).entity(tags); + + ClientResponse response = webResource.accept(RangerRESTUtils.REST_MIME_TYPE_JSON).put(ClientResponse.class); + + if(response != null && response.getStatus() == 200) { + ret = response.getEntity(getGenericType(new RangerTaggedResource())); + } else { + RESTResponse resp = RESTResponse.fromClientResponse(response); + LOG.error("Error setting taggedResources. request=" + webResource.toString() + + ", response=" + resp.toString() + ", key=" + keys + ", tags=" + tags); + throw new Exception(resp.getMessage()); + } + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerAdminRESTClient.setResource()"); + } + + return ret; + } + + @Override + public RangerTaggedResource setResource(RangerTaggedResourceKey key, List<RangerTaggedResource.RangerResourceTag> tags) throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerAdminRESTClient.setResource()"); + } + + RangerTaggedResource ret = null; + + WebResource webResource = createWebResource(RangerRESTUtils.REST_URL_SET_TAGGED_RESOURCE); + webResource.entity(key).entity(tags); + + ClientResponse response = webResource.accept(RangerRESTUtils.REST_MIME_TYPE_JSON).put(ClientResponse.class); + + if(response != null && response.getStatus() == 200) { + ret = response.getEntity(RangerTaggedResource.class); + } else { + RESTResponse resp = RESTResponse.fromClientResponse(response); + LOG.error("Error setting taggedResource. request=" + webResource.toString() + + ", response=" + resp.toString() + ", key=" + key + ", tags=" + tags); + throw new Exception(resp.getMessage()); + } + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerAdminRESTClient.setResource()"); + } + + return ret; + } + + @Override + public RangerTaggedResource updateResourceTags(RangerTaggedResourceKey key, List<RangerTaggedResource.RangerResourceTag> tagsToAdd, + List<RangerTaggedResource.RangerResourceTag> tagsToDelete) throws Exception { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerAdminRESTClient.updateResourceTags()"); + } + + RangerTaggedResource ret = null; + + WebResource webResource = createWebResource(RangerRESTUtils.REST_URL_UPDATE_TAGGED_RESOURCE); + webResource.entity(key).entity(tagsToAdd).entity(tagsToDelete); + + ClientResponse response = webResource.accept(RangerRESTUtils.REST_MIME_TYPE_JSON).put(ClientResponse.class); + + if(response != null && response.getStatus() == 200) { + ret = response.getEntity(RangerTaggedResource.class); + } else { + RESTResponse resp = RESTResponse.fromClientResponse(response); + LOG.error("Error updating taggedResource. request=" + webResource.toString() + + ", response=" + resp.toString() + ", key=" + key + ", tagsToAdd=" + tagsToAdd + ", tagsToDelete=" + tagsToDelete); + throw new Exception(resp.getMessage()); + } + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerAdminRESTClient.updateResourceTags()"); + } + + return ret; + } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c930d563/agents-common/src/main/java/org/apache/ranger/common/RangerConfigPropertyRepository.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/common/RangerConfigPropertyRepository.java b/agents-common/src/main/java/org/apache/ranger/common/RangerConfigPropertyRepository.java new file mode 100644 index 0000000..21c6bb2 --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/common/RangerConfigPropertyRepository.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.common; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +/** + * Interface to get configuration values for Ranger (both security-admin and agents-common) + */ + +public abstract class RangerConfigPropertyRepository { + private static final Log LOG = LogFactory.getLog(RangerConfigPropertyRepository.class); + + + protected static volatile RangerConfigPropertyRepository instance = null; + + public static String getProperty(String name) { + + String ret = null; + + if (instance != null) { + ret = instance.getPropertyValue(name); + } else { + LOG.error("RangerConfigPropertyRepository.getPropery() - Object not created correctly."); + } + + return ret; + } + + abstract protected String getPropertyValue(String name); + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c930d563/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAdminTagRetriever.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAdminTagRetriever.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAdminTagRetriever.java index c6fc852..3f10650 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAdminTagRetriever.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAdminTagRetriever.java @@ -19,49 +19,55 @@ package org.apache.ranger.plugin.contextenricher; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.ranger.admin.client.RangerAdminRESTClient; -import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; +import org.apache.ranger.admin.client.RangerAdminClient; import org.apache.ranger.plugin.model.RangerTaggedResource; +import org.apache.ranger.plugin.service.RangerBasePlugin; +import org.apache.ranger.plugin.util.TagServiceResources; +import org.apache.ranger.services.tag.RangerServiceTag; -import java.util.HashMap; +import java.util.Date; import java.util.List; import java.util.Map; public class RangerAdminTagRetriever extends RangerTagRefresher { private static final Log LOG = LogFactory.getLog(RangerAdminTagRetriever.class); + private static String propertyPrefixPreamble = "ranger.plugin."; + private static String appId = "tag-retriever"; private final String componentType; private final String tagServiceName; + private final String propertyPrefix; + private RangerTagReceiver receiver; - private RangerAdminRESTClient rangerAdminRESTClient; + private RangerAdminClient adminClient; + private Long lastTimestamp; public RangerAdminTagRetriever(final String componentType, final String tagServiceName, final long pollingIntervalMs, final RangerTagReceiver enricher) { super(pollingIntervalMs); this.componentType = componentType; this.tagServiceName = tagServiceName; setReceiver(enricher); + propertyPrefix = propertyPrefixPreamble + componentType; + this.lastTimestamp = 0L; } @Override - public void init(Map<String, Object> options) { - - String propertyPrefix = "ranger.plugin.tag"; - String url = RangerConfiguration.getInstance().get(propertyPrefix + ".provider.rest.url", "http://node-1.example.com:6080";); - String sslConfigFileName = RangerConfiguration.getInstance().get(propertyPrefix + ".provider.rest.ssl.config.file", "abcd"); - String userName = RangerConfiguration.getInstance().get(propertyPrefix + ".provider.login.username", "admin"); - String password = RangerConfiguration.getInstance().get(propertyPrefix + ".provider.login.password", "admin"); - - Map<String, String> configs = new HashMap<String, String>(); + public void init(Map<String, String> options) { - configs.put("URL", url); - configs.put("SSL_CONFIG_FILE_NAME", sslConfigFileName); - configs.put("username", userName); - configs.put("password", password); + if (MapUtils.isNotEmpty(options)) { + String useTestTagProvider = options.get("useTestTagProvider"); - rangerAdminRESTClient = new RangerAdminRESTClient(); - rangerAdminRESTClient.init(tagServiceName, configs); + if (useTestTagProvider != null && useTestTagProvider.equals("true")) { + adminClient = RangerServiceTag.createAdminClient(tagServiceName); + } + } + if (adminClient == null) { + adminClient = RangerBasePlugin.createAdminClient(tagServiceName, appId, propertyPrefix); + } } @@ -72,22 +78,25 @@ public class RangerAdminTagRetriever extends RangerTagRefresher { @Override public void retrieveTags() { - if (rangerAdminRESTClient != null) { + if (adminClient != null) { List<RangerTaggedResource> resources = null; try { - resources = rangerAdminRESTClient.getTaggedResources(componentType); + long before = new Date().getTime(); + TagServiceResources taggedResources = adminClient.getTaggedResources(tagServiceName, componentType, lastTimestamp); + resources = taggedResources.getTaggedResources(); + lastTimestamp = before; } catch (Exception exp) { LOG.error("RangerAdminTagRetriever.retrieveTags() - Error retrieving resources"); } - if (receiver != null && resources != null) { + if (receiver != null && CollectionUtils.isNotEmpty(resources)) { receiver.setRangerTaggedResources(resources); } else { - LOG.error("RangerAdminTagRetriever.retrieveTags() - No receiver to send resources to !!"); + LOG.error("RangerAdminTagRetriever.retrieveTags() - No receiver to send resources to .. OR .. no updates to tagged resources!!"); } } else { - LOG.error("RangerAdminTagRetriever.retrieveTags() - No TagFileStore ..."); + LOG.error("RangerAdminTagRetriever.retrieveTags() - No Tag Provider ..."); } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c930d563/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagFileStoreRetriever.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagFileStoreRetriever.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagFileStoreRetriever.java index fece433..acaac75 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagFileStoreRetriever.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagFileStoreRetriever.java @@ -19,11 +19,15 @@ package org.apache.ranger.plugin.contextenricher; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.ranger.plugin.model.RangerTaggedResource; +import org.apache.ranger.plugin.store.TagStore; import org.apache.ranger.plugin.store.file.TagFileStore; +import org.apache.ranger.plugin.util.TagServiceResources; +import java.util.Date; import java.util.List; import java.util.Map; @@ -33,18 +37,21 @@ public class RangerTagFileStoreRetriever extends RangerTagRefresher { private final String componentType; private final String tagServiceName; private RangerTagReceiver receiver; - private TagFileStore tagFileStore; + + private TagStore tagStore; + private Long lastTimestamp; public RangerTagFileStoreRetriever(final String componentType, final String tagServiceName, final long pollingIntervalMs, final RangerTagReceiver enricher) { super(pollingIntervalMs); this.componentType = componentType; this.tagServiceName = tagServiceName; + this.lastTimestamp = 0L; setReceiver(enricher); } @Override - public void init(Map<String, Object> options) { - tagFileStore = TagFileStore.getInstance(); + public void init(Map<String, String> options) { + tagStore = TagFileStore.getInstance(); } @Override @@ -54,19 +61,22 @@ public class RangerTagFileStoreRetriever extends RangerTagRefresher { @Override public void retrieveTags() { - if (tagFileStore != null) { + if (tagStore != null) { List<RangerTaggedResource> resources = null; try { - resources = tagFileStore.getResources(tagServiceName, componentType); + long before = new Date().getTime(); + TagServiceResources tagServiceResources = tagStore.getResources(tagServiceName, componentType, lastTimestamp); + resources = tagServiceResources.getTaggedResources(); + lastTimestamp = before; } catch (Exception exp) { LOG.error("RangerTagFileStoreRetriever.retrieveTags() - Error retrieving resources"); } - if (receiver != null) { + if (receiver != null && CollectionUtils.isNotEmpty(resources)) { receiver.setRangerTaggedResources(resources); } else { - LOG.error("RangerTagFileStoreRetriever.retrieveTags() - No receiver to send resources to !!"); + LOG.error("RangerAdminTagRetriever.retrieveTags() - No receiver to send resources to .. OR .. no updates to tagged resources!!"); } } else { LOG.error("RangerTagFileStoreRetriever.retrieveTags() - No TagFileStore ..."); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c930d563/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagProvider.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagProvider.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagProvider.java index 9f56c83..ca8e903 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagProvider.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagProvider.java @@ -56,6 +56,8 @@ public class RangerTagProvider extends RangerAbstractContextEnricher implements super.init(); + Map<String, String> options = enricherDef != null ? enricherDef.getEnricherOptions() : null; + String tagProviderTypeString = getOption("TagProviderType", "FILE_BASED_TAG_PROVIDER"); long pollingIntervalMs = getLongOption("pollingInterval", 60 * 1000); @@ -74,7 +76,7 @@ public class RangerTagProvider extends RangerAbstractContextEnricher implements // Provide additional options if (tagRefresher != null) { - tagRefresher.init(null); + tagRefresher.init(options); tagRefresher.retrieveTags(); } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c930d563/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagRetriever.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagRetriever.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagRetriever.java index 9176664..9384256 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagRetriever.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagRetriever.java @@ -23,7 +23,7 @@ import java.util.Map; public interface RangerTagRetriever { - void init (Map<String, Object> options); + void init (Map<String, String> options); void setReceiver(RangerTagReceiver receiver); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c930d563/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerDefaultPolicyResourceMatcher.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerDefaultPolicyResourceMatcher.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerDefaultPolicyResourceMatcher.java index b78b908..4d6a00a 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerDefaultPolicyResourceMatcher.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerDefaultPolicyResourceMatcher.java @@ -500,9 +500,9 @@ public class RangerDefaultPolicyResourceMatcher implements RangerPolicyResourceM } @Override - public boolean isSingleAndExactMatch(Map<String, RangerPolicyResource> resources) { + public boolean isExactMatch(Map<String, RangerPolicyResource> resources) { if(LOG.isDebugEnabled()) { - LOG.debug("==> RangerDefaultPolicyResourceMatcher.isSingleAndExactMatch(" + resources + ")"); + LOG.debug("==> RangerDefaultPolicyResourceMatcher.isExactMatch(" + resources + ")"); } boolean ret = false; @@ -521,18 +521,12 @@ public class RangerDefaultPolicyResourceMatcher implements RangerPolicyResourceM for(RangerResourceDef resourceDef : serviceDef.getResources()) { String resourceName = resourceDef.getName(); RangerPolicyResource resourceValues = resources == null ? null : resources.get(resourceName); - RangerResourceMatcher matcher = matchers == null ? null : matchers.get(resourceName); + RangerPolicyResource policyValues = policyResources == null ? null : policyResources.get(resourceName); if(resourceValues == null || CollectionUtils.isEmpty(resourceValues.getValues())) { - ret = matcher == null; - } else if(matcher != null) { - for(String resourceValue : resourceValues.getValues()) { - ret = matcher.isMatch(resourceValue); - - if(! ret) { - break; - } - } + ret = (policyValues == null || CollectionUtils.isEmpty(policyValues.getValues())); + } else if(policyValues != null && CollectionUtils.isNotEmpty(policyValues.getValues())) { + ret = CollectionUtils.isEqualCollection(resourceValues.getValues(), policyValues.getValues()); } if(! ret) { @@ -541,13 +535,13 @@ public class RangerDefaultPolicyResourceMatcher implements RangerPolicyResourceM } } else { if(LOG.isDebugEnabled()) { - LOG.debug("isSingleAndExactMatch(): keysMatch=false. resourceKeys=" + resourceKeys + "; policyKeys=" + policyKeys); + LOG.debug("isExactMatch(): keysMatch=false. resourceKeys=" + resourceKeys + "; policyKeys=" + policyKeys); } } } if(LOG.isDebugEnabled()) { - LOG.debug("<== RangerDefaultPolicyResourceMatcher.isSingleAndExactMatch(" + resources + "): " + ret); + LOG.debug("<== RangerDefaultPolicyResourceMatcher.isExactMatch(" + resources + "): " + ret); } return ret; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c930d563/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceMatcher.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceMatcher.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceMatcher.java index cc80c8a..bf46748 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceMatcher.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceMatcher.java @@ -42,7 +42,7 @@ public interface RangerPolicyResourceMatcher { boolean isExactHeadMatch(RangerAccessResource resource); - boolean isSingleAndExactMatch(Map<String, RangerPolicyResource> resources); + boolean isExactMatch(Map<String, RangerPolicyResource> resources); StringBuilder toString(StringBuilder sb); } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c930d563/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java index 715424d..6190943 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java @@ -35,6 +35,7 @@ import org.apache.ranger.plugin.policyengine.*; import org.apache.ranger.plugin.policyevaluator.RangerPolicyEvaluator; import org.apache.ranger.plugin.util.GrantRevokeRequest; import org.apache.ranger.plugin.util.PolicyRefresher; +import org.apache.ranger.plugin.util.RangerPluginConfigPropertyRepository; import org.apache.ranger.plugin.util.ServicePolicies; @@ -56,6 +57,8 @@ public class RangerBasePlugin { public RangerBasePlugin(String serviceType, String appId) { this.serviceType = serviceType; this.appId = appId; + + RangerPluginConfigPropertyRepository.getInstance(); } public String getServiceType() { @@ -100,7 +103,7 @@ public class RangerBasePlugin { policyEngineOptions.disableCustomConditions = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".policyengine.option.disable.custom.conditions", false); policyEngineOptions.disableTagPolicyEvaluation = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".policyengine.option.disable.tagpolicy.evaluation", false); - RangerAdminClient admin = createAdminClient(propertyPrefix); + RangerAdminClient admin = createAdminClient(serviceName, appId, propertyPrefix); refresher = new PolicyRefresher(this, serviceType, appId, serviceName, admin, pollingIntervalMs, cacheDir); refresher.startRefresher(); @@ -226,10 +229,9 @@ public class RangerBasePlugin { } } - - private RangerAdminClient createAdminClient(String propertyPrefix) { + public static RangerAdminClient createAdminClient(String rangerServiceName, String applicationId, String propertyPrefix) { if(LOG.isDebugEnabled()) { - LOG.debug("==> RangerAdminRESTClient.createAdminClient(" + propertyPrefix + ")"); + LOG.debug("==> RangerAdminRESTClient.createAdminClient(" + rangerServiceName + ", " + applicationId + ", " + propertyPrefix + ")"); } RangerAdminClient ret = null; @@ -259,10 +261,10 @@ public class RangerBasePlugin { ret = new RangerAdminRESTClient(); } - ret.init(serviceName, appId, propertyPrefix); + ret.init(rangerServiceName, applicationId, propertyPrefix); if(LOG.isDebugEnabled()) { - LOG.debug("<== RangerAdminRESTClient.createAdminClient(" + propertyPrefix + "): policySourceImpl=" + policySourceImpl + ", client=" + ret); + LOG.debug("<== RangerAdminRESTClient.createAdminClient(" + rangerServiceName + ", " + applicationId + ", " + propertyPrefix + "): policySourceImpl=" + policySourceImpl + ", client=" + ret); } return ret; } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c930d563/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractTagStore.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractTagStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractTagStore.java index 0f29c25..64972ba 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractTagStore.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractTagStore.java @@ -34,12 +34,7 @@ public abstract class AbstractTagStore implements TagStore { @Override public void init() throws Exception { - - if (svcStore == null) { - throw new Exception("TagStore does not reference a valid ServiceStore"); - } - - svcStore.init(); + // Empty } @Override http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c930d563/agents-common/src/main/java/org/apache/ranger/plugin/store/TagPredicateUtil.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/TagPredicateUtil.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/TagPredicateUtil.java index dc78855..dda2779 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/TagPredicateUtil.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/TagPredicateUtil.java @@ -44,6 +44,7 @@ public class TagPredicateUtil extends AbstractPredicateUtil { addPredicateForTagResourceServiceName(filter.getParam(SearchFilter.TAG_RESOURCE_SERVICE_NAME), predicates); addPredicateForTagResourceComponentType(filter.getParam(SearchFilter.TAG_RESOURCE_COMPONENT_TYPE), predicates); + addPredicateForTagResourceTimestamp(filter.getParamAsLong(SearchFilter.TAG_RESOURCE_TIMESTAMP), predicates); addPredicateForTagResourceId(filter.getParam(SearchFilter.TAG_RESOURCE_ID), predicates); } @@ -206,4 +207,39 @@ public class TagPredicateUtil extends AbstractPredicateUtil { return ret; } -} + private Predicate addPredicateForTagResourceTimestamp(final Long lastTimestamp, List<Predicate> predicates) { + final int uploadInterval = 1*1000; + // Assumption: it may take maximum of one second for a taggedResource to be persisted after the timestamp + // was generated for it. The round-trip time is already taken into consideration by client. + + + if (lastTimestamp == null) { + return null; + } + + Predicate ret = new Predicate() { + @Override + public boolean evaluate(Object object) { + + boolean ret = false; + + if (object == null) { + return ret; + } + + if (object instanceof RangerTaggedResource) { + RangerTaggedResource rangerResource = (RangerTaggedResource) object; + + ret = rangerResource.getUpdateTime().getTime() >= (lastTimestamp - uploadInterval); + } + + return ret; + } + }; + + if (predicates != null) { + predicates.add(ret); + } + + return ret; + }} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c930d563/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java index 384251b..9200ee2 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java @@ -24,6 +24,7 @@ import org.apache.ranger.plugin.model.RangerTaggedResourceKey; import org.apache.ranger.plugin.model.RangerTaggedResource; import org.apache.ranger.plugin.model.RangerTagDef; import org.apache.ranger.plugin.util.SearchFilter; +import org.apache.ranger.plugin.util.TagServiceResources; import java.util.List; import java.util.Map; @@ -52,7 +53,7 @@ public interface TagStore { PList<RangerTagDef> getPaginatedTagDefs(SearchFilter filter) throws Exception; - RangerTaggedResource createResource(RangerTaggedResource resource) throws Exception; + RangerTaggedResource createResource(RangerTaggedResource resource, boolean createOrUpdate) throws Exception; RangerTaggedResource updateResource(RangerTaggedResource resource) throws Exception; @@ -60,16 +61,15 @@ public interface TagStore { RangerTaggedResource getResource(Long id) throws Exception; - List<RangerTaggedResource> getResources(String tagServiceName, String componentType) throws Exception; + TagServiceResources getResources(String tagServiceName, String componentType, Long lastTimestamp) throws Exception; List<RangerTaggedResource> getResources(SearchFilter filter) throws Exception; PList<RangerTaggedResource> getPaginatedResources(SearchFilter filter) throws Exception; - Set<String> getTags(String tagServiceName, String serviceType) throws Exception; + List<String> getTags(String tagServiceName, String serviceType) throws Exception; - Set<String> lookupTags(String tagServiceName, String serviceType, String tagNamePattern) throws Exception; + List<String> lookupTags(String tagServiceName, String serviceType, String tagNamePattern) throws Exception; - //List<RangerTaggedResource> getResources(String componentType, Map<String, RangerPolicy.RangerPolicyResource> resourceSpec) throws Exception; - List<RangerTaggedResource> getResources(RangerTaggedResourceKey key) throws Exception; + RangerTaggedResource getResource(RangerTaggedResourceKey key) throws Exception; } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c930d563/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java index b641902..6913212 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java @@ -29,11 +29,11 @@ import org.apache.hadoop.fs.Path; import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; import org.apache.ranger.plugin.model.*; import org.apache.ranger.plugin.policyresourcematcher.RangerDefaultPolicyResourceMatcher; -import org.apache.ranger.plugin.service.ResourceLookupContext; import org.apache.ranger.plugin.store.AbstractTagStore; import org.apache.ranger.plugin.store.TagPredicateUtil; -import org.apache.ranger.plugin.store.rest.ServiceRESTStore; +import org.apache.ranger.plugin.store.TagStore; import org.apache.ranger.plugin.util.SearchFilter; +import org.apache.ranger.plugin.util.TagServiceResources; import java.util.*; import java.util.regex.Matcher; @@ -56,7 +56,7 @@ public class TagFileStore extends AbstractTagStore { private volatile static TagFileStore instance = null; - public static TagFileStore getInstance() { + public static TagStore getInstance() { if (instance == null) { synchronized (TagFileStore.class) { if (instance == null) { @@ -287,36 +287,51 @@ public class TagFileStore extends AbstractTagStore { } @Override - public RangerTaggedResource createResource(RangerTaggedResource resource) throws Exception { + public RangerTaggedResource createResource(RangerTaggedResource resource, boolean createOrUpdate) throws Exception { if (LOG.isDebugEnabled()) { LOG.debug("==> TagFileStore.createResource(" + resource + ")"); } + RangerTaggedResource ret = null; RangerTaggedResource existing = null; - if (resource.getId() != null) { - existing = getResource(resource.getId()); - } + boolean updateResource = false; + + existing = getResource(resource.getKey()); if (existing != null) { - throw new Exception(resource.getId() + ": resource already exists (id=" + existing.getId() + ")"); + if (!createOrUpdate) { + throw new Exception("resource(s) with same specification already exists"); + } else { + updateResource = true; + } } - List<RangerTaggedResource> existingResources = getResources(resource.getKey()); + if (! updateResource) { + if (resource.getId() != null) { + existing = getResource(resource.getId()); + } - if (CollectionUtils.isNotEmpty(existingResources)) { - throw new Exception("resource(s) with same specification already exists"); + if (existing != null) { + if (! createOrUpdate) { + throw new Exception(resource.getId() + ": resource already exists (id=" + existing.getId() + ")"); + } else { + updateResource = true; + } + } } - RangerTaggedResource ret; - try { - preCreate(resource); + if (updateResource) { + ret = updateResource(resource); + } else { + preCreate(resource); - resource.setId(nextTagResourceId); + resource.setId(nextTagResourceId); - ret = fileStoreUtil.saveToFile(resource, new Path(fileStoreUtil.getDataFile(FILE_PREFIX_TAG_RESOURCE, nextTagResourceId++)), false); + ret = fileStoreUtil.saveToFile(resource, new Path(fileStoreUtil.getDataFile(FILE_PREFIX_TAG_RESOURCE, nextTagResourceId++)), false); - postCreate(ret); + postCreate(ret); + } } catch (Exception excp) { LOG.warn("TagFileStore.createResource(): failed to save resource '" + resource.getId() + "'", excp); @@ -341,7 +356,7 @@ public class TagFileStore extends AbstractTagStore { throw new Exception(resource.getId() + ": resource does not exist (id=" + resource.getId() + ")"); } - RangerTaggedResource ret; + RangerTaggedResource ret = null; try { preUpdate(existing); @@ -419,18 +434,22 @@ public class TagFileStore extends AbstractTagStore { } @Override - public List<RangerTaggedResource> getResources(RangerTaggedResourceKey key) throws Exception { + public RangerTaggedResource getResource(RangerTaggedResourceKey key) throws Exception { if (LOG.isDebugEnabled()) { - LOG.debug("==> TagFileStore.getResources( " + key.getTagServiceName() + ", " + key.getComponentType() + " )"); + LOG.debug("==> TagFileStore.getResource( " + key.getTagServiceName() + ", " + key.getComponentType() + " )"); } if (this.svcStore == null) { - LOG.error("TagFileStore.getResources() - TagFileStore object does not have reference to a valid ServiceStore."); + LOG.error("TagFileStore.getResource() - TagFileStore object does not have reference to a valid ServiceStore."); throw new Exception("TagFileStore.getResources() - TagFileStore object does not have reference to a valid ServiceStore."); } - List<RangerTaggedResource> ret = null; + if (key == null) { + LOG.error("TagFileStore.getResource() - parameter 'key' is null"); + throw new Exception("TagFileStore.getResources() - parameter 'key' is null."); + } + RangerServiceDef serviceDef = null; try { @@ -440,13 +459,17 @@ public class TagFileStore extends AbstractTagStore { throw new Exception("Invalid component-type: " + key.getComponentType()); } + List<RangerTaggedResource> resources = null; + if (MapUtils.isNotEmpty(key.getResourceSpec())) { - ret = getResources(key.getTagServiceName(), key.getComponentType()); + TagServiceResources tagServiceResources = getResources(key.getTagServiceName(), key.getComponentType(), 0L); + resources = tagServiceResources.getTaggedResources(); + List<RangerTaggedResource> notMatchedResources = new ArrayList<>(); - if (CollectionUtils.isNotEmpty(ret)) { - for (RangerTaggedResource resource : ret) { + if (CollectionUtils.isNotEmpty(resources)) { + for (RangerTaggedResource resource : resources) { RangerDefaultPolicyResourceMatcher policyResourceMatcher = new RangerDefaultPolicyResourceMatcher(); @@ -457,7 +480,7 @@ public class TagFileStore extends AbstractTagStore { policyResourceMatcher.init(); - boolean isMatch = policyResourceMatcher.isSingleAndExactMatch(key.getResourceSpec()); + boolean isMatch = policyResourceMatcher.isExactMatch(key.getResourceSpec()); if (! isMatch) { notMatchedResources.add(resource); @@ -466,23 +489,27 @@ public class TagFileStore extends AbstractTagStore { } - ret.removeAll(notMatchedResources); + resources.removeAll(notMatchedResources); } } else { - ret = null; + resources = null; } + + RangerTaggedResource ret = (resources == null || CollectionUtils.isEmpty(resources)) ? null : resources.get(0); + if (LOG.isDebugEnabled()) { - LOG.debug("==> TagFileStore.getResources( " + key.getTagServiceName() + ", " + key.getComponentType() + " )" + ret); + LOG.debug("==> TagFileStore.getResource( " + key.getTagServiceName() + ", " + key.getComponentType() + " )" + ret); } + return ret; } @Override - public List<RangerTaggedResource> getResources(String tagServiceName, String componentType) throws Exception { + public TagServiceResources getResources(String tagServiceName, String componentType, Long lastTimeStamp) throws Exception { if (LOG.isDebugEnabled()) { - LOG.debug("==> TagFileStore.getResources(" + tagServiceName + ", " + componentType + ")"); + LOG.debug("==> TagFileStore.getResources(" + tagServiceName + ", " + componentType + ", " + lastTimeStamp + ")"); } - List<RangerTaggedResource> ret; + List<RangerTaggedResource> taggedResources; SearchFilter filter = new SearchFilter(); @@ -494,13 +521,25 @@ public class TagFileStore extends AbstractTagStore { filter.setParam(SearchFilter.TAG_RESOURCE_COMPONENT_TYPE, componentType); } - ret = getResources(filter); + if (lastTimeStamp != null) { + filter.setParam(SearchFilter.TAG_RESOURCE_TIMESTAMP, Long.toString(lastTimeStamp.longValue())); + } + + taggedResources = getResources(filter); if (LOG.isDebugEnabled()) { - LOG.debug("<== TagFileStore.getResources(" + tagServiceName + ", " + componentType + ")"); + LOG.debug("<== TagFileStore.getResources(" + tagServiceName + ", " + componentType + ", " + lastTimeStamp + ")"); } + + TagServiceResources ret = new TagServiceResources(); + ret.setTaggedResources(taggedResources); + // TBD + ret.setLastUpdateTime(new Date()); + ret.setVersion(1L); + return ret; + } @Override @@ -621,22 +660,24 @@ public class TagFileStore extends AbstractTagStore { } @Override - public Set<String> getTags(String tagServiceName, String componentType) throws Exception { + public List<String> getTags(String tagServiceName, String componentType) throws Exception { if (LOG.isDebugEnabled()) { LOG.debug("==> TagFileStore.getTags(" + tagServiceName + ", " + componentType + ")"); } - Set<String> ret = new HashSet<String>(); + SortedSet<String> tagNameSet = new TreeSet<String>(); + + TagServiceResources tagServiceResources = getResources(tagServiceName, componentType, 0L); + List<RangerTaggedResource> resources = tagServiceResources.getTaggedResources(); - List<RangerTaggedResource> resources = getResources(tagServiceName, componentType); if (CollectionUtils.isNotEmpty(resources)) { for (RangerTaggedResource resource : resources) { List<RangerTaggedResource.RangerResourceTag> tags = resource.getTags(); if (CollectionUtils.isNotEmpty(tags)) { for (RangerTaggedResource.RangerResourceTag tag : tags) { - ret.add(tag.getName()); + tagNameSet.add(tag.getName()); } } } @@ -646,22 +687,22 @@ public class TagFileStore extends AbstractTagStore { LOG.debug("<== TagFileStore.getTags(" + tagServiceName + ", " + componentType + ")"); } - return ret; + return new ArrayList<String>(tagNameSet); } @Override - public Set<String> lookupTags(String tagServiceName, String componentType, String tagNamePattern) throws Exception { + public List<String> lookupTags(String tagServiceName, String componentType, String tagNamePattern) throws Exception { if (LOG.isDebugEnabled()) { LOG.debug("==> TagFileStore.lookupTags(" + tagServiceName + ", " + componentType + ", " + tagNamePattern + ")"); } - Set<String> tagNameSet = getTags(tagServiceName, componentType); - Set<String> matchedTagSet = new HashSet<String>(); + List<String> tagNameList = getTags(tagServiceName, componentType); + List<String> matchedTagList = new ArrayList<String>(); - if (CollectionUtils.isNotEmpty(tagNameSet)) { + if (CollectionUtils.isNotEmpty(tagNameList)) { Pattern p = Pattern.compile(tagNamePattern); - for (String tagName : tagNameSet) { + for (String tagName : tagNameList) { Matcher m = p.matcher(tagName); if (LOG.isDebugEnabled()) { LOG.debug("TagFileStore.lookupTags) - Trying to match .... tagNamePattern=" + tagNamePattern + ", tagName=" + tagName); @@ -670,7 +711,7 @@ public class TagFileStore extends AbstractTagStore { if (LOG.isDebugEnabled()) { LOG.debug("TagFileStore.lookupTags) - Match found.... tagNamePattern=" + tagNamePattern + ", tagName=" + tagName); } - matchedTagSet.add(tagName); + matchedTagList.add(tagName); } } } @@ -679,7 +720,7 @@ public class TagFileStore extends AbstractTagStore { LOG.debug("<== TagFileStore.lookupTags(" + tagServiceName + ", " + componentType + ", " + tagNamePattern + ")"); } - return matchedTagSet; + return matchedTagList; } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c930d563/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPluginConfigPropertyRepository.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPluginConfigPropertyRepository.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPluginConfigPropertyRepository.java new file mode 100644 index 0000000..0c8e653 --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPluginConfigPropertyRepository.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.util; + +import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; +import org.apache.ranger.common.RangerConfigPropertyRepository; + +public class RangerPluginConfigPropertyRepository extends RangerConfigPropertyRepository { + + public static RangerConfigPropertyRepository getInstance() { + RangerConfigPropertyRepository ret = instance; + + if (ret == null) { + synchronized(RangerConfigPropertyRepository.class) { + ret = instance; + if (ret == null) { + ret = instance = new RangerPluginConfigPropertyRepository(); + } + } + } + + return ret; + } + + @Override + protected final String getPropertyValue(String propertyName) { + return RangerConfiguration.getInstance().get(propertyName); + } + + private RangerPluginConfigPropertyRepository() {} +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c930d563/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRESTUtils.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRESTUtils.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRESTUtils.java index af35c32..d0eda13 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRESTUtils.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRESTUtils.java @@ -40,13 +40,16 @@ public class RangerRESTUtils { public static final String REST_URL_SERVICE_GRANT_ACCESS = "/service/plugins/services/grant/"; public static final String REST_URL_SERVICE_REVOKE_ACCESS = "/service/plugins/services/revoke/"; - public static final String REST_URL_GET_TAGGED_RESOURCES = "/service/tag-def/v1/resources"; + public static final String REST_URL_GET_UPDATED_TAGGED_RESOURCES = "/service/tag-def/v1/resources/updated-resources"; + public static final String REST_URL_SET_TAGGED_RESOURCE = "/service/tag-def/v1/resources/set-resource"; + public static final String REST_URL_SET_TAGGED_RESOURCES = "/service/tag-def/v1/resources/set-resources"; + public static final String REST_URL_UPDATE_TAGGED_RESOURCE = "/service/tag-def/v1/resources/update-resource"; public static final String TAG_SERVICE_NAME_PARAM = "tagservicename"; public static final String COMPONENT_TYPE_PARAM = "componenttype"; + public static final String TAG_TIMESTAMP_PARAM = "tagtimestamp"; public static final String TAG_PATTERN_PARAM = "tagpattern"; - public static final String REST_URL_GET_TAG_NAMES = "/service/tag-def/v1/tag-names"; - public static final String REST_URL_LOOKUP_TAG_NAMES = "/service/tag-def/v1/lookup-tags"; + public static final String REST_URL_LOOKUP_TAG_NAMES = "/service/tag-def/v1/resources/lookup-tags"; public static final String REST_EXPECTED_MIME_TYPE = "application/json" ; public static final String REST_MIME_TYPE_JSON = "application/json" ; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c930d563/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java index 1aadbf5..2585138 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java @@ -55,6 +55,7 @@ public class SearchFilter { public static final String TAG_RESOURCE_ID = "tagResourceId"; // search public static final String TAG_RESOURCE_SERVICE_NAME = "tagResourceServiceName"; // search public static final String TAG_RESOURCE_COMPONENT_TYPE = "tagResourceComponentType"; // search + public static final String TAG_RESOURCE_TIMESTAMP = "tagResourceTimestamp"; // search @@ -90,6 +91,18 @@ public class SearchFilter { return params == null ? null : params.get(name); } + public Long getParamAsLong(String name) { + + String stringValue = params == null ? null : params.get(name); + Long ret = null; + try { + ret = Long.valueOf(stringValue); + } catch (NumberFormatException exception) { + // Ignore + } + return ret; + } + public void setParam(String name, String value) { if(StringUtils.isEmpty(name) || StringUtils.isEmpty(value)) { return; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c930d563/agents-common/src/main/java/org/apache/ranger/plugin/util/TagServiceResources.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/TagServiceResources.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/TagServiceResources.java new file mode 100644 index 0000000..9eef939 --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/TagServiceResources.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.util; + + +import java.util.Date; +import java.util.List; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.ranger.plugin.model.RangerTaggedResource; +import org.codehaus.jackson.annotate.JsonAutoDetect; +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.annotate.JsonAutoDetect.Visibility; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +import com.google.common.base.Objects; + +@JsonAutoDetect(getterVisibility=Visibility.NONE, setterVisibility=Visibility.NONE, fieldVisibility=Visibility.ANY) +@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL ) +@JsonIgnoreProperties(ignoreUnknown=true) +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class TagServiceResources implements java.io.Serializable { + private static final long serialVersionUID = 1L; + + private Long version; // RFU + private Date lastUpdateTime; // RFU + private List<RangerTaggedResource> taggedResources; + + public Long getVersion() { + return version; + } + /** + * @param version the version to set + */ + public void setVersion(Long version) { + this.version = version; + } + /** + * @return the lastUpdateTime + */ + public Date getLastUpdateTime() { + return lastUpdateTime; + } + /** + * @param lastUpdateTime the lastUpdateTime to set + */ + public void setLastUpdateTime(Date lastUpdateTime) { + this.lastUpdateTime = lastUpdateTime; + } + /** + * @return the tagged resources + */ + public List<RangerTaggedResource> getTaggedResources() { + return taggedResources; + } + /** + * @param taggedResources the taggedResources to set + */ + public void setTaggedResources(List<RangerTaggedResource> taggedResources) { + this.taggedResources = taggedResources; + } + + @Override + public String toString() { + return Objects.toStringHelper(this.getClass()) + .add("version", version) + .add("lastUpdateTime", lastUpdateTime) + .add("taggedResources", taggedResources) + .toString(); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c930d563/agents-common/src/main/java/org/apache/ranger/services/tag/RangerServiceTag.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/services/tag/RangerServiceTag.java b/agents-common/src/main/java/org/apache/ranger/services/tag/RangerServiceTag.java index 7c7fa96..c6dc334 100644 --- a/agents-common/src/main/java/org/apache/ranger/services/tag/RangerServiceTag.java +++ b/agents-common/src/main/java/org/apache/ranger/services/tag/RangerServiceTag.java @@ -3,13 +3,18 @@ package org.apache.ranger.services.tag; import java.util.*; import org.apache.commons.collections.MapUtils; +import org.apache.ranger.admin.client.RangerAdminClient; import org.apache.ranger.admin.client.RangerAdminRESTClient; import org.apache.ranger.plugin.model.RangerService; import org.apache.ranger.plugin.model.RangerServiceDef; +import org.apache.ranger.plugin.service.RangerBasePlugin; import org.apache.ranger.plugin.service.RangerBaseService; import org.apache.ranger.plugin.service.ResourceLookupContext; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.ranger.plugin.store.ServiceStore; +import org.apache.ranger.plugin.store.TagStore; +import org.apache.ranger.plugin.store.file.TagFileStore; public class RangerServiceTag extends RangerBaseService { @@ -17,6 +22,10 @@ public class RangerServiceTag extends RangerBaseService { public static final String TAG = "tag"; + public static final String propertyPrefix = "ranger.plugin.tag"; + + public static final String applicationId = "Ranger-GUI"; + public RangerServiceTag() { super(); } @@ -29,39 +38,24 @@ public class RangerServiceTag extends RangerBaseService { @Override public HashMap<String,Object> validateConfig() throws Exception { HashMap<String, Object> ret = new HashMap<String, Object>(); - String serviceName = getServiceName(); - boolean connectivityStatus = false; - String message = null; + String serviceName = getServiceName(); + boolean connectivityStatus = false; + String message = null; if(LOG.isDebugEnabled()) { LOG.debug("==> RangerServiceTag.validateConfig - Service: (" + serviceName + " )"); } - if (MapUtils.isEmpty(configs)) { - message = "Configuration is null or empty"; - - } else { - String url = configs.get("URL"); - String sslConfigFileName = configs.get("SSL_CONFIG_FILE_NAME"); - String userName = configs.get("username"); - String password = configs.get("password"); - - if (url == null || sslConfigFileName == null || userName == null || password == null) { - message = "Either URL, SSL_CONFIG_FILE_NAME, username or password not provided in configuration"; - } else { - - RangerAdminRESTClient adminRESTClient = new RangerAdminRESTClient(); - adminRESTClient.init(serviceName, configs); + RangerAdminClient adminClient = createAdminClient(serviceName); - try { - adminRESTClient.getTagNames(null, ".*"); // Dont care about componentType - connectivityStatus = true; - } catch (Exception e) { - LOG.error("RangerServiceTag.validateConfig() Error:" + e); - connectivityStatus = false; - message = "Cannot connect to TagResource Repository, " + e; - } - } + try { + adminClient.getTagNames(serviceName, null, ".*"); // Don't care about componentType + connectivityStatus = true; + } catch (Exception e) { + LOG.error("RangerServiceTag.validateConfig() Error:" + e); + connectivityStatus = false; + message = "Cannot connect to TagResource Repository, Exception={" + e + "}. " + "Please check " + + propertyPrefix + " sub-properties."; } ret.put("connectivityStatus", connectivityStatus); @@ -77,70 +71,63 @@ public class RangerServiceTag extends RangerBaseService { @Override public List<String> lookupResource(ResourceLookupContext context) throws Exception { String serviceName = getServiceName(); - Map<String,String> configs = getConfigs(); if(LOG.isDebugEnabled()) { LOG.debug("==> RangerServiceTag.lookupResource - Context: (" + context + ")"); } - Set<String> tagNameSet = new HashSet<>(); - - if (MapUtils.isNotEmpty(configs)) { - String url = configs.get("URL"); - String sslConfigFileName = configs.get("SSL_CONFIG_FILE_NAME"); - String userName = configs.get("username"); - String password = configs.get("password"); + List<String> tagNameList = new ArrayList<>(); - if (url != null && sslConfigFileName != null && userName != null && password != null) { + if (context != null) { - if (context != null) { + String userInput = context.getUserInput(); + String resource = context.getResourceName(); + Map<String, List<String>> resourceMap = context.getResources(); + final List<String> userProvidedTagList = new ArrayList<>(); - String userInput = context.getUserInput(); - String resource = context.getResourceName(); - Map<String, List<String>> resourceMap = context.getResources(); - final Set<String> userProvidedTagSet = new HashSet<String>(); + if (resource != null && resourceMap != null && resourceMap.get(TAG) != null) { - if (resource != null && resourceMap != null && resourceMap.get(TAG) != null) { + for (String tag : resourceMap.get(TAG)) { + userProvidedTagList.add(tag); + } - for (String tag : resourceMap.get(TAG)) { - userProvidedTagSet.add(tag); - } + String suffix = ".*"; + String tagNamePattern; - try { - String suffix = ".*"; - String tagNamePattern; + if (userInput == null) { + tagNamePattern = suffix; + } else { + tagNamePattern = userInput + suffix; + } - if (userInput == null) { - tagNamePattern = suffix; - } else { - tagNamePattern = userInput + suffix; - } + if (LOG.isDebugEnabled()) { + LOG.debug("RangerServiceTag.lookupResource - tagNamePattern : (" + tagNamePattern + ")"); + } - if(LOG.isDebugEnabled()) { - LOG.debug("RangerServiceTag.lookupResource - tagNamePattern : (" + tagNamePattern + ")"); - } + try { - RangerAdminRESTClient adminRESTClient = new RangerAdminRESTClient(); - adminRESTClient.init(serviceName, configs); + RangerAdminClient adminClient = createAdminClient(serviceName); - tagNameSet = adminRESTClient.getTagNames(null, tagNamePattern); // Dont care about componentType + tagNameList = adminClient.getTagNames(serviceName, null, tagNamePattern); // Don't care about componentType - tagNameSet.removeAll(userProvidedTagSet); + tagNameList.removeAll(userProvidedTagList); - } catch (Exception e) { - LOG.error("RangerServiceTag.lookupResource - Error : " + e); - } - } + } catch (Exception e) { + LOG.error("RangerServiceTag.lookupResource - Exception={" + e + "}. " + "Please check " + + propertyPrefix + " sub-properties."); } - } - } if(LOG.isDebugEnabled()) { LOG.debug("<== RangerServiceTag.lookupResource()"); } - return new ArrayList<String>(tagNameSet); + return tagNameList; } + + public static RangerAdminClient createAdminClient( String tagServiceName ) { + return RangerBasePlugin.createAdminClient(tagServiceName, applicationId, propertyPrefix); + } + } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c930d563/agents-common/src/main/resources/service-defs/ranger-servicedef-tag.json ---------------------------------------------------------------------- diff --git a/agents-common/src/main/resources/service-defs/ranger-servicedef-tag.json b/agents-common/src/main/resources/service-defs/ranger-servicedef-tag.json index 1593280..de384ad 100644 --- a/agents-common/src/main/resources/service-defs/ranger-servicedef-tag.json +++ b/agents-common/src/main/resources/service-defs/ranger-servicedef-tag.json @@ -7,8 +7,7 @@ "guid": "0d047248-baff-4cf9-8e9e-d5d377284b2e", "options": { - "ui.pages":"tag-based-policies", - "ui.isMultiline":"true" + "ui.pages":"tag-based-policies" }, "resources": [ @@ -92,7 +91,7 @@ "name" : "TagEnricher", "enricher" : "org.apache.ranger.plugin.contextenricher.RangerTagProvider", "enricherOptions" : { - "TagProviderType":"FILESTORE_BASED_TAG_PROVIDER", + "TagProviderType":"RANGER_ADMIN_TAG_PROVIDER", "pollingInterval":60000 } } @@ -104,7 +103,7 @@ "itemId":1, "name":"ScriptConditionEvaluator", "evaluator": "org.apache.ranger.plugin.conditionevaluator.RangerScriptConditionEvaluator", - "evaluatorOptions" : {"engineName":"JavaScript"}, + "evaluatorOptions" : {"engineName":"JavaScript", "ui.isMultiline":"true" }, "label":"Script", "description": "Script to execute" } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c930d563/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java ---------------------------------------------------------------------- diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java b/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java index 775f221..7c4118d 100644 --- a/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java +++ b/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java @@ -22,22 +22,25 @@ package org.apache.ranger.plugin.policyengine; import com.google.gson.*; import com.google.gson.reflect.TypeToken; import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.ranger.audit.provider.AuditHandler; import org.apache.ranger.audit.provider.AuditProviderFactory; +import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; import org.apache.ranger.plugin.audit.RangerDefaultAuditHandler; import org.apache.ranger.plugin.model.RangerPolicy; import org.apache.ranger.plugin.model.RangerTaggedResource; import org.apache.ranger.plugin.model.RangerServiceDef; import org.apache.ranger.plugin.policyengine.TestPolicyEngine.PolicyEngineTestCase.TestData; +import org.apache.ranger.plugin.util.RangerPluginConfigPropertyRepository; import org.apache.ranger.plugin.util.ServicePolicies; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; -import java.io.File; -import java.io.FileInputStream; -import java.io.InputStream; -import java.io.InputStreamReader; +import java.io.*; import java.lang.reflect.Type; import java.util.List; import java.util.Map; @@ -54,10 +57,10 @@ public class TestPolicyEngine { @BeforeClass public static void setUpBeforeClass() throws Exception { gsonBuilder = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z") - .setPrettyPrinting() - .registerTypeAdapter(RangerAccessRequest.class, new RangerAccessRequestDeserializer()) - .registerTypeAdapter(RangerAccessResource.class, new RangerResourceDeserializer()) - .create(); + .setPrettyPrinting() + .registerTypeAdapter(RangerAccessRequest.class, new RangerAccessRequestDeserializer()) + .registerTypeAdapter(RangerAccessResource.class, new RangerResourceDeserializer()) + .create(); // For setting up auditProvider Properties auditProperties = new Properties(); @@ -66,7 +69,7 @@ public class TestPolicyEngine { File propFile = new File(AUDIT_PROPERTIES_FILE); - if(propFile.exists()) { + if (propFile.exists()) { System.out.println("Loading Audit properties file" + AUDIT_PROPERTIES_FILE); auditProperties.load(new FileInputStream(propFile)); @@ -96,6 +99,32 @@ public class TestPolicyEngine { System.out.println("provider=" + provider.toString()); + // For setting up TestTagProvider + + Path filePath = new Path("file:///tmp/ranger-admin-test-site.xml"); + Configuration config = new Configuration(); + + FileSystem fs = filePath.getFileSystem(config); + + FSDataOutputStream outStream = fs.create(filePath, true); + OutputStreamWriter writer = null; + + + writer = new OutputStreamWriter(outStream); + + writer.write("<configuration>\n" + + " <property>\n" + + " <name>ranger.plugin.tag.policy.rest.url</name>\n" + + " <value>http://node-1.example.com:6080</value>\n" + + " </property>\n" + + "</configuration>\n"); + + writer.close(); + + RangerConfiguration rangerConfig = RangerConfiguration.getInstance(); + rangerConfig.addResource(filePath); + + RangerPluginConfigPropertyRepository.getInstance(); } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c930d563/agents-common/src/test/java/org/apache/ranger/plugin/store/TestTagStore.java ---------------------------------------------------------------------- diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/store/TestTagStore.java b/agents-common/src/test/java/org/apache/ranger/plugin/store/TestTagStore.java index 65913f7..e36f318 100644 --- a/agents-common/src/test/java/org/apache/ranger/plugin/store/TestTagStore.java +++ b/agents-common/src/test/java/org/apache/ranger/plugin/store/TestTagStore.java @@ -76,6 +76,10 @@ public class TestTagStore { " <value>admin</value>\n" + " </property>\n" + " <property>\n" + + " <name>ranger.plugin.tag.policy.rest.url</name>\n" + + " <value>http://node-1.example.com:6080</value>\n" + + " </property>\n" + + " <property>\n" + " <name>ranger.tag.store.file.dir</name>\n" + " <value>file:///etc/ranger/data</value>\n" + " </property>\n" + @@ -87,8 +91,12 @@ public class TestTagStore { config.addResource(filePath); tagStore = TagFileStore.getInstance(); - tagStore.setServiceStore(new ServiceRESTStore()); tagStore.init(); + + ServiceStore svcStore = new ServiceRESTStore(); + svcStore.init(); + + tagStore.setServiceStore(svcStore); */ } @@ -133,6 +141,7 @@ public class TestTagStore { /* List<RangerTaggedResource> taggedResources = tagStore.getResources(filter); + RangerTaggedResource taggedResource = null; int initResourceCount = taggedResources == null ? 0 : taggedResources.size(); @@ -163,19 +172,19 @@ public class TestTagStore { rr.setTags(tags); - RangerTaggedResource createdResource = tagStore.createResource(rr); + RangerTaggedResource createdResource = tagStore.createResource(rr, false); assertNotNull("createResource() failed", createdResource); taggedResources = tagStore.getResources(filter); assertEquals("createResource() failed", initResourceCount + 1, taggedResources == null ? 0 : taggedResources.size()); - taggedResources = tagStore.getResources(rr.getKey()); - assertEquals("createResource() failed", initResourceCount + 1, taggedResources == null ? 0 : taggedResources.size()); + taggedResource = tagStore.getResource(rr.getKey()); + assertNotNull("createResource() failed", taggedResource); rr.getKey().getResourceSpec().remove("column"); - taggedResources = tagStore.getResources(rr.getKey()); - assertEquals("createResource() failed", initResourceCount, taggedResources == null ? 0 : taggedResources.size()); + taggedResource = tagStore.getResource(rr.getKey()); + assertNull("createResource() failed", taggedResource); */ } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c930d563/agents-common/src/test/resources/policyengine/test_policyengine_tag_hdfs.json ---------------------------------------------------------------------- diff --git a/agents-common/src/test/resources/policyengine/test_policyengine_tag_hdfs.json b/agents-common/src/test/resources/policyengine/test_policyengine_tag_hdfs.json index 0c14dfa..b1b18f2 100644 --- a/agents-common/src/test/resources/policyengine/test_policyengine_tag_hdfs.json +++ b/agents-common/src/test/resources/policyengine/test_policyengine_tag_hdfs.json @@ -105,7 +105,7 @@ "itemId": 1, "name" : "TagEnricher", "enricher" : "org.apache.ranger.plugin.contextenricher.RangerTagProvider", - "enricherOptions" : {"TagProviderType":"FILESTORE_BASED_TAG_PROVIDER", "pollingInterval":-1, "dataFile":"/etc/ranger/data/resourceTags.txt"} + "enricherOptions" : {"TagProviderType":"FILESTORE_BASED_TAG_PROVIDER", "pollingInterval":-1, "useTestTagProvider":"false", "dataFile":"/etc/ranger/data/resourceTags.txt"} } ], "policyConditions": [ http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c930d563/agents-common/src/test/resources/policyengine/test_policyengine_tag_hive.json ---------------------------------------------------------------------- diff --git a/agents-common/src/test/resources/policyengine/test_policyengine_tag_hive.json b/agents-common/src/test/resources/policyengine/test_policyengine_tag_hive.json index d604c44..87f2bac 100644 --- a/agents-common/src/test/resources/policyengine/test_policyengine_tag_hive.json +++ b/agents-common/src/test/resources/policyengine/test_policyengine_tag_hive.json @@ -130,7 +130,7 @@ "itemId": 1, "name" : "TagEnricher", "enricher" : "org.apache.ranger.plugin.contextenricher.RangerTagProvider", - "enricherOptions" : {"TagProviderType":"FILESTORE_BASED_TAG_PROVIDER", "pollingInterval":-1, "dataFile":"/etc/ranger/data/resourceTags.txt"} + "enricherOptions" : {"TagProviderType":"RANGER_ADMIN_TAG_PROVIDER", "pollingInterval":-1, "useTestTagProvider":"true", "dataFile":"/etc/ranger/data/resourceTags.txt"} } ], "policyConditions": [ http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c930d563/knox-agent/src/main/java/org/apache/ranger/admin/client/RangerAdminJersey2RESTClient.java ---------------------------------------------------------------------- diff --git a/knox-agent/src/main/java/org/apache/ranger/admin/client/RangerAdminJersey2RESTClient.java b/knox-agent/src/main/java/org/apache/ranger/admin/client/RangerAdminJersey2RESTClient.java index 7fc33f9..f5ac163 100644 --- a/knox-agent/src/main/java/org/apache/ranger/admin/client/RangerAdminJersey2RESTClient.java +++ b/knox-agent/src/main/java/org/apache/ranger/admin/client/RangerAdminJersey2RESTClient.java @@ -21,6 +21,7 @@ package org.apache.ranger.admin.client; import java.lang.reflect.Type; import java.util.Date; +import java.util.List; import javax.net.ssl.HostnameVerifier; import javax.net.ssl.SSLContext; @@ -33,10 +34,9 @@ import javax.ws.rs.core.Response; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.security.AccessControlException; -import org.apache.ranger.plugin.util.GrantRevokeRequest; -import org.apache.ranger.plugin.util.RangerRESTUtils; -import org.apache.ranger.plugin.util.RangerSslHelper; -import org.apache.ranger.plugin.util.ServicePolicies; +import org.apache.ranger.plugin.model.RangerTaggedResource; +import org.apache.ranger.plugin.model.RangerTaggedResourceKey; +import org.apache.ranger.plugin.util.*; import com.google.gson.Gson; import com.google.gson.GsonBuilder; @@ -203,6 +203,16 @@ public class RangerAdminJersey2RESTClient implements RangerAdminClient { } } + @Override + public TagServiceResources getTaggedResources(String tagServiceName, String componentType, Long lastTimestamp) throws Exception { + throw new Exception("RangerAdminjersey2RESTClient.getTaggedResource() -- *** NOT IMPLEMENTED *** "); + } + + @Override + public List<String> getTagNames(String tagServiceName, String componentType, String tagNamePattern) throws Exception { + throw new Exception("RangerAdminjersey2RESTClient.getTagNames() -- *** NOT IMPLEMENTED *** "); + } + // We get date from the policy manager as unix long! This deserializer exists to deal with it. Remove this class once we start send date/time per RFC 3339 public static class GsonUnixDateDeserializer implements JsonDeserializer<Date> { @@ -213,6 +223,28 @@ public class RangerAdminJersey2RESTClient implements RangerAdminClient { } + @Override + public TagServiceResources getAllTaggedResources() throws Exception { + throw new Exception("RangerAdminjersey2RESTClient.getAllTaggedResources() -- *** NOT IMPLEMENTED *** "); + } + + @Override + public List<RangerTaggedResource> setResources(List<RangerTaggedResourceKey> keys, List<RangerTaggedResource.RangerResourceTag> tags) throws Exception { + throw new Exception("RangerAdminjersey2RESTClient.setResources() -- *** NOT IMPLEMENTED *** "); + } + + @Override + public RangerTaggedResource setResource(RangerTaggedResourceKey key, List<RangerTaggedResource.RangerResourceTag> tags) throws Exception { + throw new Exception("RangerAdminjersey2RESTClient.setResource() -- *** NOT IMPLEMENTED *** "); + } + + @Override + public RangerTaggedResource updateResourceTags(RangerTaggedResourceKey key, List<RangerTaggedResource.RangerResourceTag> tagsToAdd, + List<RangerTaggedResource.RangerResourceTag> tagsToDelete) throws Exception { + throw new Exception("RangerAdminjersey2RESTClient.updateResourceTags() -- *** NOT IMPLEMENTED *** "); + + } + // package level methods left so (and not private only for testability!) Not intended for use outside this class!! Gson getGson() { return new GsonBuilder() http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c930d563/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java index b529a22..8b2bde2 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java @@ -44,6 +44,7 @@ import org.apache.ranger.common.RangerConstants; import org.apache.ranger.common.RangerFactory; import org.apache.ranger.common.StringUtil; import org.apache.ranger.common.UserSessionBase; +import org.apache.ranger.common.RangerAdminConfigPropertyRepository; import org.apache.ranger.db.RangerDaoManager; import org.apache.ranger.db.XXAccessTypeDefDao; import org.apache.ranger.db.XXAccessTypeDefGrantsDao; @@ -209,6 +210,8 @@ public class ServiceDBStore extends AbstractServiceStore { if(! legacyServiceDefsInitDone) { synchronized(ServiceDBStore.class) { if(!legacyServiceDefsInitDone) { + + RangerAdminConfigPropertyRepository.getInstance(); TransactionTemplate txTemplate = new TransactionTemplate(txManager); final ServiceDBStore dbStore = this; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c930d563/security-admin/src/main/java/org/apache/ranger/biz/ServiceMgr.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/biz/ServiceMgr.java b/security-admin/src/main/java/org/apache/ranger/biz/ServiceMgr.java index 218eee3..641ab91 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/ServiceMgr.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/ServiceMgr.java @@ -40,6 +40,7 @@ import org.apache.ranger.plugin.model.RangerService; import org.apache.ranger.plugin.model.RangerServiceDef; import org.apache.ranger.plugin.service.RangerBaseService; import org.apache.ranger.plugin.service.ResourceLookupContext; +import org.apache.ranger.plugin.store.EmbeddedServiceDefsUtil; import org.apache.ranger.plugin.store.ServiceStore; import org.apache.ranger.service.RangerServiceService; import org.apache.ranger.view.VXMessage; @@ -77,9 +78,13 @@ public class ServiceMgr { } if(svc != null) { - LookupCallable callable = new LookupCallable(svc, context); - long time = getTimeoutValueForLookupInMilliSeconds(svc); - ret = timedExecutor.timedTask(callable, time, TimeUnit.MILLISECONDS); + if (StringUtils.equals(svc.getServiceDef().getName(), EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME)) { + ret = svc.lookupResource(context); + } else { + LookupCallable callable = new LookupCallable(svc, context); + long time = getTimeoutValueForLookupInMilliSeconds(svc); + ret = timedExecutor.timedTask(callable, time, TimeUnit.MILLISECONDS); + } } if(LOG.isDebugEnabled()) {
