Repository: incubator-ranger Updated Branches: refs/heads/tag-policy 5876e2272 -> f8bea62bc
RANGER-274: Fixed minor bugs, potential NPE, REST resource path. Fixed service creation Signed-off-by: Madhan Neethiraj <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/f8bea62b Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/f8bea62b Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/f8bea62b Branch: refs/heads/tag-policy Commit: f8bea62bccbca1eaf16083dec4cf1b43ac9b439d Parents: 5876e22 Author: Abhay Kulkarni <[email protected]> Authored: Fri Aug 28 15:31:02 2015 -0700 Committer: Madhan Neethiraj <[email protected]> Committed: Fri Aug 28 18:11:08 2015 -0700 ---------------------------------------------------------------------- .../admin/client/RangerAdminRESTClient.java | 3 +- .../RangerAdminTagRetriever.java | 24 +++----- .../contextenricher/RangerTagProvider.java | 7 +-- .../ranger/plugin/store/file/TagFileStore.java | 8 +-- .../ranger/plugin/util/RangerRESTUtils.java | 4 +- .../ranger/plugin/store/TestTagStore.java | 64 ++++++++++---------- .../org/apache/ranger/biz/ServiceDBStore.java | 1 + .../apache/ranger/rest/TagRESTConstants.java | 4 +- .../service/RangerServiceServiceBase.java | 4 ++ 9 files changed, 58 insertions(+), 61 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/f8bea62b/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java b/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java index 5ec7996..de138ed 100644 --- a/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java +++ b/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java @@ -204,8 +204,7 @@ public class RangerAdminRESTClient implements RangerAdminClient { ServiceTags ret; - WebResource webResource = createWebResource(RangerRESTUtils.REST_URL_GET_SERVICE_TAGS_IF_UPDATED) - .queryParam(RangerRESTUtils.SERVICE_NAME_PARAM, serviceName) + WebResource webResource = createWebResource(RangerRESTUtils.REST_URL_GET_SERVICE_TAGS_IF_UPDATED + serviceName) .queryParam(RangerRESTUtils.LAST_KNOWN_TAG_VERSION_PARAM, Long.toString(lastKnownVersion)) .queryParam(RangerRESTUtils.REST_PARAM_PLUGIN_ID, pluginId); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/f8bea62b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAdminTagRetriever.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAdminTagRetriever.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAdminTagRetriever.java index 987ed45..889fbbc 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAdminTagRetriever.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAdminTagRetriever.java @@ -69,29 +69,25 @@ public class RangerAdminTagRetriever extends RangerTagRefresher { @Override public void retrieveTags() { - if (adminClient != null) { + if (adminClient != null && receiver != null) { ServiceTags serviceTags = null; - long savedLastKnownVersion = lastKnownVersion; try { serviceTags = adminClient.getServiceTagsIfUpdated(lastKnownVersion); - lastKnownVersion = serviceTags.getTagVersion(); } catch (Exception exp) { - LOG.error("RangerAdminTagRetriever.retrieveTags() - Error retrieving resources"); + LOG.error("RangerAdminTagRetriever.retrieveTags() - Error retrieving resources, exception=", exp); } - if (receiver != null && serviceTags != null) { - if (serviceTags.getTagVersion() != null && serviceTags.getTagVersion().longValue() > savedLastKnownVersion) { - receiver.setServiceTags(serviceTags); - } else { - if (LOG.isDebugEnabled()) { - LOG.debug("RangerAdminTagRetriever.retrieveTags() - no updates to tags !!"); - } - } + if (serviceTags != null) { + LOG.info("RangerAdminTagRetriever.retrieveTags() - Updating tags-cache to new version of tags, lastKnownVersion=" + lastKnownVersion + "; newVersion=" + serviceTags.getTagVersion()); + lastKnownVersion = serviceTags.getTagVersion(); + receiver.setServiceTags(serviceTags); } else { - LOG.error("RangerAdminTagRetriever.retrieveTags() - No receiver to send resources to "); + if (LOG.isDebugEnabled()) { + LOG.debug("RangerAdminTagRetriever.retrieveTags() - No need to update tags-cache. lastKnownVersion=" + lastKnownVersion); + } } } else { - LOG.error("RangerAdminTagRetriever.retrieveTags() - No Tag Provider ..."); + LOG.error("RangerAdminTagRetriever.retrieveTags() - No admin client to get tags from or no tag receiver to update tag-cache"); } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/f8bea62b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagProvider.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagProvider.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagProvider.java index f78515c..83600c9 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagProvider.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagProvider.java @@ -161,17 +161,14 @@ public class RangerTagProvider extends RangerAbstractContextEnricher implements for (RangerServiceResourceMatcher resourceMatcher : resourceMatchers) { - RangerServiceResource serviceResource = resourceMatcher.getServiceResource(); - RangerPolicyResourceMatcher matcher = resourceMatcher.getPolicyResourceMatcher(); - - boolean matchResult = matcher.isMatch(resource); + boolean matchResult = resourceMatcher.isMatch(resource); if (matchResult) { if (ret == null) { ret = new ArrayList<RangerTag>(); } // Find tags from serviceResource - ret.addAll(getTagsForServiceResource(serviceTags, serviceResource)); + ret.addAll(getTagsForServiceResource(serviceTags, resourceMatcher.getServiceResource())); } } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/f8bea62b/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java index 6117034..0b4f0ca 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java @@ -1026,6 +1026,7 @@ public class TagFileStore extends AbstractTagStore { filter.setParam(SearchFilter.TAG_RESOURCE_SERVICE_NAME, serviceName); List<RangerServiceResource> serviceResources = getServiceResources(filter); + List<RangerServiceResource> filteredServiceResources = new ArrayList<RangerServiceResource>(); Map<Long, RangerTag> tagsMap = new HashMap<Long, RangerTag>(); Map<Long, List<Long>> resourceToTagIdsMap = new HashMap<Long, List<Long>>(); @@ -1040,14 +1041,11 @@ public class TagFileStore extends AbstractTagStore { tagIdList.add(tag.getId()); } resourceToTagIdsMap.put(serviceResource.getId(), tagIdList); + filteredServiceResources.add(serviceResource); } } - if (MapUtils.isEmpty(resourceToTagIdsMap)) { - serviceResources.clear(); - } - - ret.setServiceResources(serviceResources); + ret.setServiceResources(filteredServiceResources); ret.setResourceToTagIds(resourceToTagIdsMap); ret.setTags(tagsMap); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/f8bea62b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRESTUtils.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRESTUtils.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRESTUtils.java index e162c5c..ad113fe 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRESTUtils.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRESTUtils.java @@ -40,9 +40,9 @@ public class RangerRESTUtils { public static final String REST_URL_SERVICE_GRANT_ACCESS = "/service/plugins/services/grant/"; public static final String REST_URL_SERVICE_REVOKE_ACCESS = "/service/plugins/services/revoke/"; - public static final String REST_URL_GET_SERVICE_TAGS_IF_UPDATED = "/service/tags/download"; + public static final String REST_URL_GET_SERVICE_TAGS_IF_UPDATED = "/service/tags/download/"; public static final String SERVICE_NAME_PARAM = "serviceName"; - public static final String LAST_KNOWN_TAG_VERSION_PARAM = "tagVersion"; + public static final String LAST_KNOWN_TAG_VERSION_PARAM = "lastKnownVersion"; public static final String PATTERN_PARAM = "pattern"; public static final String REST_URL_LOOKUP_TAG_NAMES = "/service/tags/lookup"; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/f8bea62b/agents-common/src/test/java/org/apache/ranger/plugin/store/TestTagStore.java ---------------------------------------------------------------------- diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/store/TestTagStore.java b/agents-common/src/test/java/org/apache/ranger/plugin/store/TestTagStore.java index e0f4865..5424158 100644 --- a/agents-common/src/test/java/org/apache/ranger/plugin/store/TestTagStore.java +++ b/agents-common/src/test/java/org/apache/ranger/plugin/store/TestTagStore.java @@ -40,68 +40,66 @@ import org.apache.ranger.plugin.store.file.TagFileStore; import org.apache.ranger.plugin.util.SearchFilter; import org.apache.ranger.plugin.util.ServiceTags; import org.junit.BeforeClass; +import org.junit.AfterClass; import org.junit.Test; public class TestTagStore { static TagStore tagStore = null; static TagValidator validator = null; - + static RangerServiceDef serviceDef = null; + static RangerService service = null; static SearchFilter filter = null; - static Path filePath = new Path("file:///etc/ranger/data/ranger-admin-test-site.xml"); + + static String tmpDir = null; + static Path filePath = null; + static Configuration config = new Configuration(); static final String serviceDefJsonFile = "/admin/service-defs/test-hive-servicedef.json"; static final String serviceName = "tag-unit-test-TestTagStore"; - static final String crcSuffix = ".crc"; - static final String jsonSuffix = ".json"; - static Gson gsonBuilder = null; - static RangerServiceDef serviceDef = null; - static RangerService service = null; @BeforeClass public static void setupTest() throws Exception { - tearDownAfterClass(crcSuffix); - tearDownAfterClass(jsonSuffix); + tmpDir = "file://" + System.getProperty("java.io.tmpdir"); - FileSystem fs = filePath.getFileSystem(config); - - FSDataOutputStream outStream = fs.create(filePath, true); - OutputStreamWriter writer = null; - - writer = new OutputStreamWriter(outStream); - - writer.write("<configuration>\n" + + String textTemplate = "<configuration>\n" + " <property>\n" + " <name>ranger.tag.store.file.dir</name>\n" + - " <value>file:///etc/ranger/data</value>\n" + + " <value>%s</value>\n" + " </property>\n" + " <property>\n" + " <name>ranger.service.store.file.dir</name>\n" + - " <value>file:///etc/ranger/data</value>\n" + + " <value>%s</value>\n" + " </property>\n" + - "</configuration>\n"); + "</configuration>\n"; + + String text = String.format(textTemplate, tmpDir, tmpDir); + String fileName = tmpDir + "/ranger-admin-test-site.xml"; + filePath = new Path(fileName); + FileSystem fs = filePath.getFileSystem(config); + + FSDataOutputStream outStream = fs.create(filePath, true); + + OutputStreamWriter writer = new OutputStreamWriter(outStream); + writer.write(text); writer.close(); RangerConfiguration config = RangerConfiguration.getInstance(); config.addResource(filePath); - tagStore = TagFileStore.getInstance(); - tagStore.init(); - - ServiceStore svcStore; - - svcStore = new ServiceFileStore(); + ServiceStore svcStore = new ServiceFileStore(); svcStore.init(); + tagStore = TagFileStore.getInstance(); + tagStore.init(); tagStore.setServiceStore(svcStore); validator = new TagValidator(); - validator.setTagStore(tagStore); gsonBuilder = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z") @@ -110,17 +108,20 @@ public class TestTagStore { InputStream inStream = TestTagStore.class.getResourceAsStream(serviceDefJsonFile); InputStreamReader reader = new InputStreamReader(inStream); + serviceDef = gsonBuilder.fromJson(reader, RangerServiceDef.class); + service = svcStore.createService(new RangerService(serviceDef.getName(), serviceName, serviceName, null, null)); + reader.close(); inStream.close(); } - //@AfterClass - public static void tearDownAfterClass(String suffix) throws Exception { + @AfterClass + public static void tearDownAfterClass() throws Exception { - Path dirPath = new Path("file:///etc/ranger/data"); + Path dirPath = new Path(tmpDir); FileSystem fs = dirPath.getFileSystem(config); try { @@ -132,7 +133,7 @@ public class TestTagStore { while (files.hasNext()) { LocatedFileStatus fileStatus = files.next(); Path path = fileStatus.getPath(); - if (fs.isFile(path) && path.getName().endsWith(suffix)) { + if (fs.isFile(path)) { fs.delete(path, true); } } @@ -142,6 +143,7 @@ public class TestTagStore { } fs.delete(filePath, true); + } @Test http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/f8bea62b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java index 1fec611..a4765ac 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java @@ -1027,6 +1027,7 @@ public class ServiceDBStore extends AbstractServiceStore { // While creating, value of version should be 1. service.setVersion(new Long(1)); + service.setTagVersion(new Long(1)); if (populateExistingBaseFields) { svcServiceWithAssignedId.setPopulateExistingBaseFields(true); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/f8bea62b/security-admin/src/main/java/org/apache/ranger/rest/TagRESTConstants.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/rest/TagRESTConstants.java b/security-admin/src/main/java/org/apache/ranger/rest/TagRESTConstants.java index 07a559c..91525df 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/TagRESTConstants.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/TagRESTConstants.java @@ -29,8 +29,8 @@ public class TagRESTConstants { static final String RESOURCES_RESOURCE = "/resources/"; static final String RESOURCE_RESOURCE = "/resource/"; static final String TAGRESOURCEMAPS_RESOURCE = "/tagresourcemaps/"; - static final String TAGNAMES_RESOURCE = "/tags/names/"; - static final String LOOKUP_TAGS_RESOURCE = "/tags/lookup/"; + static final String TAGNAMES_RESOURCE = "/names/"; + static final String LOOKUP_TAGS_RESOURCE = "/lookup/"; static final String TAGS_DOWNLOAD = "/download/"; public static final String SERVICE_NAME_PARAM = "serviceName"; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/f8bea62b/security-admin/src/main/java/org/apache/ranger/service/RangerServiceServiceBase.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/service/RangerServiceServiceBase.java b/security-admin/src/main/java/org/apache/ranger/service/RangerServiceServiceBase.java index e594bb5..3cbeaf0 100755 --- a/security-admin/src/main/java/org/apache/ranger/service/RangerServiceServiceBase.java +++ b/security-admin/src/main/java/org/apache/ranger/service/RangerServiceServiceBase.java @@ -95,7 +95,9 @@ public abstract class RangerServiceServiceBase<T extends XXServiceBase, V extend xObj.setName(vObj.getName()); xObj.setTagService(tagServiceId); xObj.setPolicyVersion(vObj.getPolicyVersion()); + xObj.setTagVersion(vObj.getTagVersion()); xObj.setPolicyUpdateTime(vObj.getPolicyUpdateTime()); + xObj.setTagUpdateTime(vObj.getTagUpdateTime()); xObj.setDescription(vObj.getDescription()); xObj.setIsEnabled(vObj.getIsEnabled()); return xObj; @@ -113,7 +115,9 @@ public abstract class RangerServiceServiceBase<T extends XXServiceBase, V extend vObj.setDescription(xObj.getDescription()); vObj.setTagService(xTagService != null ? xTagService.getName() : null); vObj.setPolicyVersion(xObj.getPolicyVersion()); + vObj.setTagVersion(xObj.getTagVersion()); vObj.setPolicyUpdateTime(xObj.getPolicyUpdateTime()); + vObj.setTagUpdateTime(xObj.getTagUpdateTime()); vObj.setIsEnabled(xObj.getIsenabled()); return vObj; }
