RANGER-660: implement TagSync process to populate Ranger with tag details Signed-off-by: Madhan Neethiraj <[email protected]>
Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/eb874038 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/eb874038 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/eb874038 Branch: refs/heads/tag-policy Commit: eb8740380fc53a353ddfc239455c5e9254476056 Parents: 9c42a8a Author: Abhay Kulkarni <[email protected]> Authored: Thu Sep 10 17:49:16 2015 -0700 Committer: Madhan Neethiraj <[email protected]> Committed: Mon Sep 28 09:57:15 2015 -0700 ---------------------------------------------------------------------- .../plugin/model/RangerTagResourceMap.java | 4 +- .../apache/ranger/plugin/store/TagStore.java | 2 + .../ranger/plugin/store/TagValidator.java | 169 ++++-- .../ranger/plugin/store/file/TagFileStore.java | 26 +- .../apache/ranger/plugin/util/ServiceTags.java | 18 + .../atlas-client-0.6-incubating-SNAPSHOT.jar | Bin 0 -> 34445 bytes ...las-notification-0.6-incubating-SNAPSHOT.jar | Bin 0 -> 34596 bytes ...atlas-typesystem-0.6-incubating-SNAPSHOT.jar | Bin 0 -> 355378 bytes pom.xml | 7 + .../java/org/apache/ranger/biz/TagDBStore.java | 16 + .../apache/ranger/db/XXTagResourceMapDao.java | 9 + .../ranger/rest/ServiceTagsProcessor.java | 370 ++++++++++++ .../java/org/apache/ranger/rest/TagREST.java | 117 +++- .../apache/ranger/rest/TagRESTConstants.java | 1 + .../service/RangerTagResourceMapService.java | 6 + .../resources/META-INF/jpa_named_queries.xml | 6 + src/main/assembly/tagsync.xml | 48 +- tagsync/conf.dist/log4j.xml | 4 +- .../conf.dist/ranger-tagsync-default-site.xml | 36 -- tagsync/conf.dist/ranger-tagsync-site.xml | 67 +++ tagsync/pom.xml | 8 +- tagsync/samples/tags.json | 55 ++ tagsync/scripts/ranger-tagsync-services.sh | 22 +- .../java/org/apache/ranger/model/TagSink.java | 3 - .../java/org/apache/ranger/model/TagSource.java | 13 +- .../apache/ranger/process/TagSyncConfig.java | 132 ++--- .../apache/ranger/process/TagSynchronizer.java | 60 +- .../ranger/sink/policymgr/TagRESTSink.java | 14 +- .../source/atlas/AtlasNotificationMapper.java | 391 ++++++++++++ .../ranger/source/atlas/TagAtlasSource.java | 588 +++++++++++++++++++ .../ranger/source/file/TagFileSource.java | 183 +----- .../main/resources/ranger-tagsync-default.xml | 91 +++ .../src/main/resources/ranger-tagsync-site.xml | 67 --- .../ranger/process/TestTagSynchronizer.java | 17 +- 34 files changed, 2024 insertions(+), 526 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerTagResourceMap.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerTagResourceMap.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerTagResourceMap.java index 3b69ee7..3ee0b3e 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerTagResourceMap.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerTagResourceMap.java @@ -71,9 +71,9 @@ public class RangerTagResourceMap extends RangerBaseModelObject { sb.append("{ "); - sb.append("tagId=").append(tagId).append("} "); + sb.append("resourceId=").append(resourceId).append(", "); - sb.append("resourceId={").append(resourceId).append("} "); + sb.append("tagId=").append(tagId); sb.append(" }"); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java index 1c48ea0..ed20c51 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java @@ -66,6 +66,8 @@ public interface TagStore { RangerTag getTagByGuid(String guid) throws Exception; + List<Long> getTagIdsForResourceId(Long resourceId) throws Exception; + List<RangerTag> getTagsByType(String name) throws Exception; List<RangerTag> getTagsForResourceId(Long resourceId) throws Exception; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/agents-common/src/main/java/org/apache/ranger/plugin/store/TagValidator.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/TagValidator.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/TagValidator.java index 4eda166..c334e39 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/TagValidator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/TagValidator.java @@ -35,10 +35,31 @@ public class TagValidator { this.tagStore = tagStore; } - public void preCreateTag(final RangerTag tag) throws Exception { - if (StringUtils.isBlank(tag.getType())) { + public RangerTagDef preCreateTagDef(final RangerTagDef tagDef, boolean updateIfExists) throws Exception { + String name = tagDef.getName(); + + if (StringUtils.isBlank(name)) { + throw new Exception("TagDef has no name"); + } + + RangerTagDef existing = tagStore.getTagDefByName(name); + + return existing; + } + + public RangerTag preCreateTag(final RangerTag tag) throws Exception { + if(StringUtils.isBlank(tag.getType()) ) { throw new Exception("Tag has no type"); } + + RangerTag ret = null; + + String guid = tag.getGuid(); + if (! StringUtils.isBlank(guid)) { + ret = tagStore.getTagByGuid(guid); + } + + return ret; } public void preUpdateTag(final Long id, final RangerTag tag) throws Exception { @@ -50,12 +71,14 @@ public class TagValidator { throw new Exception("Invalid/null id"); } - RangerTag exist = tagStore.getTag(id); + RangerTag existing = tagStore.getTag(id); - if (exist == null) { + if (existing == null) { throw new Exception("Attempt to update nonexistant tag, id=" + id); } - tag.setId(exist.getId()); + + tag.setId(existing.getId()); + tag.setGuid(existing.getGuid()); } public void preUpdateTagByGuid(String guid, final RangerTag tag) throws Exception { @@ -72,39 +95,27 @@ public class TagValidator { tag.setGuid(existing.getGuid()); } - public void preUpdateTagByName(String name, final RangerTag tag) throws Exception { - if (StringUtils.isNotBlank(tag.getType())) { - throw new Exception("tag has no type"); - } - - List<RangerTag> exist = tagStore.getTagsByType(name); - if (CollectionUtils.isEmpty(exist) || CollectionUtils.size(exist) != 1) { - throw new Exception("Attempt to update nonexistent or multiple tags, name=" + name); + public RangerTag preDeleteTag(Long id) throws Exception { + if (id == null) { + throw new Exception("Invalid/null id"); } - RangerTag onlyTag = exist.get(0); + RangerTag existing = tagStore.getTag(id); - tag.setId(onlyTag.getId()); - tag.setType(name); - - } - - public RangerTag preDeleteTag(Long id) throws Exception { - RangerTag exist; - exist = tagStore.getTag(id); - if (exist == null) { + if (existing == null) { throw new Exception("Attempt to delete nonexistent tag, id=" + id); } - List<RangerTagResourceMap> associations = tagStore.getTagResourceMapsForTagId(exist.getId()); + List<RangerTagResourceMap> associations = tagStore.getTagResourceMapsForTagId(existing.getId()); if (CollectionUtils.isNotEmpty(associations)) { throw new Exception("Attempt to delete tag which is associated with a service-resource, id=" + id); } - return exist; + return existing; } public RangerTag preDeleteTagByGuid(String guid) throws Exception { RangerTag exiting = tagStore.getTagByGuid(guid); + if (exiting == null) { throw new Exception("Attempt to delete nonexistent tag, guid=" + guid); } @@ -116,37 +127,23 @@ public class TagValidator { return exiting; } - public RangerTag preDeleteTagByName(String name) throws Exception { - List<RangerTag> exist; - exist = tagStore.getTagsByType(name); - if (CollectionUtils.isEmpty(exist) || CollectionUtils.size(exist) != 1) { - throw new Exception("Attempt to delete nonexistent or multiple tags, name=" + name); - } - RangerTag ret = exist.get(0); - List<RangerTagResourceMap> associations = tagStore.getTagResourceMapsForTagId(ret.getId()); - if (CollectionUtils.isNotEmpty(associations)) { - throw new Exception("Attempt to delete tag which is associated with a service-resource, name=" + name); - } - return ret; - - } + public RangerServiceResource preCreateServiceResource(RangerServiceResource resource) throws Exception { + RangerServiceResource ret = null; - public void preCreateServiceResource(RangerServiceResource resource) throws Exception { if (StringUtils.isBlank(resource.getServiceName()) || MapUtils.isEmpty(resource.getResourceElements())) { throw new Exception("No serviceName or resource in RangerServiceResource"); } - RangerServiceResourceSignature serializer = new RangerServiceResourceSignature(resource); - - String resourceSignature = serializer.getSignature(); + String guid = resource.getGuid(); + if (! StringUtils.isBlank(guid)) { + ret = tagStore.getServiceResourceByGuid(guid); + } - RangerServiceResource exist = tagStore.getServiceResourceByResourceSignature(resourceSignature); + RangerServiceResourceSignature serializer = new RangerServiceResourceSignature(resource); - if (exist != null) { - throw new Exception("Attempt to create existing resource, serviceName=" + resource.getServiceName()); - } + resource.setResourceSignature(serializer.getSignature()); - resource.setResourceSignature(resourceSignature); + return ret; } public void preUpdateServiceResource(Long id, RangerServiceResource resource) throws Exception { @@ -158,15 +155,16 @@ public class TagValidator { throw new Exception("Invalid/null id"); } - RangerServiceResource exist = tagStore.getServiceResource(id); - if (exist == null) { + RangerServiceResource existing = tagStore.getServiceResource(id); + if (existing == null) { throw new Exception("Attempt to update nonexistent resource, id=" + id); } - resource.setId(exist.getId()); RangerServiceResourceSignature serializer = new RangerServiceResourceSignature(resource); - resource.setResourceSignature(serializer.getSignature()); + resource.setId(existing.getId()); + resource.setGuid(existing.getGuid()); + resource.setResourceSignature(serializer.getSignature()); } public void preUpdateServiceResourceByGuid(String guid, RangerServiceResource resource) throws Exception { @@ -179,35 +177,40 @@ public class TagValidator { throw new Exception("Attempt to update nonexistent resource, guid=" + guid); } + RangerServiceResourceSignature serializer = new RangerServiceResourceSignature(resource); + resource.setId(existing.getId()); resource.setGuid(guid); - - RangerServiceResourceSignature serializer = new RangerServiceResourceSignature(resource); resource.setResourceSignature(serializer.getSignature()); } public RangerServiceResource preDeleteServiceResource(Long id) throws Exception { - RangerServiceResource exist; - exist = tagStore.getServiceResource(id); - if (exist == null) { + RangerServiceResource existing = tagStore.getServiceResource(id); + + if (existing == null) { throw new Exception("Attempt to delete nonexistent resource, id=" + id); } - List<RangerTagResourceMap> associations = tagStore.getTagResourceMapsForResourceId(exist.getId()); + + List<RangerTagResourceMap> associations = tagStore.getTagResourceMapsForResourceId(existing.getId()); if (CollectionUtils.isNotEmpty(associations)) { throw new Exception("Attempt to delete serviceResource which is associated with a tag, id=" + id); } - return exist; + + return existing; } - public RangerServiceResource preDeleteServiceResourceByGuid(String guid) throws Exception { + public RangerServiceResource preDeleteServiceResourceByGuid(String guid, boolean deleteReferences) throws Exception { RangerServiceResource existing = tagStore.getServiceResourceByGuid(guid); + if (existing == null) { throw new Exception("Attempt to delete nonexistent resource, guid=" + guid); } + List<RangerTagResourceMap> associations = tagStore.getTagResourceMapsForResourceId(existing.getId()); - if (CollectionUtils.isNotEmpty(associations)) { + if (CollectionUtils.isNotEmpty(associations) && !deleteReferences) { throw new Exception("Attempt to delete serviceResource which is associated with a tag, guid=" + guid); } + return existing; } @@ -216,8 +219,9 @@ public class TagValidator { throw new Exception("Both resourceGuid and resourceId need to be non-empty"); } - RangerTagResourceMap exist = tagStore.getTagResourceMapForTagAndResourceGuid(tagGuid, resourceGuid); - if (exist != null) { + RangerTagResourceMap existing = tagStore.getTagResourceMapForTagAndResourceGuid(tagGuid, resourceGuid); + + if (existing != null) { throw new Exception("Attempt to create existing association between resourceId=" + resourceGuid + " and tagId=" + tagGuid); } @@ -240,8 +244,35 @@ public class TagValidator { return newTagResourceMap; } + public RangerTagResourceMap preCreateTagResourceMapByIds(Long tagId, Long resourceId) throws Exception { + RangerTagResourceMap existing = tagStore.getTagResourceMapForTagAndResourceId(tagId, resourceId); + + if (existing != null) { + throw new Exception("Attempt to create existing association between resourceId=" + resourceId + " and tagId=" + tagId); + } + + RangerServiceResource existingServiceResource = tagStore.getServiceResource(resourceId); + + if(existingServiceResource == null) { + throw new Exception("No resource found for id=" + resourceId); + } + + RangerTag existingTag = tagStore.getTag(tagId); + + if(existingTag == null) { + throw new Exception("No tag found for id=" + tagId); + } + + RangerTagResourceMap newTagResourceMap = new RangerTagResourceMap(); + newTagResourceMap.setResourceId(resourceId); + newTagResourceMap.setTagId(tagId); + + return newTagResourceMap; + } + public RangerTagResourceMap preDeleteTagResourceMap(Long id) throws Exception { RangerTagResourceMap existing = tagStore.getTagResourceMap(id); + if (existing == null) { throw new Exception("Attempt to delete nonexistent tagResourceMap(id=" + id + ")"); } @@ -251,6 +282,7 @@ public class TagValidator { public RangerTagResourceMap preDeleteTagResourceMapByGuid(String guid) throws Exception { RangerTagResourceMap existing = tagStore.getTagResourceMapByGuid(guid); + if (existing == null) { throw new Exception("Attempt to delete nonexistent tagResourceMap(guid=" + guid + ")"); } @@ -260,10 +292,21 @@ public class TagValidator { public RangerTagResourceMap preDeleteTagResourceMap(String tagGuid, String resourceGuid) throws Exception { RangerTagResourceMap existing = tagStore.getTagResourceMapForTagAndResourceGuid(tagGuid, resourceGuid); + if (existing == null) { throw new Exception("Attempt to delete nonexistent association between resourceId=" + resourceGuid + " and tagId=" + tagGuid); } return existing; } + + public RangerTagResourceMap preDeleteTagResourceMapByIds(Long tagId, Long resourceId) throws Exception { + RangerTagResourceMap existing = tagStore.getTagResourceMapForTagAndResourceId(tagId, resourceId); + + if (existing == null) { + throw new Exception("Attempt to delete nonexistent association between resourceId=" + resourceId + " and tagId=" + tagId); + } + + return existing; + } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java index 27b4779..8f94164 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java @@ -31,6 +31,7 @@ import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; import org.apache.ranger.plugin.model.*; import org.apache.ranger.plugin.store.AbstractTagStore; import org.apache.ranger.plugin.store.PList; +import org.apache.ranger.plugin.store.RangerServiceResourceSignature; import org.apache.ranger.plugin.store.TagPredicateUtil; import org.apache.ranger.plugin.store.TagStore; import org.apache.ranger.plugin.util.SearchFilter; @@ -42,6 +43,7 @@ public class TagFileStore extends AbstractTagStore { public static final String PROPERTY_TAG_FILE_STORE_DIR = "ranger.tag.store.file.dir"; + protected static final String FILE_PREFIX_TAG_DEF = "ranger-tagdef-"; protected static final String FILE_PREFIX_TAG = "ranger-tag-"; protected static final String FILE_PREFIX_RESOURCE = "ranger-serviceresource-"; @@ -116,12 +118,6 @@ public class TagFileStore extends AbstractTagStore { LOG.debug("==> TagFileStore.createTagDef(" + tagDef + ")"); } - RangerTagDef existing = getTagDefByName(tagDef.getName()); - - if (existing != null) { - throw new Exception(tagDef.getName() + ": tag-def already exists (id=" + existing.getId() + ")"); - } - RangerTagDef ret = null; try { @@ -160,7 +156,7 @@ public class TagFileStore extends AbstractTagStore { throw new Exception("tag-def does not exist: name=" + tagDef.getName()); } } else { - existing = this.getTagDef(tagDef.getId()); + existing = getTagDef(tagDef.getId()); if (existing == null) { throw new Exception("tag-def does not exist: id=" + tagDef.getId()); @@ -504,6 +500,21 @@ public class TagFileStore extends AbstractTagStore { } @Override + public List<Long> getTagIdsForResourceId(Long resourceId) throws Exception { + List<Long> ret = new ArrayList<Long>(); + + List<RangerTag> tags = getTagsForResourceId(resourceId); + + if(CollectionUtils.isNotEmpty(tags)) { + for(RangerTag tag : tags) { + ret.add(tag.getId()); + } + } + + return ret; + } + + @Override public List<RangerTag> getTagsForResourceId(Long resourceId) throws Exception { if (LOG.isDebugEnabled()) { LOG.debug("==> TagFileStore.getTagsForResourceId(" + resourceId + ")"); @@ -1063,6 +1074,7 @@ public class TagFileStore extends AbstractTagStore { } ServiceTags ret = new ServiceTags(); + ret.setOp(ServiceTags.OP_ADD_OR_UPDATE); boolean tagsChanged = true; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceTags.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceTags.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceTags.java index e56e942..146c3f9 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceTags.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceTags.java @@ -44,6 +44,10 @@ import org.codehaus.jackson.map.annotate.JsonSerialize; public class ServiceTags implements java.io.Serializable { private static final long serialVersionUID = 1L; + public static final String OP_ADD_OR_UPDATE = "add_or_update"; + public static final String OP_DELETE = "delete"; + + private String op; private String serviceName; private Long tagVersion; private Date tagUpdateTime; @@ -53,6 +57,13 @@ public class ServiceTags implements java.io.Serializable { private Map<Long, List<Long>> resourceToTagIds; /** + * @return the op + */ + public String getOp() { + return op; + } + + /** * @return the serviceName */ public String getServiceName() { @@ -60,6 +71,13 @@ public class ServiceTags implements java.io.Serializable { } /** + * @param op the op to set + */ + public void setOp(String op) { + this.op = op; + } + + /** * @param serviceName the serviceName to set */ public void setServiceName(String serviceName) { http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/lib/local/apache-atlas-0.6-incubating-SNAPSHOT/atlas-client-0.6-incubating-SNAPSHOT.jar ---------------------------------------------------------------------- diff --git a/lib/local/apache-atlas-0.6-incubating-SNAPSHOT/atlas-client-0.6-incubating-SNAPSHOT.jar b/lib/local/apache-atlas-0.6-incubating-SNAPSHOT/atlas-client-0.6-incubating-SNAPSHOT.jar new file mode 100644 index 0000000..feaaa38 Binary files /dev/null and b/lib/local/apache-atlas-0.6-incubating-SNAPSHOT/atlas-client-0.6-incubating-SNAPSHOT.jar differ http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/lib/local/apache-atlas-0.6-incubating-SNAPSHOT/atlas-notification-0.6-incubating-SNAPSHOT.jar ---------------------------------------------------------------------- diff --git a/lib/local/apache-atlas-0.6-incubating-SNAPSHOT/atlas-notification-0.6-incubating-SNAPSHOT.jar b/lib/local/apache-atlas-0.6-incubating-SNAPSHOT/atlas-notification-0.6-incubating-SNAPSHOT.jar new file mode 100644 index 0000000..3c2ecb6 Binary files /dev/null and b/lib/local/apache-atlas-0.6-incubating-SNAPSHOT/atlas-notification-0.6-incubating-SNAPSHOT.jar differ http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/lib/local/apache-atlas-0.6-incubating-SNAPSHOT/atlas-typesystem-0.6-incubating-SNAPSHOT.jar ---------------------------------------------------------------------- diff --git a/lib/local/apache-atlas-0.6-incubating-SNAPSHOT/atlas-typesystem-0.6-incubating-SNAPSHOT.jar b/lib/local/apache-atlas-0.6-incubating-SNAPSHOT/atlas-typesystem-0.6-incubating-SNAPSHOT.jar new file mode 100644 index 0000000..8e5d1bf Binary files /dev/null and b/lib/local/apache-atlas-0.6-incubating-SNAPSHOT/atlas-typesystem-0.6-incubating-SNAPSHOT.jar differ http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index aa40882..002885a 100644 --- a/pom.xml +++ b/pom.xml @@ -255,6 +255,13 @@ <enabled>false</enabled> </snapshots> </repository> + <repository> + <id>repo</id> + <url>file://${user.home}/local-repo</url> + <snapshots> + <enabled>true</enabled> + </snapshots> + </repository> </repositories> <dependencyManagement> <dependencies> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java index 962bfde..98a6e87 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java @@ -719,6 +719,21 @@ public class TagDBStore extends AbstractTagStore { } @Override + public List<Long> getTagIdsForResourceId(Long resourceId) throws Exception { + if (LOG.isDebugEnabled()) { + LOG.debug("==> TagDBStore.getTagIdsForResourceId(" + resourceId + ")"); + } + + List<Long> ret = rangerTagResourceMapService.getTagIdsForResourceId(resourceId); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== TagDBStore.getTagIdsForResourceId(" + resourceId + "): count=" + (ret == null ? 0 : ret.size())); + } + + return ret; + } + + @Override public List<RangerTagResourceMap> getTagResourceMapsForResourceId(Long resourceId) throws Exception { if (LOG.isDebugEnabled()) { LOG.debug("==> TagDBStore.getTagResourceMapsForResourceId(" + resourceId + ")"); @@ -875,6 +890,7 @@ public class TagDBStore extends AbstractTagStore { } ret = new ServiceTags(); + ret.setOp(ServiceTags.OP_ADD_OR_UPDATE); ret.setServiceName(xxService.getName()); ret.setTagVersion(xxService.getTagVersion()); ret.setTagUpdateTime(xxService.getTagUpdateTime()); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/security-admin/src/main/java/org/apache/ranger/db/XXTagResourceMapDao.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXTagResourceMapDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXTagResourceMapDao.java index 0cac1ef..72e0cd9 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXTagResourceMapDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXTagResourceMapDao.java @@ -70,6 +70,15 @@ public class XXTagResourceMapDao extends BaseDao<XXTagResourceMap> { } } + public List<Long> findTagIdsForResourceId(Long resourceId) { + try { + return getEntityManager().createNamedQuery("XXTagResourceMap.getTagIdsForResourceId", Long.class) + .setParameter("resourceId", resourceId).getResultList(); + } catch (NoResultException e) { + return new ArrayList<Long>(); + } + } + public List<XXTagResourceMap> findByTagId(Long tagId) { if (tagId == null) { return new ArrayList<XXTagResourceMap>(); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/security-admin/src/main/java/org/apache/ranger/rest/ServiceTagsProcessor.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/rest/ServiceTagsProcessor.java b/security-admin/src/main/java/org/apache/ranger/rest/ServiceTagsProcessor.java new file mode 100644 index 0000000..7f0a92c --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/rest/ServiceTagsProcessor.java @@ -0,0 +1,370 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.rest; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.commons.lang.StringUtils; +import org.apache.ranger.plugin.model.RangerServiceResource; +import org.apache.ranger.plugin.model.RangerTag; +import org.apache.ranger.plugin.model.RangerTagDef; +import org.apache.ranger.plugin.model.RangerTagResourceMap; +import org.apache.ranger.plugin.store.RangerServiceResourceSignature; +import org.apache.ranger.plugin.store.TagStore; +import org.apache.ranger.plugin.util.ServiceTags; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class ServiceTagsProcessor { + private static final Log LOG = LogFactory.getLog(ServiceTagsProcessor.class); + + private final TagStore tagStore; + + public ServiceTagsProcessor(TagStore tagStore) { + this.tagStore = tagStore; + } + + public void process(ServiceTags serviceTags) throws Exception { + if (LOG.isDebugEnabled()) { + LOG.debug("==> ServiceTagsProcessor.process()"); + } + + if (tagStore != null && serviceTags != null) { + String op = serviceTags.getOp(); + + if (StringUtils.equalsIgnoreCase(op, ServiceTags.OP_ADD_OR_UPDATE)) { + addOrUpdate(serviceTags); + } else if (StringUtils.equalsIgnoreCase(op, ServiceTags.OP_DELETE)) { + delete(serviceTags); + } else { + LOG.error("Unknown op, op=" + op); + } + } else { + if(tagStore == null) { + LOG.error("tagStore is null!!"); + } + + if (serviceTags == null) { + LOG.error("No ServiceTags to import!!"); + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== ServiceTagsProcessor.process()"); + } + } + + // Map tagdef, tag, serviceResource ids to created ids and use them in tag-resource-mapping + private void addOrUpdate(ServiceTags serviceTags) throws Exception { + if (LOG.isDebugEnabled()) { + LOG.debug("==> ServiceTagsProcessor.createOrUpdate()"); + } + + Map<Long, RangerTagDef> tagDefsInStore = new HashMap<Long, RangerTagDef>(); + Map<Long, RangerTag> tagsInStore = new HashMap<Long, RangerTag>(); + Map<Long, RangerServiceResource> resourcesInStore = new HashMap<Long, RangerServiceResource>(); + + boolean createOrUpdate = true; + + if (MapUtils.isNotEmpty(serviceTags.getTagDefinitions())) { + RangerTagDef tagDef = null; + + try { + for (Map.Entry<Long, RangerTagDef> entry : serviceTags.getTagDefinitions().entrySet()) { + tagDef = entry.getValue(); + + RangerTagDef existing = null; + + if(createOrUpdate) { + if(StringUtils.isNotEmpty(tagDef.getGuid())) { + existing = tagStore.getTagDefByGuid(tagDef.getGuid()); + } + + if(existing == null && StringUtils.isNotEmpty(tagDef.getName())) { + existing = tagStore.getTagDefByName(tagDef.getName()); + } + } + + RangerTagDef tagDefInStore = null; + + if(existing == null) { + tagDefInStore = tagStore.createTagDef(tagDef); + } else { + tagDef.setId(existing.getId()); + tagDef.setGuid(existing.getGuid()); + + tagDefInStore = tagStore.updateTagDef(tagDef); + } + + if(tagDefsInStore != null) { + tagDefsInStore.put(entry.getKey(), tagDefInStore); + } + } + } catch (Exception exception) { + LOG.error("createTagDef failed, tagDef=" + tagDef, exception); + throw exception; + } + } + + if (MapUtils.isNotEmpty(serviceTags.getTags())) { + RangerTag tag = null; + + try { + for (Map.Entry<Long, RangerTag> entry : serviceTags.getTags().entrySet()) { + tag = entry.getValue(); + + RangerTag existing = null; + + if(createOrUpdate) { + if(StringUtils.isNotEmpty(tag.getGuid())) { + existing = tagStore.getTagByGuid(tag.getGuid()); + } + } + + RangerTag tagInStore = null; + + if(existing == null) { + tagInStore = tagStore.createTag(tag); + } else { + tag.setId(existing.getId()); + tag.setGuid(existing.getGuid()); + + tagInStore = tagStore.updateTag(tag); + } + + if(tagsInStore != null) { + tagsInStore.put(entry.getKey(), tagInStore); + } + } + } catch (Exception exception) { + LOG.error("createTag failed, tag=" + tag, exception); + throw exception; + } + } + + List<RangerServiceResource> resources = serviceTags.getServiceResources(); + if (CollectionUtils.isNotEmpty(resources)) { + RangerServiceResource resource = null; + + try { + for (int i = 0; i < resources.size(); i++) { + resource = resources.get(i); + + RangerServiceResource existing = null; + String resourceSignature = null; + Long resourceId = resource.getId(); + + if(createOrUpdate) { + if(StringUtils.isNotEmpty(resource.getGuid())) { + existing = tagStore.getServiceResourceByGuid(resource.getGuid()); + } + + if(existing == null) { + if(MapUtils.isNotEmpty(resource.getResourceElements())) { + RangerServiceResourceSignature serializer = new RangerServiceResourceSignature(resource); + + resourceSignature = serializer.getSignature(); + + existing = tagStore.getServiceResourceByResourceSignature(resourceSignature); + } + } + + if(existing != null) { + resourceSignature = existing.getResourceSignature(); + } + } + + if(StringUtils.isEmpty(resourceSignature)) { + RangerServiceResourceSignature serializer = new RangerServiceResourceSignature(resource); + + resourceSignature = serializer.getSignature(); + } + + RangerServiceResource resourceInStore = null; + + if (existing == null) { + resource.setResourceSignature(resourceSignature); + + resourceInStore = tagStore.createServiceResource(resource); + } else { + resource.setId(existing.getId()); + resource.setGuid(existing.getGuid()); + resource.setResourceSignature(resourceSignature); + + resourceInStore = tagStore.updateServiceResource(resource); + } + + resourcesInStore.put(resourceId, resourceInStore); + } + } catch (Exception exception) { + LOG.error("createServiceResource failed, resource=" + resource, exception); + throw exception; + } + } + + if (MapUtils.isNotEmpty(serviceTags.getResourceToTagIds())) { + for (Map.Entry<Long, List<Long>> entry : serviceTags.getResourceToTagIds().entrySet()) { + Long resourceId = entry.getKey(); + + RangerServiceResource resourceInStore = resourcesInStore.get(resourceId); + + if (resourceInStore == null) { + LOG.error("Resource (id=" + resourceId + ") not found. Skipping tags update"); + continue; + } + + // Get all tags associated with this resourceId + List<Long> tagMapsToDelete = null; + try { + tagMapsToDelete = tagStore.getTagIdsForResourceId(resourceInStore.getId()); + } catch (Exception exception) { + LOG.error("RangerTags cannot be retrieved for resource with guid=" + resourceInStore.getGuid()); + throw exception; + } + + List<Long> tagIds = entry.getValue(); + try { + for (Long tagId : tagIds) { + RangerTag tagInStore = tagsInStore.get(tagId); + + if (tagInStore == null) { + LOG.error("Tag (id=" + tagId + ") not found. Skipping addition of this tag for resource (id=" + resourceId + ")"); + continue; + } + + RangerTagResourceMap existing = null; + + if(createOrUpdate) { + existing = tagStore.getTagResourceMapForTagAndResourceId(tagInStore.getId(), resourceInStore.getId()); + } + + if(existing == null) { + RangerTagResourceMap tagResourceMap = new RangerTagResourceMap(); + + tagResourceMap.setTagId(tagInStore.getId()); + tagResourceMap.setResourceId(resourceInStore.getId()); + + tagResourceMap = tagStore.createTagResourceMap(tagResourceMap); + } + + if(tagMapsToDelete != null) { + tagMapsToDelete.remove((Long)tagInStore.getId()); + } + } + } catch (Exception exception) { + LOG.error("createRangerTagResourceMap failed", exception); + throw exception; + } + + if (CollectionUtils.isNotEmpty(tagMapsToDelete)) { + Long tagId = null; + + try { + for(int i = 0; i < tagMapsToDelete.size(); i++) { + tagId = tagMapsToDelete.get(i); + + RangerTagResourceMap tagResourceMap = tagStore.getTagResourceMapForTagAndResourceId(tagId, resourceInStore.getId()); + + if(tagResourceMap != null) { + tagStore.deleteTagResourceMap(tagResourceMap.getId()); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("Deleted tagResourceMap(tagId=" + tagId + ", resourceId=" + resourceId); + } + } + } catch(Exception exception) { + LOG.error("deleteTagResourceMap failed, tagId=" + tagId + ", resourceId=" + resourceId); + throw exception; + } + } + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== ServiceTagsProcessor.createOrUpdate()"); + } + } + + private void delete(ServiceTags serviceTags) throws Exception { + if (LOG.isDebugEnabled()) { + LOG.debug("==> ServiceTagsProcessor.delete()"); + } + + List<RangerServiceResource> serviceResources = serviceTags.getServiceResources(); + if (CollectionUtils.isNotEmpty(serviceResources)) { + for (RangerServiceResource serviceResource : serviceResources) { + try { + RangerServiceResource objToDelete = tagStore.getServiceResourceByGuid(serviceResource.getGuid()); + + if(objToDelete != null) { + tagStore.deleteServiceResource(objToDelete.getId()); + } + } catch (Exception exception) { + LOG.error("deleteServiceResourceByGuid failed, guid=" + serviceResource.getGuid(), exception); + throw exception; + } + } + } + + Map<Long, RangerTag> tagsMap = serviceTags.getTags(); + if (MapUtils.isNotEmpty(tagsMap)) { + for (Map.Entry<Long, RangerTag> entry : tagsMap.entrySet()) { + RangerTag tag = entry.getValue(); + try { + RangerTag objToDelete = tagStore.getTagByGuid(tag.getGuid()); + + if(objToDelete != null) { + tagStore.deleteTag(objToDelete.getId()); + } + } catch (Exception exception) { + LOG.error("deleteTag failed, guid=" + tag.getGuid(), exception); + throw exception; + } + } + } + + Map<Long, RangerTagDef> tagDefsMap = serviceTags.getTagDefinitions(); + if (MapUtils.isNotEmpty(tagDefsMap)) { + for (Map.Entry<Long, RangerTagDef> entry : tagDefsMap.entrySet()) { + RangerTagDef tagDef = entry.getValue(); + try { + RangerTagDef objToDelete = tagStore.getTagDefByGuid(tagDef.getGuid()); + + if(objToDelete != null) { + tagStore.deleteTagDef(objToDelete.getId()); + } + } catch (Exception exception) { + LOG.error("deleteTagDef failed, guid=" + tagDef.getGuid(), exception); + throw exception; + } + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== ServiceTagsProcessor.delete()"); + } + } + +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/security-admin/src/main/java/org/apache/ranger/rest/TagREST.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/rest/TagREST.java b/security-admin/src/main/java/org/apache/ranger/rest/TagREST.java index dfa4a64..378ff0b 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/TagREST.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/TagREST.java @@ -29,6 +29,7 @@ import org.apache.ranger.plugin.model.RangerServiceResource; import org.apache.ranger.plugin.model.RangerTag; import org.apache.ranger.plugin.model.RangerTagResourceMap; import org.apache.ranger.plugin.model.RangerTagDef; +import org.apache.ranger.plugin.store.TagStore; import org.apache.ranger.plugin.store.TagValidator; import org.apache.ranger.plugin.util.SearchFilter; import org.apache.ranger.plugin.util.ServiceTags; @@ -75,19 +76,30 @@ public class TagREST { validator.setTagStore(tagStore); } + TagStore getTagStore() { + return tagStore; + } + @POST @Path(TagRESTConstants.TAGDEFS_RESOURCE) @Produces({ "application/json", "application/xml" }) @PreAuthorize("hasRole('ROLE_SYS_ADMIN')") - public RangerTagDef createTagDef(RangerTagDef tagDef) { + public RangerTagDef createTagDef(RangerTagDef tagDef, @DefaultValue("true") @QueryParam("updateIfExists") boolean updateIfExists) { if(LOG.isDebugEnabled()) { - LOG.debug("==> TagREST.createTagDef(" + tagDef + ")"); + LOG.debug("==> TagREST.createTagDef(" + tagDef + ", " + updateIfExists + ")"); } RangerTagDef ret; try { - ret = tagStore.createTagDef(tagDef); + RangerTagDef exist = validator.preCreateTagDef(tagDef, updateIfExists); + if (exist == null) { + ret = tagStore.createTagDef(tagDef); + } else if (updateIfExists) { + ret = updateTagDef(exist.getId(), exist); + } else { + throw new Exception("tag-definition with Id " + exist.getId() + " already exists"); + } } catch(Exception excp) { LOG.error("createTagDef(" + tagDef + ") failed", excp); @@ -95,12 +107,13 @@ public class TagREST { } if(LOG.isDebugEnabled()) { - LOG.debug("<== TagREST.createTagDef(" + tagDef + "): " + ret); + LOG.debug("<== TagREST.createTagDef(" + tagDef + ", " + updateIfExists + "): " + ret); } return ret; } + @PUT @Path(TagRESTConstants.TAGDEF_RESOURCE + "{id}") @Produces({ "application/json", "application/xml" }) @@ -327,16 +340,22 @@ public class TagREST { @Path(TagRESTConstants.TAGS_RESOURCE) @Produces({ "application/json", "application/xml" }) @PreAuthorize("hasRole('ROLE_SYS_ADMIN')") - public RangerTag createTag(RangerTag tag) { + public RangerTag createTag(RangerTag tag, @DefaultValue("true") @QueryParam("updateIfExists") boolean updateIfExists) { if(LOG.isDebugEnabled()) { - LOG.debug("==> TagREST.createTag(" + tag + ")"); + LOG.debug("==> TagREST.createTag(" + tag + ", " + updateIfExists + ")"); } RangerTag ret; try { - validator.preCreateTag(tag); - ret = tagStore.createTag(tag); + RangerTag exist = validator.preCreateTag(tag); + if (exist == null) { + ret = tagStore.createTag(tag); + } else if (updateIfExists) { + ret = updateTag(exist.getId(), exist); + } else { + throw new Exception("tag with Id " + exist.getId() + " already exists"); + } } catch(Exception excp) { LOG.error("createTag(" + tag + ") failed", excp); @@ -344,7 +363,7 @@ public class TagREST { } if(LOG.isDebugEnabled()) { - LOG.debug("<== TagREST.createTag(" + tag + "): " + ret); + LOG.debug("<== TagREST.createTag(" + tag + ", " + updateIfExists + "): " + ret); } return ret; @@ -553,16 +572,22 @@ public class TagREST { @Path(TagRESTConstants.RESOURCES_RESOURCE) @Produces({ "application/json", "application/xml" }) @PreAuthorize("hasRole('ROLE_SYS_ADMIN')") - public RangerServiceResource createServiceResource(RangerServiceResource resource) { + public RangerServiceResource createServiceResource(RangerServiceResource resource, @DefaultValue("true") @QueryParam("updateIfExists") boolean updateIfExists) { if(LOG.isDebugEnabled()) { - LOG.debug("==> TagREST.createServiceResource(" + resource + ")"); + LOG.debug("==> TagREST.createServiceResource(" + resource + ", " + updateIfExists + ")"); } RangerServiceResource ret; try { - validator.preCreateServiceResource(resource); - ret = tagStore.createServiceResource(resource); + RangerServiceResource exist = validator.preCreateServiceResource(resource); + if (exist == null) { + ret = tagStore.createServiceResource(resource); + } else if (updateIfExists) { + ret = updateServiceResource(exist.getId(), exist); + } else { + throw new Exception("resource with Id " + exist.getId() + " already exists"); + } } catch(Exception excp) { LOG.error("createServiceResource(" + resource + ") failed", excp); @@ -570,7 +595,7 @@ public class TagREST { } if(LOG.isDebugEnabled()) { - LOG.debug("<== TagREST.createServiceResource(" + resource + "): " + ret); + LOG.debug("<== TagREST.createServiceResource(" + resource + ", " + updateIfExists + "): " + ret); } return ret; @@ -650,22 +675,30 @@ public class TagREST { @Path(TagRESTConstants.RESOURCE_RESOURCE + "guid/{guid}") @Produces({ "application/json", "application/xml" }) @PreAuthorize("hasRole('ROLE_SYS_ADMIN')") - public void deleteServiceResourceByGuid(@PathParam("guid") String guid) { + public void deleteServiceResourceByGuid(@PathParam("guid") String guid, @DefaultValue("false") @QueryParam("deleteReferences") boolean deleteReferences) { if(LOG.isDebugEnabled()) { - LOG.debug("==> TagREST.deleteServiceResourceByGuid(" + guid + ")"); + LOG.debug("==> TagREST.deleteServiceResourceByGuid(" + guid + ", " + deleteReferences + ")"); } try { - RangerServiceResource exist = validator.preDeleteServiceResourceByGuid(guid); + RangerServiceResource exist = validator.preDeleteServiceResourceByGuid(guid, deleteReferences); + if (deleteReferences) { + List<RangerTagResourceMap> tagResourceMaps = tagStore.getTagResourceMapsForResourceGuid(exist.getGuid()); + if (CollectionUtils.isNotEmpty(tagResourceMaps)) { + for (RangerTagResourceMap tagResourceMap : tagResourceMaps) { + deleteTagResourceMap(tagResourceMap.getId()); + } + } + } tagStore.deleteServiceResource(exist.getId()); } catch(Exception excp) { - LOG.error("deleteServiceResourceByGuid(" + guid + ") failed", excp); + LOG.error("deleteServiceResourceByGuid(" + guid + ", " + deleteReferences + ") failed", excp); throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, excp.getMessage(), true); } if(LOG.isDebugEnabled()) { - LOG.debug("<== TagREST.deleteServiceResourceByGuid(" + guid + ")"); + LOG.debug("<== TagREST.deleteServiceResourceByGuid(" + guid + ", " + deleteReferences + ")"); } } @@ -800,24 +833,31 @@ public class TagREST { @Path(TagRESTConstants.TAGRESOURCEMAPS_RESOURCE) @Produces({ "application/json", "application/xml" }) @PreAuthorize("hasRole('ROLE_SYS_ADMIN')") - public RangerTagResourceMap createTagResourceMap(@QueryParam("tag-guid") String tagGuid, @QueryParam("resource-guid") String resourceGuid) { + public RangerTagResourceMap createTagResourceMap(@QueryParam("tag-guid") String tagGuid, @QueryParam("resource-guid") String resourceGuid, + @DefaultValue("false") @QueryParam("lenient") boolean lenient) { if(LOG.isDebugEnabled()) { - LOG.debug("==> TagREST.createTagResourceMap(" + tagGuid + ", " + resourceGuid + ")"); + LOG.debug("==> TagREST.createTagResourceMap(" + tagGuid + ", " + resourceGuid + ", " + lenient + ")"); } RangerTagResourceMap tagResourceMap; try { - tagResourceMap = validator.preCreateTagResourceMap(tagGuid, resourceGuid); - tagResourceMap = tagStore.createTagResourceMap(tagResourceMap); + tagResourceMap = tagStore.getTagResourceMapForTagAndResourceGuid(tagGuid, resourceGuid); + if (tagResourceMap == null) { + tagResourceMap = validator.preCreateTagResourceMap(tagGuid, resourceGuid); + + tagResourceMap = tagStore.createTagResourceMap(tagResourceMap); + } else if (!lenient) { + throw new Exception("tagResourceMap with tag-guid=" + tagGuid + " and resource-guid=" + resourceGuid + " already exists"); + } } catch(Exception excp) { - LOG.error("createTagResourceMap(" + tagGuid + ", " + resourceGuid + ") failed", excp); + LOG.error("createTagResourceMap(" + tagGuid + ", " + resourceGuid + ", " + lenient + ") failed", excp); throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, excp.getMessage(), true); } if(LOG.isDebugEnabled()) { - LOG.debug("==> TagREST.createTagResourceMap(" + tagGuid + ", " + resourceGuid + ")"); + LOG.debug("==> TagREST.createTagResourceMap(" + tagGuid + ", " + resourceGuid + ", " + lenient + ")"); } return tagResourceMap; @@ -998,8 +1038,35 @@ public class TagREST { return ret; } + // This API is used by tag-sync to upload tag-objects + + @PUT + @Path(TagRESTConstants.IMPORT_SERVICETAGS_RESOURCE) + @Produces({ "application/json", "application/xml" }) + @PreAuthorize("hasRole('ROLE_SYS_ADMIN')") + public void importServiceTags(ServiceTags serviceTags) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> TagREST.importServiceTags()"); + } + + try { + + ServiceTagsProcessor serviceTagsProcessor = new ServiceTagsProcessor(tagStore); + serviceTagsProcessor.process(serviceTags); + + } catch (Exception excp) { + LOG.error("importServiceTags() failed", excp); + + throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, excp.getMessage(), true); + + } + if(LOG.isDebugEnabled()) { + LOG.debug("<== TagREST.importServiceTags()"); + } + } // This API is typically used by plug-in to get selected tagged resources from RangerAdmin + @GET @Path(TagRESTConstants.TAGS_DOWNLOAD + "{serviceName}") @Produces({ "application/json", "application/xml" }) http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/security-admin/src/main/java/org/apache/ranger/rest/TagRESTConstants.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/rest/TagRESTConstants.java b/security-admin/src/main/java/org/apache/ranger/rest/TagRESTConstants.java index 977e487..919f814 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/TagRESTConstants.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/TagRESTConstants.java @@ -29,6 +29,7 @@ public class TagRESTConstants { static final String RESOURCES_RESOURCE = "/resources/"; static final String RESOURCE_RESOURCE = "/resource/"; static final String TAGRESOURCEMAPS_RESOURCE = "/tagresourcemaps/"; + static final String IMPORT_SERVICETAGS_RESOURCE = "/importservicetags/"; static final String TAGRESOURCEMAP_RESOURCE = "/tagresourcemap/"; static final String TAGTYPES_RESOURCE = "/types/"; static final String TAGTYPES_LOOKUP_RESOURCE = "/types/lookup/"; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/security-admin/src/main/java/org/apache/ranger/service/RangerTagResourceMapService.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/service/RangerTagResourceMapService.java b/security-admin/src/main/java/org/apache/ranger/service/RangerTagResourceMapService.java index d50a937..40d9d64 100644 --- a/security-admin/src/main/java/org/apache/ranger/service/RangerTagResourceMapService.java +++ b/security-admin/src/main/java/org/apache/ranger/service/RangerTagResourceMapService.java @@ -132,6 +132,12 @@ public class RangerTagResourceMapService extends RangerTagResourceMapServiceBase return ret; } + public List<Long> getTagIdsForResourceId(Long resourceId) { + List<Long> ret = daoMgr.getXXTagResourceMap().findTagIdsForResourceId(resourceId); + + return ret; + } + public List<RangerTagResourceMap> getByResourceGuid(String resourceGuid) { List<RangerTagResourceMap> ret = new ArrayList<RangerTagResourceMap>(); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/security-admin/src/main/resources/META-INF/jpa_named_queries.xml ---------------------------------------------------------------------- diff --git a/security-admin/src/main/resources/META-INF/jpa_named_queries.xml b/security-admin/src/main/resources/META-INF/jpa_named_queries.xml index a5f4a6c..993e7d6 100644 --- a/security-admin/src/main/resources/META-INF/jpa_named_queries.xml +++ b/security-admin/src/main/resources/META-INF/jpa_named_queries.xml @@ -599,6 +599,12 @@ where obj.resourceId = res.id and res.serviceId = :serviceId order by obj.resourceId</query> </named-query> + + <named-query name="XXTagResourceMap.getTagIdsForResourceId"> + <query>select obj.tagId from XXTagResourceMap obj + where obj.resourceId = :resourceId + </query> + </named-query> <named-query name="XXTagDef.findByGuid"> <query>select obj from XXTagDef obj where obj.guid = :guid</query> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/src/main/assembly/tagsync.xml ---------------------------------------------------------------------- diff --git a/src/main/assembly/tagsync.xml b/src/main/assembly/tagsync.xml index d7262ac..4f43851 100644 --- a/src/main/assembly/tagsync.xml +++ b/src/main/assembly/tagsync.xml @@ -47,14 +47,24 @@ <include>org.slf4j:slf4j-api</include> <include>org.apache.hadoop:hadoop-common</include> <include>org.apache.commons:commons-csv</include> - <include>org.apache.ranger:credentialbuilder</include> - <include>org.apache.ranger:ranger-util</include> <include>commons-io:commons-io:jar:${commons.io.version}</include> <include>org.apache.htrace:htrace-core</include> <include>org.codehaus.jackson:jackson-core-asl</include> <include>org.codehaus.jackson:jackson-jaxrs</include> <include>org.codehaus.jackson:jackson-mapper-asl</include> <include>org.codehaus.jackson:jackson-xc</include> + <include>org.apache.atlas:atlas-notification</include> + <include>org.apache.atlas:atlas-typesystem</include> + <include>org.apache.atlas:atlas-client</include> + <include>com.google.inject:guice</include> + <include>aopalliance:aopalliance</include> + <include>javax.inject:javax.inject</include> + <include>org.apache.kafka:kafka_2.10</include> + <include>org.apache.kafka:kafka-clients</include> + <include>com.yammer.metrics:metrics-core</include> + <include>org.scala-lang:scala-library</include> + <include>com.101tec:zkclient</include> + <include>org.apache.zookeeper:zookeeper</include> <include>security_plugins.ranger-plugins-common:ranger-plugins-common</include> </includes> <unpack>false</unpack> @@ -66,6 +76,29 @@ <include>org.apache.ranger:ranger-tagsync</include> </includes> </moduleSet> + <moduleSet> + <binaries> + <includeDependencies>false</includeDependencies> + <unpack>false</unpack> + <directoryMode>755</directoryMode> + <fileMode>644</fileMode> + <dependencySets> + <dependencySet> + <outputDirectory>/lib</outputDirectory> + <includes> + <include>org.apache.ranger:credentialbuilder</include> + <include>org.apache.ranger:ranger-util</include> + </includes> + <unpack>false</unpack> + </dependencySet> + </dependencySets> + <outputDirectory>/lib</outputDirectory> + </binaries> + <includes> + <include>org.apache.ranger:credentialbuilder</include> + <include>org.apache.ranger:ranger-util</include> + </includes> + </moduleSet> </moduleSets> <fileSets> <fileSet> @@ -73,6 +106,15 @@ <fileMode>644</fileMode> <outputDirectory>/conf.dist</outputDirectory> <directory>tagsync/conf.dist</directory> + <excludes> + <exclude>etc</exclude> + </excludes> + </fileSet> + <fileSet> + <directoryMode>755</directoryMode> + <fileMode>644</fileMode> + <outputDirectory>/conf.dist/etc/ranger/data</outputDirectory> + <directory>tagsync/src/main/resources/etc/ranger/data</directory> </fileSet> <fileSet> <directoryMode>755</directoryMode> @@ -98,7 +140,7 @@ <file> <source>tagsync/scripts/initd</source> <outputDirectory>/</outputDirectory> - <destName>ranger-usersync.sh</destName> + <destName>ranger-tagsync.sh</destName> <fileMode>755</fileMode> </file> </files> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/tagsync/conf.dist/log4j.xml ---------------------------------------------------------------------- diff --git a/tagsync/conf.dist/log4j.xml b/tagsync/conf.dist/log4j.xml index fb6986f..d455509 100644 --- a/tagsync/conf.dist/log4j.xml +++ b/tagsync/conf.dist/log4j.xml @@ -23,14 +23,14 @@ <param name="file" value="${logdir}/tagsync.log" /> <param name="DatePattern" value="'.'yyyy-MM-dd" /> <layout class="org.apache.log4j.PatternLayout"> - <param name="ConversionPattern" value="%d{dd MMM yyyy HH:mm:ss} %5p %c{1} [%t] - %m%n"/> + <param name="ConversionPattern" value="%d{dd MMM yyyy HH:mm:ss} %5p %c{1} [%t] - %L %m%n"/> </layout> </appender> <appender name="console" class="org.apache.log4j.ConsoleAppender"> <param name="Target" value="System.out"/> <layout class="org.apache.log4j.PatternLayout"> - <param name="ConversionPattern" value="%d{dd MMM yyyy HH:mm:ss} %5p %c{1} [%t] - %m%n"/> + <param name="ConversionPattern" value="%d{dd MMM yyyy HH:mm:ss} %5p %c{1} [%t] - %L %m%n"/> </layout> </appender> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/tagsync/conf.dist/ranger-tagsync-default-site.xml ---------------------------------------------------------------------- diff --git a/tagsync/conf.dist/ranger-tagsync-default-site.xml b/tagsync/conf.dist/ranger-tagsync-default-site.xml deleted file mode 100644 index b098740..0000000 --- a/tagsync/conf.dist/ranger-tagsync-default-site.xml +++ /dev/null @@ -1,36 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> -<!-- - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. See accompanying LICENSE file. ---> - -<!-- Put site-specific property overrides in this file. --> - -<configuration> - <property> - <name>ranger.tagsync.port</name> - <value>6161</value> - </property> - <property> - <name>ranger.tagsync.ssl</name> - <value>true</value> - </property> - <property> - <name>ranger.tagsync.enabled</name> - <value>true</value> - </property> - <property> - <name>ranger.tagsync.logdir</name> - <value>./log</value> - </property> -</configuration> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/tagsync/conf.dist/ranger-tagsync-site.xml ---------------------------------------------------------------------- diff --git a/tagsync/conf.dist/ranger-tagsync-site.xml b/tagsync/conf.dist/ranger-tagsync-site.xml new file mode 100644 index 0000000..21accd4 --- /dev/null +++ b/tagsync/conf.dist/ranger-tagsync-site.xml @@ -0,0 +1,67 @@ +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> + + +<configuration> + <property> + <name>ranger.authentication.method</name> + <value>NONE</value> + <description></description> + </property> + <property> + <name>ranger.tagsync.tagadmin.rest.url</name> + <value>http://localhost:6080</value> + <description></description> + </property> + <property> + <name>ranger.tagsync.tagadmin.rest.ssl.config.file</name> + <value></value> + <description></description> + </property> + <property> + <name>ranger.tagsync.policymanager.basicauth.username</name> + <value>admin</value> + <description></description> + </property> + <property> + <name>ranger.tagsync.policymanager.basicauth.password</name> + <value>admin</value> + <description></description> + </property> + <property> + <name>ranger.tagsync.sleeptimeinmillisbetweensynccycle</name> + <value>60000</value> + <description></description> + </property> + <property> + <name>ranger.tagsync.source.file</name> + <value>/etc/ranger/data/tags.json</value> + <description></description> + </property> + <property> + <name>ranger.tagsync.source.impl.class</name> + <value>org.apache.ranger.source.file.TagFileSource</value> + <description></description> + </property> + <property> + <name>ranger.tagsync.sink.impl.class</name> + <value>org.apache.ranger.sink.policymgr.TagRESTSink</value> + <description></description> + </property> + <property> + <name>atlas.endpoint</name> + <value>http://localhost:21000/</value> + <description></description> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/tagsync/pom.xml ---------------------------------------------------------------------- diff --git a/tagsync/pom.xml b/tagsync/pom.xml index 83011d2..18c684c 100644 --- a/tagsync/pom.xml +++ b/tagsync/pom.xml @@ -86,7 +86,6 @@ <artifactId>ranger-plugins-common</artifactId> <version>${project.version}</version> </dependency> - <!-- <dependency> <groupId>com.google.inject</groupId> <artifactId>guice</artifactId> @@ -96,17 +95,22 @@ <groupId>org.apache.atlas</groupId> <artifactId>atlas-notification</artifactId> <version>0.6-incubating-SNAPSHOT</version> + <scope>system</scope> + <systemPath>${local.lib.dir}/apache-atlas-0.6-incubating-SNAPSHOT/atlas-notification-0.6-incubating-SNAPSHOT.jar</systemPath> </dependency> <dependency> <groupId>org.apache.atlas</groupId> <artifactId>atlas-typesystem</artifactId> <version>0.6-incubating-SNAPSHOT</version> + <scope>system</scope> + <systemPath>${local.lib.dir}/apache-atlas-0.6-incubating-SNAPSHOT/atlas-typesystem-0.6-incubating-SNAPSHOT.jar</systemPath> </dependency> <dependency> <groupId>org.apache.atlas</groupId> <artifactId>atlas-client</artifactId> <version>0.6-incubating-SNAPSHOT</version> + <scope>system</scope> + <systemPath>${local.lib.dir}/apache-atlas-0.6-incubating-SNAPSHOT/atlas-client-0.6-incubating-SNAPSHOT.jar</systemPath> </dependency> - --> </dependencies> </project> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/tagsync/samples/tags.json ---------------------------------------------------------------------- diff --git a/tagsync/samples/tags.json b/tagsync/samples/tags.json new file mode 100644 index 0000000..1b24fc8 --- /dev/null +++ b/tagsync/samples/tags.json @@ -0,0 +1,55 @@ +{ + "op": "add_or_update", + "serviceName": "cl1_hive", + "tagVersion": 1, + "tagUpdateTime": "20150924-22:26:33.000-+0000", + "tagDefinitions": { + "1":{"id":1, "guid":"tagdef-1", "name":"PII", "attributeDefs":[]}, + "2":{"id":2, "guid":"tagdef-2", "name":"EXPIRES_ON", "attributeDefs":[{"name":"expiry_date", "type":"date"}]}, + "3":{"id":3, "guid":"tagdef-3", "name":"FINANCE", "attributeDefs":[]}, + "4":{"id":4, "guid":"tagdef-4", "name":"AUDIT", "attributeDefs":[]} + }, + "tags": { + "1":{"id":1, "guid":"tag-1", "type":"PII", "attributes":{}}, + "2":{"id":2, "guid":"tag-2", "type":"EXPIRES_ON", "attributes":{"expiry_date":"2015/08/31"}}, + "3":{"id":3, "guid":"tag-3", "type":"FINANCE", "attributes":{}}, + "4":{"id":4, "guid":"tag-4", "type":"AUDIT", "attributes":{}} + }, + "serviceResources": [ + { + "id":1, + "guid":"resource-1", + "serviceName":"cl1_hive", + "resourceElements":{ + "database":{"values":["hr"]}, + "table":{"values":["employee"]}, + "column":{"values":["ssn"]} + } + }, + { + "id":2, + "guid":"resource-2", + "serviceName":"cl1_hive", + "resourceElements":{ + "database":{"values":["finance"]}, + "table":{"values":["tax_2010"]} + } + }, + { + "id":3, + "guid":"resource-3", + "serviceName":"cl1_hive", + "resourceElements":{ + "database":{"values":["finance"]}, + "table":{"values":["tax_2010"]}, + "column":{"values":["ssn"]} + } + } + ], + + "resourceToTagIds": { + "1":[1], + "2":[2], + "3":[3] + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/tagsync/scripts/ranger-tagsync-services.sh ---------------------------------------------------------------------- diff --git a/tagsync/scripts/ranger-tagsync-services.sh b/tagsync/scripts/ranger-tagsync-services.sh index 5ca5b95..2f4b255 100755 --- a/tagsync/scripts/ranger-tagsync-services.sh +++ b/tagsync/scripts/ranger-tagsync-services.sh @@ -27,8 +27,13 @@ realScriptDir=`dirname $realScriptPath` cd $realScriptDir cdir=`pwd` -pidf=/var/run/ranger/tagsync.pid +pidd=/var/run/ranger +if [ -d $pidd ]; then + mkdir -p $pidd +fi + +pidf=${pidd}/tagsync.pid if [ "${action}" == "START" ]; then @@ -47,9 +52,13 @@ if [ "${action}" == "START" ]; then export PATH=$JAVA_HOME/bin:$PATH fi - logdir=/var/log/ranger/tagsync + logdir=/var/log/ranger/tagsync + + if [ ! -d $logdir ]; then + mkdir -p $logdir + fi - cp="${cdir}/dist/*:${cdir}/lib/*:${cdir}/conf" + cp="${cdir}/conf.dist:${cdir}/dist/*:${cdir}/lib/*" if [ -f $pidf ]; then PID=`cat $pidf` @@ -81,7 +90,6 @@ if [ "${action}" == "START" ]; then elif [ "${action}" == "STOP" ]; then if [ -f $pidf ]; then - pidf=/var/run/ranger/tagsync.pid PID=`cat $pidf` > /dev/null 2>&1 kill -9 $PID > /dev/null 2>&1 rm -f $pidf @@ -103,8 +111,8 @@ elif [ "${action}" == "VERSION" ]; then java -cp ranger-util-*.jar org.apache.ranger.common.RangerVersionInfo exit else - echo "Invalid argument [$1];" - echo "Usage: Only start | stop | restart | version, are supported." - exit; + echo "Invalid argument [$1];" + echo "Usage: Only start | stop | restart | version, are supported." + exit; fi http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/tagsync/src/main/java/org/apache/ranger/model/TagSink.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/model/TagSink.java b/tagsync/src/main/java/org/apache/ranger/model/TagSink.java index 74ea5cf..5256570 100644 --- a/tagsync/src/main/java/org/apache/ranger/model/TagSink.java +++ b/tagsync/src/main/java/org/apache/ranger/model/TagSink.java @@ -26,9 +26,6 @@ import java.util.Map; import java.util.Properties; -/** - * Created by akulkarni on 9/10/15. - */ public interface TagSink extends TagStore { boolean initialize(Properties properties); void uploadServiceTags(ServiceTags serviceTags) throws Exception; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/tagsync/src/main/java/org/apache/ranger/model/TagSource.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/model/TagSource.java b/tagsync/src/main/java/org/apache/ranger/model/TagSource.java index 568d3c7..0ccceae 100644 --- a/tagsync/src/main/java/org/apache/ranger/model/TagSource.java +++ b/tagsync/src/main/java/org/apache/ranger/model/TagSource.java @@ -26,9 +26,6 @@ import java.util.List; import java.util.Map; import java.util.Properties; -/** - * Created by akulkarni on 9/10/15. - */ public interface TagSource { boolean initialize(Properties properties); @@ -37,16 +34,8 @@ public interface TagSource { void updateSink() throws Exception; - void start(); + Thread start(); boolean isChanged(); - List<RangerTagDef> fetchAllTagDefs(String syncSentinel) throws Exception; - - List<RangerTagDef> receiveUpdatesToTagDefs() throws Exception; - - List<RangerTagResourceMap> fetchAllTaggedEntities() throws Exception; - - List<RangerTagResourceMap> receiveUpdatesToTaggedEntities() throws Exception; - } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/tagsync/src/main/java/org/apache/ranger/process/TagSyncConfig.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/process/TagSyncConfig.java b/tagsync/src/main/java/org/apache/ranger/process/TagSyncConfig.java index 1c995b1..c80eff9 100644 --- a/tagsync/src/main/java/org/apache/ranger/process/TagSyncConfig.java +++ b/tagsync/src/main/java/org/apache/ranger/process/TagSyncConfig.java @@ -20,27 +20,20 @@ package org.apache.ranger.process; import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; import org.apache.log4j.Logger; -import org.w3c.dom.Document; -import org.w3c.dom.Element; -import org.w3c.dom.Node; -import org.w3c.dom.NodeList; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; import java.io.*; +import java.net.MalformedURLException; import java.net.URL; import java.util.Properties; -/** - * Created by akulkarni on 9/11/15. - */ -public class TagSyncConfig { +public class TagSyncConfig extends Configuration { private static final Logger LOG = Logger.getLogger(TagSyncConfig.class) ; public static final String CONFIG_FILE = "ranger-tagsync-site.xml"; - public static final String DEFAULT_CONFIG_FILE = "ranger-tagsync-default-site.xml"; + public static final String DEFAULT_CONFIG_FILE = "ranger-tagsync-default.xml"; public static final String TAGSYNC_ENABLED_PROP = "ranger.tagsync.enabled" ; @@ -50,9 +43,9 @@ public class TagSyncConfig { public static final String TAGSYNC_LOGDIR_PROP = "ranger.tagsync.logdir" ; - private static final String TAGSYNC_PM_URL_PROP = "ranger.tagsync.policymanager.baseURL"; + private static final String TAGSYNC_TAGADMIN_REST_URL_PROP = "ranger.tagsync.tagadmin.rest.url"; - private static final String TAGSYNC_PM_SSL_CONFIG_FILE_PROP = "ranger.tagsync.policymanager.ssl.config.file"; + private static final String TAGSYNC_TAGADMIN_REST_SSL_CONFIG_FILE_PROP = "ranger.tagsync.tagadmin.rest.ssl.config.file"; private static final String TAGSYNC_PM_SSL_BASICAUTH_USERNAME_PROP = "ranger.tagsync.policymanager.basicauth.username"; @@ -68,9 +61,13 @@ public class TagSyncConfig { private static final String TAGSYNC_SOURCE_ATLAS_PROP = "atlas.endpoint"; - private static volatile TagSyncConfig instance = null; + private static final String TAGSYNC_SERVICENAME_MAPPER_PROP_PREFIX = "ranger.tagsync.atlas."; + + private static final String TAGSYNC_SERVICENAME_MAPPER_PROP_SUFFIX = ".ranger.service"; - private Properties prop = new Properties() ; + private static final String TAGSYNC_DEFAULT_CLUSTERNAME_AND_COMPONENTNAME_SEPARATOR = "_"; + + private static volatile TagSyncConfig instance = null; public static TagSyncConfig getInstance() { /* @@ -85,12 +82,11 @@ public class TagSyncConfig { } */ TagSyncConfig newConfig = new TagSyncConfig(); - newConfig.init(); return newConfig; } public Properties getProperties() { - return prop; + return getProps(); } public static InputStream getFileInputStream(String path) throws FileNotFoundException { @@ -158,7 +154,7 @@ public class TagSyncConfig { LOG.error(path + " is not a file", exception); } } else { - LOG.error("URL not found for " + path + " or no privilege for reading file " + path); + LOG.warn("URL not found for " + path + " or no privilege for reading file " + path); } } } @@ -170,16 +166,10 @@ public class TagSyncConfig { public String toString() { StringBuffer sb = new StringBuffer(); - sb.append("CONFIG_FILE=").append(CONFIG_FILE).append(", ") - .append("DEFAULT_CONFIG_FILE=").append(DEFAULT_CONFIG_FILE).append("\n"); - - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - PrintStream printStream = new PrintStream(outputStream); - prop.list(printStream); - printStream.close(); - sb.append(outputStream.toString()); + sb.append("DEFAULT_CONFIG_FILE=").append(DEFAULT_CONFIG_FILE).append(", ") + .append("CONFIG_FILE=").append(CONFIG_FILE).append("\n\n"); - return sb.toString(); + return sb.toString() + super.toString(); } static public boolean isTagSyncEnabled(Properties prop) { @@ -217,13 +207,13 @@ public class TagSyncConfig { return val; } - static public String getPolicyMgrUrl(Properties prop) { - String val = prop.getProperty(TAGSYNC_PM_URL_PROP); + static public String getTagAdminRESTUrl(Properties prop) { + String val = prop.getProperty(TAGSYNC_TAGADMIN_REST_URL_PROP); return val; } - static public String getPolicyMgrSslConfigFile(Properties prop) { - String val = prop.getProperty(TAGSYNC_PM_SSL_CONFIG_FILE_PROP); + static public String getTagAdminRESTSslConfigFile(Properties prop) { + String val = prop.getProperty(TAGSYNC_TAGADMIN_REST_SSL_CONFIG_FILE_PROP); return val; } @@ -251,75 +241,39 @@ public class TagSyncConfig { return ""; } + static public String getServiceName(String componentName, String instanceName, Properties prop) { + String propName = TAGSYNC_SERVICENAME_MAPPER_PROP_PREFIX + componentName + + ".instance." + instanceName + + TAGSYNC_SERVICENAME_MAPPER_PROP_SUFFIX; + String val = prop.getProperty(propName); + if (StringUtils.isBlank(val)) { + val = instanceName + TAGSYNC_DEFAULT_CLUSTERNAME_AND_COMPONENTNAME_SEPARATOR + componentName; + } + return val; + } + private TagSyncConfig() { + super(false); init() ; } private void init() { - readConfigFile(CONFIG_FILE); readConfigFile(DEFAULT_CONFIG_FILE); + readConfigFile(CONFIG_FILE); } private void readConfigFile(String fileName) { - try { - InputStream in = getFileInputStream(fileName); - if (in != null) { - try { - DocumentBuilderFactory xmlDocumentBuilderFactory = DocumentBuilderFactory - .newInstance(); - xmlDocumentBuilderFactory.setIgnoringComments(true); - xmlDocumentBuilderFactory.setNamespaceAware(true); - DocumentBuilder xmlDocumentBuilder = xmlDocumentBuilderFactory - .newDocumentBuilder(); - Document xmlDocument = xmlDocumentBuilder.parse(in); - xmlDocument.getDocumentElement().normalize(); - - NodeList nList = xmlDocument - .getElementsByTagName("property"); - - for (int temp = 0; temp < nList.getLength(); temp++) { - - Node nNode = nList.item(temp); - - if (nNode.getNodeType() == Node.ELEMENT_NODE) { - - Element eElement = (Element) nNode; - - String propertyName = ""; - String propertyValue = ""; - if (eElement.getElementsByTagName("name").item( - 0) != null) { - propertyName = eElement - .getElementsByTagName("name") - .item(0).getTextContent().trim(); - } - if (eElement.getElementsByTagName("value") - .item(0) != null) { - propertyValue = eElement - .getElementsByTagName("value") - .item(0).getTextContent().trim(); - } - - if (prop.get(propertyName) != null) { - prop.remove(propertyName) ; - } - - prop.put(propertyName, propertyValue); - } - } - } - finally { - try { - in.close() ; - } - catch(IOException ioe) { - // Ignore IOE when closing stream - } - } + if (StringUtils.isNotBlank(fileName)) { + String fName = getResourceFileName(fileName); + if (StringUtils.isBlank(fName)) { + LOG.warn("Cannot find configuration file " + fileName + " in the classpath"); + } else { + LOG.info("Loading configuration from " + fName); + addResource(fileName); } - } catch (Throwable e) { - throw new RuntimeException("Unable to load configuration file [" + CONFIG_FILE + "]", e) ; + } else { + LOG.error("Configuration fileName is null"); } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/tagsync/src/main/java/org/apache/ranger/process/TagSynchronizer.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/process/TagSynchronizer.java b/tagsync/src/main/java/org/apache/ranger/process/TagSynchronizer.java index c7e8d6e..8ab2a1a 100644 --- a/tagsync/src/main/java/org/apache/ranger/process/TagSynchronizer.java +++ b/tagsync/src/main/java/org/apache/ranger/process/TagSynchronizer.java @@ -24,16 +24,14 @@ import org.apache.log4j.Logger; import org.apache.ranger.model.TagSink; import org.apache.ranger.model.TagSource; +import java.util.Map; import java.util.Properties; -/** - * Created by akulkarni on 9/11/15. - */ public class TagSynchronizer implements Runnable { private static final Logger LOG = Logger.getLogger(TagSynchronizer.class); - private final static int MAX_INIT_RETRIES = 1; + private final static int MAX_INIT_RETRIES = 5; private boolean shutdownFlag = false; private TagSink tagSink = null; @@ -46,12 +44,6 @@ public class TagSynchronizer implements Runnable { TagSyncConfig config = TagSyncConfig.getInstance(); Properties props = config.getProperties(); - LOG.info("--------------------------------"); - LOG.info(""); - LOG.info("Ranger-TagSync Configuration: {\n" + config + "\n}"); - LOG.info(""); - LOG.info("--------------------------------"); - TagSynchronizer tagSynchronizer = new TagSynchronizer(props); tagSynchronizer.run(); @@ -86,15 +78,28 @@ public class TagSynchronizer implements Runnable { if (initDone) { - tagSource.start(); + Thread tagSourceThread = tagSource.start(); - while (!shutdownFlag) { - try { - LOG.debug("Sleeping for [" + sleepTimeBetweenCycleInMillis + "] milliSeconds"); - Thread.sleep(sleepTimeBetweenCycleInMillis); - } catch (InterruptedException e) { - LOG.error("Failed to wait for [" + sleepTimeBetweenCycleInMillis + "] milliseconds before attempting to synchronize tag information", e); + if (tagSourceThread != null) { + while (!shutdownFlag) { + try { + LOG.debug("Sleeping for [" + sleepTimeBetweenCycleInMillis + "] milliSeconds"); + Thread.sleep(sleepTimeBetweenCycleInMillis); + } catch (InterruptedException e) { + LOG.error("Failed to wait for [" + sleepTimeBetweenCycleInMillis + "] milliseconds before attempting to synchronize tag information", e); + } } + if (shutdownFlag) { + LOG.info("Interrupting tagSourceThread..."); + tagSourceThread.interrupt(); + try { + tagSourceThread.join(); + } catch (InterruptedException interruptedException) { + LOG.error("tagSourceThread.join() was interrupted"); + } + } + } else { + LOG.error("Could not start tagSource monitoring thread"); } } else { LOG.error("Failed to initialize TagSynchonizer after " + MAX_INIT_RETRIES + " retries. Exiting thread"); @@ -121,6 +126,8 @@ public class TagSynchronizer implements Runnable { for (int initRetries = 0; initRetries < MAX_INIT_RETRIES && !ret; initRetries++) { + printConfigurationProperties(); + ret = init(); if (!ret) { @@ -128,6 +135,7 @@ public class TagSynchronizer implements Runnable { try { LOG.debug("Sleeping for [" + sleepTimeBetweenCycleInMillis + "] milliSeconds"); Thread.sleep(sleepTimeBetweenCycleInMillis); + properties = TagSyncConfig.getInstance().getProperties(); } catch (Exception e) { LOG.error("Failed to wait for [" + sleepTimeBetweenCycleInMillis + "] milliseconds before attempting to initialize tag source/sink", e); } @@ -181,7 +189,23 @@ public class TagSynchronizer implements Runnable { } public void shutdown(String reason) { - LOG.error("Received shutdown(), reason=" + reason); + LOG.info("Received shutdown(), reason=" + reason); this.shutdownFlag = true; } + + public void printConfigurationProperties() { + LOG.info("--------------------------------"); + LOG.info(""); + LOG.info("Ranger-TagSync Configuration: {\n"); + if (MapUtils.isNotEmpty(properties)) { + for (Map.Entry<Object, Object> entry : properties.entrySet()) { + LOG.info("\tProperty-Name:" + entry.getKey()); + LOG.info("\tProperty-Value:" + entry.getValue()); + LOG.info("\n"); + } + } + LOG.info("\n}"); + LOG.info(""); + LOG.info("--------------------------------"); + } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/tagsync/src/main/java/org/apache/ranger/sink/policymgr/TagRESTSink.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/sink/policymgr/TagRESTSink.java b/tagsync/src/main/java/org/apache/ranger/sink/policymgr/TagRESTSink.java index 98410d6..e934ce4 100644 --- a/tagsync/src/main/java/org/apache/ranger/sink/policymgr/TagRESTSink.java +++ b/tagsync/src/main/java/org/apache/ranger/sink/policymgr/TagRESTSink.java @@ -39,9 +39,6 @@ import java.util.List; import java.util.Map; import java.util.Properties; -/** - * Created by akulkarni on 9/11/15. - */ public class TagRESTSink implements TagSink { private static final Log LOG = LogFactory.getLog(TagRESTSink.class); @@ -57,8 +54,6 @@ public class TagRESTSink implements TagSink { private static final String REST_URL_TAG_RESOURCE = REST_PREFIX + MODULE_PREFIX + "/tag/" ; private static final String REST_URL_TAGRESOURCEMAP_IDS_RESOURCE = REST_PREFIX + MODULE_PREFIX + "/tagresourcemapids/"; private static final String REST_URL_IMPORT_SERVICETAGS_RESOURCE = REST_PREFIX + MODULE_PREFIX + "/importservicetags/"; - public static final String REST_URL_IMPORT_SERVICETAGS_PARAM = "op"; - private RangerRESTClient tagRESTClient = null; @@ -73,8 +68,8 @@ public class TagRESTSink implements TagSink { boolean ret = false; - String restUrl = TagSyncConfig.getPolicyMgrUrl(properties); - String sslConfigFile = TagSyncConfig.getPolicyMgrSslConfigFile(properties); + String restUrl = TagSyncConfig.getTagAdminRESTUrl(properties); + String sslConfigFile = TagSyncConfig.getTagAdminRESTSslConfigFile(properties); String userName = TagSyncConfig.getPolicyMgrUserName(properties); String password = TagSyncConfig.getPolicyMgrPassword(properties); @@ -267,6 +262,11 @@ public class TagRESTSink implements TagSink { } @Override + public List<Long> getTagIdsForResourceId(Long resourceId) throws Exception { + throw new Exception("Not implemented"); + } + + @Override public List<RangerTag> getTagsForResourceId(Long resourceId) throws Exception { throw new Exception("Not implemented"); }
