Repository: incubator-ranger Updated Branches: refs/heads/master 868c62b72 -> 74959da16
RANGER-1177: updated tagsync to support Atlas notification for hdfs_path entity-type Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/74959da1 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/74959da1 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/74959da1 Branch: refs/heads/master Commit: 74959da1617f65bdf74048ecd714dc3aa1d384e9 Parents: 868c62b Author: Madhan Neethiraj <[email protected]> Authored: Fri Sep 23 17:51:03 2016 -0700 Committer: Madhan Neethiraj <[email protected]> Committed: Mon Sep 26 14:48:39 2016 -0700 ---------------------------------------------------------------------- tagsync/pom.xml | 4 + .../source/atlas/AtlasHdfsResourceMapper.java | 98 ++++++++++ .../source/atlas/AtlasHiveResourceMapper.java | 188 +++++-------------- .../source/atlas/AtlasResourceMapper.java | 92 +++++++-- .../source/atlas/AtlasResourceMapperUtil.java | 15 +- .../tagsync/process/TestHdfsResourceMapper.java | 167 ++++++++++++++++ .../tagsync/process/TestHiveResourceMapper.java | 173 +++++++++++++++++ .../tagsync/process/TestTagSynchronizer.java | 31 --- 8 files changed, 572 insertions(+), 196 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/74959da1/tagsync/pom.xml ---------------------------------------------------------------------- diff --git a/tagsync/pom.xml b/tagsync/pom.xml index 0e3fb39..2e63502 100644 --- a/tagsync/pom.xml +++ b/tagsync/pom.xml @@ -36,6 +36,10 @@ <version>${junit.version}</version> </dependency> <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-core</artifactId> + </dependency> + <dependency> <groupId>log4j</groupId> <artifactId>log4j</artifactId> <version>${log4j.version}</version> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/74959da1/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasHdfsResourceMapper.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasHdfsResourceMapper.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasHdfsResourceMapper.java new file mode 100644 index 0000000..06bff90 --- /dev/null +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasHdfsResourceMapper.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.tagsync.source.atlas; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.atlas.typesystem.IReferenceableInstance; +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.fs.Path; +import org.apache.ranger.plugin.model.RangerPolicy; +import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource; +import org.apache.ranger.plugin.model.RangerServiceResource; + +public class AtlasHdfsResourceMapper extends AtlasResourceMapper { + public static final String ENTITY_TYPE_HDFS_PATH = "hdfs_path"; + public static final String RANGER_TYPE_HDFS_PATH = "path"; + + public static final String ENTITY_ATTRIBUTE_PATH = "path"; + public static final String ENTITY_ATTRIBUTE_CLUSTER_NAME = "clusterName"; + public static final String ENTITY_ATTRIBUTE_QUALIFIED_NAME = "qualifiedName"; + + public static final String[] SUPPORTED_ENTITY_TYPES = { ENTITY_TYPE_HDFS_PATH }; + + + public AtlasHdfsResourceMapper() { + super("hdfs", SUPPORTED_ENTITY_TYPES); + } + + @Override + public String getRangerServiceName(String clusterName) { + String ret = getCustomRangerServiceName(clusterName); + + if (StringUtils.isBlank(ret)) { + ret = clusterName + TAGSYNC_DEFAULT_CLUSTERNAME_AND_COMPONENTNAME_SEPARATOR + "hadoop"; + } + + return ret; + } + + @Override + public RangerServiceResource buildResource(final IReferenceableInstance entity) throws Exception { + String path = getEntityAttribute(entity, ENTITY_ATTRIBUTE_PATH, String.class); + String clusterName = getEntityAttribute(entity, ENTITY_ATTRIBUTE_CLUSTER_NAME, String.class); + String qualifiedName = getEntityAttribute(entity, ENTITY_ATTRIBUTE_QUALIFIED_NAME, String.class); + + if(StringUtils.isEmpty(path)) { + path = getResourceNameFromQualifiedName(qualifiedName); + + if(StringUtils.isEmpty(path)) { + throwExceptionWithMessage("path not found in attribute '" + ENTITY_ATTRIBUTE_PATH + "' or '" + ENTITY_ATTRIBUTE_QUALIFIED_NAME + "'"); + } + } + + if(StringUtils.isEmpty(clusterName)) { + clusterName = getClusterNameFromQualifiedName(qualifiedName); + + if(StringUtils.isEmpty(clusterName)) { + clusterName = defaultClusterName; + } + + if(StringUtils.isEmpty(clusterName)) { + throwExceptionWithMessage("attributes " + ENTITY_ATTRIBUTE_CLUSTER_NAME + ", " + ENTITY_ATTRIBUTE_QUALIFIED_NAME + "' not found in entity"); + } + } + + String entityGuid = entity.getId() != null ? entity.getId()._getId() : null; + String serviceName = getRangerServiceName(clusterName); + Boolean isExcludes = Boolean.FALSE; + Boolean isRecursive = Boolean.TRUE; + + Path pathObj = new Path(path); + + Map<String, RangerPolicyResource> elements = new HashMap<String, RangerPolicy.RangerPolicyResource>(); + elements.put(RANGER_TYPE_HDFS_PATH, new RangerPolicyResource(pathObj.toUri().getPath(), isExcludes, isRecursive)); + + RangerServiceResource ret = new RangerServiceResource(entityGuid, serviceName, elements); + + return ret; + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/74959da1/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasHiveResourceMapper.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasHiveResourceMapper.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasHiveResourceMapper.java index ca7b063..a359622 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasHiveResourceMapper.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasHiveResourceMapper.java @@ -19,177 +19,83 @@ package org.apache.ranger.tagsync.source.atlas; +import java.util.Map; +import java.util.HashMap; + import org.apache.atlas.typesystem.IReferenceableInstance; import org.apache.commons.lang.StringUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.ranger.plugin.model.RangerPolicy; +import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource; import org.apache.ranger.plugin.model.RangerServiceResource; -import java.util.Map; -import java.util.HashMap; -import java.util.List; -import java.util.ArrayList; -import java.util.Arrays; - public class AtlasHiveResourceMapper extends AtlasResourceMapper { - private static final Log LOG = LogFactory.getLog(AtlasHiveResourceMapper.class); - - public static final String COMPONENT_NAME = "hive"; - - public static final String ENTITY_TYPE_HIVE_DB = "hive_db"; - public static final String ENTITY_TYPE_HIVE_TABLE = "hive_table"; + public static final String ENTITY_TYPE_HIVE_DB = "hive_db"; + public static final String ENTITY_TYPE_HIVE_TABLE = "hive_table"; public static final String ENTITY_TYPE_HIVE_COLUMN = "hive_column"; - public static final String RANGER_TYPE_HIVE_DB = "database"; - public static final String RANGER_TYPE_HIVE_TABLE = "table"; + public static final String RANGER_TYPE_HIVE_DB = "database"; + public static final String RANGER_TYPE_HIVE_TABLE = "table"; public static final String RANGER_TYPE_HIVE_COLUMN = "column"; public static final String ENTITY_ATTRIBUTE_QUALIFIED_NAME = "qualifiedName"; + public static final String QUALIFIED_NAME_DELIMITER = "\\."; - public static final String TAGSYNC_DEFAULT_CLUSTERNAME_AND_COMPONENTNAME_SEPARATOR = "_"; - - public static final String clusterDelimiter = "@"; - - public static final String qualifiedNameDelimiter = "\\."; - - public static final String[] supportedEntityTypes = { ENTITY_TYPE_HIVE_DB, ENTITY_TYPE_HIVE_TABLE, ENTITY_TYPE_HIVE_COLUMN }; + public static final String[] SUPPORTED_ENTITY_TYPES = { ENTITY_TYPE_HIVE_DB, ENTITY_TYPE_HIVE_TABLE, ENTITY_TYPE_HIVE_COLUMN }; public AtlasHiveResourceMapper() { - super(); - } - - @Override - public List<String> getSupportedEntityTypes() { - return Arrays.asList(supportedEntityTypes); + super("hive", SUPPORTED_ENTITY_TYPES); } @Override public RangerServiceResource buildResource(final IReferenceableInstance entity) throws Exception { - - Map<String, RangerPolicy.RangerPolicyResource> elements = new HashMap<String, RangerPolicy.RangerPolicyResource>(); - - String serviceName = null; - - List<String> components = getQualifiedNameComponents(entity); - // components should contain qualifiedName, clusterName, dbName, tableName, columnName in that order - - String entityTypeName = entity.getTypeName(); - - String qualifiedName = components.get(0); - - String clusterName, dbName, tableName, columnName; - - if (components.size() > 1) { - clusterName = components.get(1); - serviceName = getRangerServiceName(clusterName); - } - - if (StringUtils.equals(entityTypeName, ENTITY_TYPE_HIVE_DB)) { - if (components.size() > 2) { - dbName = components.get(2); - RangerPolicy.RangerPolicyResource dbPolicyResource = new RangerPolicy.RangerPolicyResource(dbName); - elements.put(RANGER_TYPE_HIVE_DB, dbPolicyResource); - - } else { - LOG.error("invalid qualifiedName for HIVE_DB, qualifiedName=" + qualifiedName); - throw new Exception("invalid qualifiedName for HIVE_DB, qualifiedName=" + qualifiedName); - } - } else if (StringUtils.equals(entityTypeName, ENTITY_TYPE_HIVE_TABLE)) { - if (components.size() > 3) { - dbName = components.get(2); - tableName = components.get(3); - RangerPolicy.RangerPolicyResource dbPolicyResource = new RangerPolicy.RangerPolicyResource(dbName); - elements.put(RANGER_TYPE_HIVE_DB, dbPolicyResource); - RangerPolicy.RangerPolicyResource tablePolicyResource = new RangerPolicy.RangerPolicyResource(tableName); - elements.put(RANGER_TYPE_HIVE_TABLE, tablePolicyResource); - } else { - LOG.error("invalid qualifiedName for HIVE_TABLE, qualifiedName=" + qualifiedName); - throw new Exception("invalid qualifiedName for HIVE_TABLE, qualifiedName=" + qualifiedName); - } - } else if (StringUtils.equals(entityTypeName, ENTITY_TYPE_HIVE_COLUMN)) { - if (components.size() > 4) { - dbName = components.get(2); - tableName = components.get(3); - columnName = components.get(4); - RangerPolicy.RangerPolicyResource dbPolicyResource = new RangerPolicy.RangerPolicyResource(dbName); - elements.put(RANGER_TYPE_HIVE_DB, dbPolicyResource); - RangerPolicy.RangerPolicyResource tablePolicyResource = new RangerPolicy.RangerPolicyResource(tableName); - elements.put(RANGER_TYPE_HIVE_TABLE, tablePolicyResource); - RangerPolicy.RangerPolicyResource columnPolicyResource = new RangerPolicy.RangerPolicyResource(columnName); - elements.put(RANGER_TYPE_HIVE_COLUMN, columnPolicyResource); - } else { - LOG.error("invalid qualifiedName for HIVE_COLUMN, qualifiedName=" + qualifiedName); - throw new Exception("invalid qualifiedName for HIVE_COLUMN, qualifiedName=" + qualifiedName); - } - + String qualifiedName = getEntityAttribute(entity, ENTITY_ATTRIBUTE_QUALIFIED_NAME, String.class); + if (StringUtils.isEmpty(qualifiedName)) { + throw new Exception("attribute '" + ENTITY_ATTRIBUTE_QUALIFIED_NAME + "' not found in entity"); } - RangerServiceResource ret = new RangerServiceResource(); - - ret.setGuid(entity.getId()._getId()); - ret.setServiceName(serviceName); - ret.setResourceElements(elements); - - return ret; - } - - public String getRangerServiceName(String clusterName) { - String ret = getRangerServiceName(COMPONENT_NAME, clusterName); - - if (StringUtils.isBlank(ret)) { - ret = clusterName + TAGSYNC_DEFAULT_CLUSTERNAME_AND_COMPONENTNAME_SEPARATOR + COMPONENT_NAME; + String resourceStr = getResourceNameFromQualifiedName(qualifiedName); + if (StringUtils.isEmpty(resourceStr)) { + throwExceptionWithMessage("resource not found in attribute '" + ENTITY_ATTRIBUTE_QUALIFIED_NAME + "': " + qualifiedName); } - return ret; - } - - public final List<String> getQualifiedNameComponents(IReferenceableInstance entity) throws Exception { - - String qualifiedNameAttributeName = ENTITY_ATTRIBUTE_QUALIFIED_NAME; - String qualifiedName = getEntityAttribute(entity, qualifiedNameAttributeName, String.class); - if (StringUtils.isBlank(qualifiedName)) { - throw new Exception("Could not get a valid value for " + qualifiedNameAttributeName + " attribute from entity."); + String clusterName = getClusterNameFromQualifiedName(qualifiedName); + if (StringUtils.isEmpty(clusterName)) { + throwExceptionWithMessage("cluster-name not found in attribute '" + ENTITY_ATTRIBUTE_QUALIFIED_NAME + "': " + qualifiedName); } - if (LOG.isDebugEnabled()) { - LOG.debug("Received .... " + qualifiedNameAttributeName + "=" + qualifiedName + " for entity type " + entity.getTypeName()); - } + String entityType = entity.getTypeName(); + String entityGuid = entity.getId() != null ? entity.getId()._getId() : null; + String serviceName = getRangerServiceName(clusterName); + String[] resources = resourceStr.split(QUALIFIED_NAME_DELIMITER); + String dbName = resources.length > 0 ? resources[0] : null; + String tblName = resources.length > 1 ? resources[1] : null; + String colName = resources.length > 2 ? resources[2] : null; - List<String> ret = getQualifiedNameComponents(entity.getTypeName(), qualifiedName); + Map<String, RangerPolicyResource> elements = new HashMap<String, RangerPolicyResource>(); - if (LOG.isDebugEnabled()) { - LOG.debug("----- Entity-Id:" + entity.getId()._getId()); - LOG.debug("----- Entity-Type-Name:" + entity.getTypeName()); - LOG.debug("----- Entity-Components -----"); - int i = 0; - for (String value : ret) { - LOG.debug("----- Index:" + i++ + " Value:" + value); + if (StringUtils.equals(entityType, ENTITY_TYPE_HIVE_DB)) { + if (StringUtils.isNotEmpty(dbName)) { + elements.put(RANGER_TYPE_HIVE_DB, new RangerPolicyResource(dbName)); } - } - return ret; - } - - public final List<String> getQualifiedNameComponents(String entityTypeName, String qualifiedName) throws Exception { - - String components[] = qualifiedName.split(clusterDelimiter); - - if (components.length != 2) { - throw new Exception("Qualified Name does not contain cluster-name, qualifiedName=" + qualifiedName); + } else if (StringUtils.equals(entityType, ENTITY_TYPE_HIVE_TABLE)) { + if (StringUtils.isNotEmpty(dbName) && StringUtils.isNotEmpty(tblName)) { + elements.put(RANGER_TYPE_HIVE_DB, new RangerPolicyResource(dbName)); + elements.put(RANGER_TYPE_HIVE_TABLE, new RangerPolicyResource(tblName)); + } + } else if (StringUtils.equals(entityType, ENTITY_TYPE_HIVE_COLUMN)) { + if (StringUtils.isNotEmpty(dbName) && StringUtils.isNotEmpty(tblName) && StringUtils.isNotEmpty(colName)) { + elements.put(RANGER_TYPE_HIVE_DB, new RangerPolicyResource(dbName)); + elements.put(RANGER_TYPE_HIVE_TABLE, new RangerPolicyResource(tblName)); + elements.put(RANGER_TYPE_HIVE_COLUMN, new RangerPolicyResource(colName)); + } + } else { + throwExceptionWithMessage("unrecognized entity-type: " + entityType); } - if (LOG.isDebugEnabled()) { - LOG.debug("name-hierarchy=" + components[0] + ", cluster-name=" + components[1]); + if(elements.isEmpty()) { + throwExceptionWithMessage("invalid qualifiedName for entity-type '" + entityType + "': " + qualifiedName); } - String nameHierarchy[] = components[0].split(qualifiedNameDelimiter); - - List<String> ret = new ArrayList<String>(); - - ret.add(qualifiedName); - ret.add(components[1]); - - ret.addAll(Arrays.asList(nameHierarchy)); + RangerServiceResource ret = new RangerServiceResource(entityGuid, serviceName, elements); return ret; } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/74959da1/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasResourceMapper.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasResourceMapper.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasResourceMapper.java index d47d052..8ececdf 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasResourceMapper.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasResourceMapper.java @@ -19,44 +19,104 @@ package org.apache.ranger.tagsync.source.atlas; +import java.util.Properties; +import java.util.Map; + import org.apache.atlas.AtlasException; import org.apache.atlas.typesystem.IReferenceableInstance; +import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.ranger.plugin.model.RangerServiceResource; -import java.util.Properties; -import java.util.List; -import java.util.Map; - public abstract class AtlasResourceMapper { private static final Log LOG = LogFactory.getLog(AtlasResourceMapper.class); - protected static final String TAGSYNC_SERVICENAME_MAPPER_PROP_PREFIX = "ranger.tagsync.atlas."; + public static final String TAGSYNC_DEFAULT_CLUSTER_NAME = "ranger.tagsync.atlas.default.cluster.name"; - protected static final String TAGSYNC_SERVICENAME_MAPPER_PROP_SUFFIX = ".ranger.service"; + protected static final String TAGSYNC_SERVICENAME_MAPPER_PROP_PREFIX = "ranger.tagsync.atlas."; + protected static final String TAGSYNC_SERVICENAME_MAPPER_PROP_SUFFIX = ".ranger.service"; + protected static final String TAGSYNC_ATLAS_CLUSTER_IDENTIFIER = ".instance."; + protected static final String TAGSYNC_DEFAULT_CLUSTERNAME_AND_COMPONENTNAME_SEPARATOR = "_"; + protected static final String CLUSTER_DELIMITER = "@"; + + protected final String componentName; + protected final String[] supportedEntityTypes; - protected static final String TAGSYNC_ATLAS_CLUSTER_IDENTIFIER = ".instance."; protected Properties properties; + protected String defaultClusterName; - public AtlasResourceMapper() { + public AtlasResourceMapper(String componentName, String[] supportedEntityTypes) { + this.componentName = componentName; + this.supportedEntityTypes = supportedEntityTypes; } - public void initialize(Properties properties) { - this.properties = properties; + public final String getComponentName() { + return componentName; + } + + public final String[] getSupportedEntityTypes() { + return supportedEntityTypes; + } + + public String getRangerServiceName(String clusterName) { + String ret = getCustomRangerServiceName(clusterName); + + if (StringUtils.isBlank(ret)) { + ret = clusterName + TAGSYNC_DEFAULT_CLUSTERNAME_AND_COMPONENTNAME_SEPARATOR + componentName; + } + return ret; } - abstract public List<String> getSupportedEntityTypes(); + public void initialize(Properties properties) { + this.properties = properties; + this.defaultClusterName = properties != null ? properties.getProperty(TAGSYNC_DEFAULT_CLUSTER_NAME) : null; + } abstract public RangerServiceResource buildResource(final IReferenceableInstance entity) throws Exception; + protected String getCustomRangerServiceName(String atlasInstanceName) { + if(properties != null) { + String propName = TAGSYNC_SERVICENAME_MAPPER_PROP_PREFIX + componentName + + TAGSYNC_ATLAS_CLUSTER_IDENTIFIER + atlasInstanceName + + TAGSYNC_SERVICENAME_MAPPER_PROP_SUFFIX; + + return properties.getProperty(propName); + } else { + return null; + } + } + + protected String getResourceNameFromQualifiedName(String qualifiedName) { + if(StringUtils.isNotBlank(qualifiedName)) { + int idx = qualifiedName.lastIndexOf(CLUSTER_DELIMITER); + + if(idx != -1) { + return qualifiedName.substring(0, idx); + } else { + return qualifiedName; + } + } + + return null; + } + + protected String getClusterNameFromQualifiedName(String qualifiedName) { + if(StringUtils.isNotBlank(qualifiedName)) { + int idx = qualifiedName.lastIndexOf(CLUSTER_DELIMITER); + + if(idx != -1 && qualifiedName.length() > idx) { + return qualifiedName.substring(idx + 1); + } + } + + return null; + } - protected String getRangerServiceName(String componentName, String atlasInstanceName) { - String propName = TAGSYNC_SERVICENAME_MAPPER_PROP_PREFIX + componentName - + TAGSYNC_ATLAS_CLUSTER_IDENTIFIER + atlasInstanceName - + TAGSYNC_SERVICENAME_MAPPER_PROP_SUFFIX; + protected void throwExceptionWithMessage(String msg) throws Exception { + LOG.error(msg); - return properties.getProperty(propName); + throw new Exception(msg); } static protected <T> T getEntityAttribute(IReferenceableInstance entity, String name, Class<T> type) { http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/74959da1/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasResourceMapperUtil.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasResourceMapperUtil.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasResourceMapperUtil.java index f05d814..14b2001 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasResourceMapperUtil.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasResourceMapperUtil.java @@ -84,18 +84,17 @@ public class AtlasResourceMapperUtil { } boolean ret = true; - String allResourceMappers = "org.apache.ranger.tagsync.source.atlas.AtlasHiveResourceMapper"; + List<String> mapperNames = new ArrayList<String>(); + mapperNames.add("org.apache.ranger.tagsync.source.atlas.AtlasHiveResourceMapper"); + mapperNames.add("org.apache.ranger.tagsync.source.atlas.AtlasHdfsResourceMapper"); if (StringUtils.isNotBlank(customMapperNames)) { - allResourceMappers = allResourceMappers + MAPPER_NAME_DELIMIER + customMapperNames; + for(String customMapperName : customMapperNames.split(MAPPER_NAME_DELIMIER)) { + mapperNames.add(customMapperName.trim()); + } } - String[] mapperNamesArray = allResourceMappers.split(MAPPER_NAME_DELIMIER); - - List<String> mapperNames = Arrays.asList(mapperNamesArray); - for (String mapperName : mapperNames) { - mapperName = mapperName.trim(); try { Class clazz = Class.forName(mapperName); AtlasResourceMapper resourceMapper = (AtlasResourceMapper) clazz.newInstance(); @@ -113,7 +112,7 @@ public class AtlasResourceMapperUtil { } if (LOG.isDebugEnabled()) { - LOG.debug("<== initializeAtlasResourceMappers.initializeAtlasResourceMappers(" + allResourceMappers + "): " + ret); + LOG.debug("<== initializeAtlasResourceMappers.initializeAtlasResourceMappers(" + mapperNames + "): " + ret); } return ret; } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/74959da1/tagsync/src/test/java/org/apache/ranger/tagsync/process/TestHdfsResourceMapper.java ---------------------------------------------------------------------- diff --git a/tagsync/src/test/java/org/apache/ranger/tagsync/process/TestHdfsResourceMapper.java b/tagsync/src/test/java/org/apache/ranger/tagsync/process/TestHdfsResourceMapper.java new file mode 100644 index 0000000..0986331 --- /dev/null +++ b/tagsync/src/test/java/org/apache/ranger/tagsync/process/TestHdfsResourceMapper.java @@ -0,0 +1,167 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.tagsync.process; + +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +import org.apache.atlas.typesystem.IReferenceableInstance; +import org.apache.ranger.plugin.model.RangerServiceResource; +import org.apache.ranger.tagsync.source.atlas.AtlasHdfsResourceMapper; +import org.apache.ranger.tagsync.source.atlas.AtlasResourceMapper; +import org.junit.Test; + +import org.mockito.Mockito; +import org.junit.Assert; + + + +public class TestHdfsResourceMapper { + private static final String CLUSTER_NAME = "cl1"; + private static final String PATH = "hdfs://localhost:8020/user/testuser/finance"; + private static final String QUALIFIED_NAME = "hdfs://localhost:8020/user/testuser/finance@cl1"; + + private static final String SERVICE_NAME = "cl1_hadoop"; + private static final String RANGER_PATH = "/user/testuser/finance"; + + AtlasHdfsResourceMapper resourceMapper = new AtlasHdfsResourceMapper(); + AtlasHdfsResourceMapper resourceMapperWithDefaultClusterName = new AtlasHdfsResourceMapper(); + + { + Properties properties = new Properties(); + + properties.setProperty(AtlasResourceMapper.TAGSYNC_DEFAULT_CLUSTER_NAME, CLUSTER_NAME); + + resourceMapperWithDefaultClusterName.initialize(properties); + } + + @Test + public void testHdfsResourceFromPathAndClusterName() throws Exception { + Map<String, Object> entAttribs = new HashMap<String, Object>(); + + entAttribs.put(AtlasHdfsResourceMapper.ENTITY_ATTRIBUTE_PATH, PATH); + entAttribs.put(AtlasHdfsResourceMapper.ENTITY_ATTRIBUTE_CLUSTER_NAME, CLUSTER_NAME); + + IReferenceableInstance entity = getHdfsPathEntity(entAttribs); + RangerServiceResource resource = resourceMapper.buildResource(entity); + + assertServiceResource(resource); + } + + @Test + public void testHdfsResourceFromPathAndQualifiedName() throws Exception { + Map<String, Object> entAttribs = new HashMap<String, Object>(); + + entAttribs.put(AtlasHdfsResourceMapper.ENTITY_ATTRIBUTE_PATH, PATH); + entAttribs.put(AtlasHdfsResourceMapper.ENTITY_ATTRIBUTE_QUALIFIED_NAME, QUALIFIED_NAME); + + IReferenceableInstance entity = getHdfsPathEntity(entAttribs); + RangerServiceResource resource = resourceMapper.buildResource(entity); + + assertServiceResource(resource); + } + + @Test + public void testHdfsResourceFromClusterNameAndQualifiedName() throws Exception { + Map<String, Object> entAttribs = new HashMap<String, Object>(); + + entAttribs.put(AtlasHdfsResourceMapper.ENTITY_ATTRIBUTE_CLUSTER_NAME, CLUSTER_NAME); + entAttribs.put(AtlasHdfsResourceMapper.ENTITY_ATTRIBUTE_QUALIFIED_NAME, QUALIFIED_NAME); + + IReferenceableInstance entity = getHdfsPathEntity(entAttribs); + RangerServiceResource resource = resourceMapper.buildResource(entity); + + assertServiceResource(resource); + } + + @Test + public void testHdfsResourceFromPathAndClusterNameAndQualifiedName() throws Exception { + Map<String, Object> entAttribs = new HashMap<String, Object>(); + + entAttribs.put(AtlasHdfsResourceMapper.ENTITY_ATTRIBUTE_PATH, PATH); + entAttribs.put(AtlasHdfsResourceMapper.ENTITY_ATTRIBUTE_CLUSTER_NAME, CLUSTER_NAME); + entAttribs.put(AtlasHdfsResourceMapper.ENTITY_ATTRIBUTE_QUALIFIED_NAME, QUALIFIED_NAME); + + IReferenceableInstance entity = getHdfsPathEntity(entAttribs); + RangerServiceResource resource = resourceMapper.buildResource(entity); + + assertServiceResource(resource); + } + + @Test + public void testHdfsResourceFromQualifiedNameAndClusterNameFromDefault() throws Exception { + Map<String, Object> entAttribs = new HashMap<String, Object>(); + + entAttribs.put(AtlasHdfsResourceMapper.ENTITY_ATTRIBUTE_QUALIFIED_NAME, PATH); + + IReferenceableInstance entity = getHdfsPathEntity(entAttribs); + RangerServiceResource resource = resourceMapperWithDefaultClusterName.buildResource(entity); + + assertServiceResource(resource); + } + + @Test + public void testHdfsResourceFromPathAndClusterNameFromDefault() throws Exception { + Map<String, Object> entAttribs = new HashMap<String, Object>(); + + entAttribs.put(AtlasHdfsResourceMapper.ENTITY_ATTRIBUTE_PATH, PATH); + + IReferenceableInstance entity = getHdfsPathEntity(entAttribs); + RangerServiceResource resource = resourceMapperWithDefaultClusterName.buildResource(entity); + + assertServiceResource(resource); + } + + @Test + public void testHdfsResourceFromMissingAttribs() throws Exception { + Map<String, Object> entAttribs = new HashMap<String, Object>(); + + IReferenceableInstance entity = getHdfsPathEntity(entAttribs); + + try { + RangerServiceResource resource = resourceMapper.buildResource(entity); + + Assert.fail("expected exception. Found " + resource); + } catch(Exception excp) { + // ignore + } + } + + private IReferenceableInstance getHdfsPathEntity(Map<String, Object> entAttribs) throws Exception { + IReferenceableInstance entity = Mockito.mock(IReferenceableInstance.class); + + Mockito.when(entity.getTypeName()).thenReturn(AtlasHdfsResourceMapper.ENTITY_TYPE_HDFS_PATH); + Mockito.when(entity.getValuesMap()).thenReturn(entAttribs); + + return entity; + } + + private void assertServiceResource(RangerServiceResource resource) { + Assert.assertNotNull(resource); + Assert.assertEquals(SERVICE_NAME, resource.getServiceName()); + Assert.assertNotNull(resource.getResourceElements()); + Assert.assertEquals(1, resource.getResourceElements().size()); + Assert.assertTrue(resource.getResourceElements().containsKey(AtlasHdfsResourceMapper.RANGER_TYPE_HDFS_PATH)); + Assert.assertNotNull(resource.getResourceElements().get(AtlasHdfsResourceMapper.RANGER_TYPE_HDFS_PATH).getValues()); + Assert.assertEquals(1, resource.getResourceElements().get(AtlasHdfsResourceMapper.RANGER_TYPE_HDFS_PATH).getValues().size()); + Assert.assertEquals(RANGER_PATH, resource.getResourceElements().get(AtlasHdfsResourceMapper.RANGER_TYPE_HDFS_PATH).getValues().get(0)); + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/74959da1/tagsync/src/test/java/org/apache/ranger/tagsync/process/TestHiveResourceMapper.java ---------------------------------------------------------------------- diff --git a/tagsync/src/test/java/org/apache/ranger/tagsync/process/TestHiveResourceMapper.java b/tagsync/src/test/java/org/apache/ranger/tagsync/process/TestHiveResourceMapper.java new file mode 100644 index 0000000..aaf9590 --- /dev/null +++ b/tagsync/src/test/java/org/apache/ranger/tagsync/process/TestHiveResourceMapper.java @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.tagsync.process; + +import org.apache.atlas.typesystem.IReferenceableInstance; +import org.apache.ranger.plugin.model.RangerServiceResource; +import org.apache.ranger.tagsync.source.atlas.AtlasHiveResourceMapper; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; + +import java.util.HashMap; +import java.util.Map; + + +public class TestHiveResourceMapper { + private static final String DB_QUALIFIED_NAME = "default@cl1"; + private static final String TABLE_QUALIFIED_NAME = "default.testTable@cl1"; + private static final String COLUMN_QUALIFIED_NAME = "default.testTable.col1@cl1"; + + private static final String SERVICE_NAME = "cl1_hive"; + private static final String RANGER_DATABASE = "default"; + private static final String RANGER_TABLE = "testTable"; + private static final String RANGER_COLUMN = "col1"; + + AtlasHiveResourceMapper resourceMapper = new AtlasHiveResourceMapper(); + + @Test + public void testHiveDb() throws Exception { + Map<String, Object> entAttribs = new HashMap<String, Object>(); + + entAttribs.put(AtlasHiveResourceMapper.ENTITY_ATTRIBUTE_QUALIFIED_NAME, DB_QUALIFIED_NAME); + + IReferenceableInstance entity = getHiveDbEntity(entAttribs); + RangerServiceResource resource = resourceMapper.buildResource(entity); + + assertDbResource(resource); + } + + @Test + public void testHiveTable() throws Exception { + Map<String, Object> entAttribs = new HashMap<String, Object>(); + + entAttribs.put(AtlasHiveResourceMapper.ENTITY_ATTRIBUTE_QUALIFIED_NAME, TABLE_QUALIFIED_NAME); + + IReferenceableInstance entity = getHiveTableEntity(entAttribs); + RangerServiceResource resource = resourceMapper.buildResource(entity); + + assertTableResource(resource); + } + + @Test + public void testHiveColumn() throws Exception { + Map<String, Object> entAttribs = new HashMap<String, Object>(); + + entAttribs.put(AtlasHiveResourceMapper.ENTITY_ATTRIBUTE_QUALIFIED_NAME, COLUMN_QUALIFIED_NAME); + + IReferenceableInstance entity = getHiveColumnEntity(entAttribs); + RangerServiceResource resource = resourceMapper.buildResource(entity); + + assertColumnResource(resource); + } + + @Test + public void testHiveResourceFromMissingAttribs() throws Exception { + Map<String, Object> entAttribs = new HashMap<String, Object>(); + + IReferenceableInstance entity = getHiveDbEntity(entAttribs); + + try { + RangerServiceResource resource = resourceMapper.buildResource(entity); + + Assert.fail("expected exception. Found " + resource); + } catch(Exception excp) { + // ignore + } + } + + private IReferenceableInstance getHiveDbEntity(Map<String, Object> entAttribs) throws Exception { + IReferenceableInstance entity = Mockito.mock(IReferenceableInstance.class); + + Mockito.when(entity.getTypeName()).thenReturn(AtlasHiveResourceMapper.ENTITY_TYPE_HIVE_DB); + Mockito.when(entity.getValuesMap()).thenReturn(entAttribs); + + return entity; + } + + private IReferenceableInstance getHiveTableEntity(Map<String, Object> entAttribs) throws Exception { + IReferenceableInstance entity = Mockito.mock(IReferenceableInstance.class); + + Mockito.when(entity.getTypeName()).thenReturn(AtlasHiveResourceMapper.ENTITY_TYPE_HIVE_TABLE); + Mockito.when(entity.getValuesMap()).thenReturn(entAttribs); + + return entity; + } + + private IReferenceableInstance getHiveColumnEntity(Map<String, Object> entAttribs) throws Exception { + IReferenceableInstance entity = Mockito.mock(IReferenceableInstance.class); + + Mockito.when(entity.getTypeName()).thenReturn(AtlasHiveResourceMapper.ENTITY_TYPE_HIVE_COLUMN); + Mockito.when(entity.getValuesMap()).thenReturn(entAttribs); + + return entity; + } + + private void assertServiceResource(RangerServiceResource resource) { + Assert.assertNotNull(resource); + Assert.assertEquals(SERVICE_NAME, resource.getServiceName()); + Assert.assertNotNull(resource.getResourceElements()); + } + + private void assertDbResource(RangerServiceResource resource) { + assertServiceResource(resource); + + Assert.assertEquals(1, resource.getResourceElements().size()); + Assert.assertTrue(resource.getResourceElements().containsKey(AtlasHiveResourceMapper.RANGER_TYPE_HIVE_DB)); + Assert.assertNotNull(resource.getResourceElements().get(AtlasHiveResourceMapper.RANGER_TYPE_HIVE_DB).getValues()); + Assert.assertEquals(1, resource.getResourceElements().get(AtlasHiveResourceMapper.RANGER_TYPE_HIVE_DB).getValues().size()); + Assert.assertEquals(RANGER_DATABASE, resource.getResourceElements().get(AtlasHiveResourceMapper.RANGER_TYPE_HIVE_DB).getValues().get(0)); + } + + private void assertTableResource(RangerServiceResource resource) { + assertServiceResource(resource); + + Assert.assertEquals(2, resource.getResourceElements().size()); + Assert.assertTrue(resource.getResourceElements().containsKey(AtlasHiveResourceMapper.RANGER_TYPE_HIVE_DB)); + Assert.assertNotNull(resource.getResourceElements().get(AtlasHiveResourceMapper.RANGER_TYPE_HIVE_DB).getValues()); + Assert.assertEquals(1, resource.getResourceElements().get(AtlasHiveResourceMapper.RANGER_TYPE_HIVE_DB).getValues().size()); + Assert.assertEquals(RANGER_DATABASE, resource.getResourceElements().get(AtlasHiveResourceMapper.RANGER_TYPE_HIVE_DB).getValues().get(0)); + + Assert.assertTrue(resource.getResourceElements().containsKey(AtlasHiveResourceMapper.RANGER_TYPE_HIVE_TABLE)); + Assert.assertNotNull(resource.getResourceElements().get(AtlasHiveResourceMapper.RANGER_TYPE_HIVE_TABLE).getValues()); + Assert.assertEquals(1, resource.getResourceElements().get(AtlasHiveResourceMapper.RANGER_TYPE_HIVE_TABLE).getValues().size()); + Assert.assertEquals(RANGER_TABLE, resource.getResourceElements().get(AtlasHiveResourceMapper.RANGER_TYPE_HIVE_TABLE).getValues().get(0)); + } + + private void assertColumnResource(RangerServiceResource resource) { + assertServiceResource(resource); + + Assert.assertEquals(3, resource.getResourceElements().size()); + Assert.assertTrue(resource.getResourceElements().containsKey(AtlasHiveResourceMapper.RANGER_TYPE_HIVE_DB)); + Assert.assertNotNull(resource.getResourceElements().get(AtlasHiveResourceMapper.RANGER_TYPE_HIVE_DB).getValues()); + Assert.assertEquals(1, resource.getResourceElements().get(AtlasHiveResourceMapper.RANGER_TYPE_HIVE_DB).getValues().size()); + Assert.assertEquals(RANGER_DATABASE, resource.getResourceElements().get(AtlasHiveResourceMapper.RANGER_TYPE_HIVE_DB).getValues().get(0)); + + Assert.assertTrue(resource.getResourceElements().containsKey(AtlasHiveResourceMapper.RANGER_TYPE_HIVE_TABLE)); + Assert.assertNotNull(resource.getResourceElements().get(AtlasHiveResourceMapper.RANGER_TYPE_HIVE_TABLE).getValues()); + Assert.assertEquals(1, resource.getResourceElements().get(AtlasHiveResourceMapper.RANGER_TYPE_HIVE_TABLE).getValues().size()); + Assert.assertEquals(RANGER_TABLE, resource.getResourceElements().get(AtlasHiveResourceMapper.RANGER_TYPE_HIVE_TABLE).getValues().get(0)); + + Assert.assertTrue(resource.getResourceElements().containsKey(AtlasHiveResourceMapper.RANGER_TYPE_HIVE_COLUMN)); + Assert.assertNotNull(resource.getResourceElements().get(AtlasHiveResourceMapper.RANGER_TYPE_HIVE_COLUMN).getValues()); + Assert.assertEquals(1, resource.getResourceElements().get(AtlasHiveResourceMapper.RANGER_TYPE_HIVE_COLUMN).getValues().size()); + Assert.assertEquals(RANGER_COLUMN, resource.getResourceElements().get(AtlasHiveResourceMapper.RANGER_TYPE_HIVE_COLUMN).getValues().get(0)); + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/74959da1/tagsync/src/test/java/org/apache/ranger/tagsync/process/TestTagSynchronizer.java ---------------------------------------------------------------------- diff --git a/tagsync/src/test/java/org/apache/ranger/tagsync/process/TestTagSynchronizer.java b/tagsync/src/test/java/org/apache/ranger/tagsync/process/TestTagSynchronizer.java index c644e0d..13af2cb 100644 --- a/tagsync/src/test/java/org/apache/ranger/tagsync/process/TestTagSynchronizer.java +++ b/tagsync/src/test/java/org/apache/ranger/tagsync/process/TestTagSynchronizer.java @@ -20,13 +20,11 @@ package org.apache.ranger.tagsync.process; -import org.apache.ranger.tagsync.source.atlas.AtlasHiveResourceMapper; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; import java.io.*; -import java.util.List; import java.util.Properties; @@ -73,33 +71,4 @@ public class TestTagSynchronizer { assert(initDone); } - - @Test - public void testQualifiedNames() { - - List<String> components; - AtlasHiveResourceMapper hiveResourceBuilder = new AtlasHiveResourceMapper(); - try { - components = hiveResourceBuilder.getQualifiedNameComponents("hive_db", "database@cluster"); - printComponents(components); - - components = hiveResourceBuilder.getQualifiedNameComponents("hive_table", "database.table@cluster"); - printComponents(components); - - components = hiveResourceBuilder.getQualifiedNameComponents("hive_column", "database.table.column@cluster"); - printComponents(components); - - assert(true); - } catch (Exception e) { - System.out.println("Failed..."); - assert(false); - } - - } - private void printComponents(List<String> components) { - int i = 0; - for (String value : components) { - System.out.println("----- Index:" + i++ + " Value:" + value); - } - } }
