Repository: atlas Updated Branches: refs/heads/master 4152bc6d5 -> c2be0646d
ATLAS-2444: fix for IT failures Signed-off-by: Madhan Neethiraj <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/c2be0646 Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/c2be0646 Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/c2be0646 Branch: refs/heads/master Commit: c2be0646d95b7b38ea3b06382091e0e5bcf9988a Parents: 4152bc6 Author: apoorvnaik <[email protected]> Authored: Wed Feb 14 21:58:31 2018 -0800 Committer: Madhan Neethiraj <[email protected]> Committed: Thu Feb 15 16:23:24 2018 -0800 ---------------------------------------------------------------------- .../apache/atlas/hive/bridge/HiveMetaStoreBridge.java | 7 ++++++- .../src/test/java/org/apache/atlas/hive/HiveITBase.java | 12 ++++++++++-- .../java/org/apache/atlas/hive/hook/HiveHookIT.java | 5 ++--- .../org/apache/atlas/utils/HdfsNameServiceResolver.java | 7 ++++--- 4 files changed, 22 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/atlas/blob/c2be0646/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java index 3a775b6..4047c16 100755 --- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java +++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java @@ -615,7 +615,12 @@ public class HiveMetaStoreBridge { ref.set("nameServiceId", nameServiceID); } else { ref.set("path", pathUri); - ref.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, getHdfsPathQualifiedName(clusterName, pathUri)); + // Only append clusterName for the HDFS path + if (pathUri.startsWith(HdfsNameServiceResolver.HDFS_SCHEME)) { + ref.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, getHdfsPathQualifiedName(clusterName, pathUri)); + } else { + ref.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, pathUri); + } } ref.set(AtlasConstants.CLUSTER_NAME_ATTRIBUTE, clusterName); return ref; http://git-wip-us.apache.org/repos/asf/atlas/blob/c2be0646/addons/hive-bridge/src/test/java/org/apache/atlas/hive/HiveITBase.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/HiveITBase.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/HiveITBase.java index 01418cf..217c6aa 100644 --- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/HiveITBase.java +++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/HiveITBase.java @@ -231,13 +231,21 @@ public class HiveITBase { Referenceable hdfsPathRef = atlasClient.getEntity(hdfsPathId); Assert.assertEquals(hdfsPathRef.get("path"), testPathNormed); Assert.assertEquals(hdfsPathRef.get(NAME), Path.getPathWithoutSchemeAndAuthority(path).toString().toLowerCase()); - Assert.assertEquals(hdfsPathRef.get(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME), testPathNormed); + if (testPathNormed != null) { + Assert.assertTrue(((String)hdfsPathRef.get(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME)).startsWith(testPathNormed)); + } } } private String assertHDFSPathIsRegistered(String path) throws Exception { LOG.debug("Searching for hdfs path {}", path); - return assertEntityIsRegistered(HiveMetaStoreBridge.HDFS_PATH, AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, path, null); + // ATLAS-2444 HDFS name node federation adds the cluster name to the qualifiedName + if (path.startsWith("hdfs://")) { + String pathWithCluster = path + "@" + CLUSTER_NAME; + return assertEntityIsRegistered(HiveMetaStoreBridge.HDFS_PATH, AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, pathWithCluster, null); + } else { + return assertEntityIsRegistered(HiveMetaStoreBridge.HDFS_PATH, AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, path, null); + } } protected String assertDatabaseIsRegistered(String dbName) throws Exception { http://git-wip-us.apache.org/repos/asf/atlas/blob/c2be0646/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java index 1b1ebec..f96d47d 100755 --- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java +++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java @@ -263,11 +263,10 @@ public class HiveHookIT extends HiveITBase { Iterator<? extends Entity> iterator = expectedTables.iterator(); for(int i = 0; i < expectedTables.size(); i++) { Entity hiveEntity = iterator.next(); - if (Entity.Type.TABLE.equals(hiveEntity.getType()) || - Entity.Type.DFS_DIR.equals(hiveEntity.getType())) { + if (Entity.Type.TABLE.equals(hiveEntity.getType()) || Entity.Type.DFS_DIR.equals(hiveEntity.getType())) { Referenceable entity = atlasClient.getEntity(tableRef.get(i)._getId()); LOG.debug("Validating output {} {} ", i, entity); - Assert.assertEquals(entity.get(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME), hiveEntity.getName()); + assertTrue(((String)entity.get(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME)).startsWith(hiveEntity.getName())); } } } http://git-wip-us.apache.org/repos/asf/atlas/blob/c2be0646/common/src/main/java/org/apache/atlas/utils/HdfsNameServiceResolver.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/atlas/utils/HdfsNameServiceResolver.java b/common/src/main/java/org/apache/atlas/utils/HdfsNameServiceResolver.java index 60e1925..7f2d611 100644 --- a/common/src/main/java/org/apache/atlas/utils/HdfsNameServiceResolver.java +++ b/common/src/main/java/org/apache/atlas/utils/HdfsNameServiceResolver.java @@ -32,8 +32,9 @@ import java.util.Objects; public class HdfsNameServiceResolver { private static final Logger LOG = LoggerFactory.getLogger(HdfsNameServiceResolver.class); + public static final String HDFS_SCHEME = "hdfs://"; + private static final int DEFAULT_PORT = 8020; - private static final String HDFS_SCHEME = "hdfs://"; private static final String HDFS_NAMESERVICE_PROPERTY_KEY = "dfs.nameservices"; private static final String HDFS_INTERNAL_NAMESERVICE_PROPERTY_KEY = "dfs.internal.nameservices"; private static final String HDFS_NAMENODES_HA_NODES_PREFIX = "dfs.ha.namenodes."; @@ -83,7 +84,7 @@ public class HdfsNameServiceResolver { String ret = path; // Only handle URLs that begin with hdfs:// - if (path.indexOf(HDFS_SCHEME) == 0) { + if (path != null && path.indexOf(HDFS_SCHEME) == 0) { URI uri = new Path(path).toUri(); String nsId; @@ -114,7 +115,7 @@ public class HdfsNameServiceResolver { String ret = ""; // Only handle path URLs that begin with hdfs:// - if (path.indexOf(HDFS_SCHEME) == 0) { + if (path != null && path.indexOf(HDFS_SCHEME) == 0) { try { URI uri = new Path(path).toUri();
