This is an automated email from the ASF dual-hosted git repository.
amestry pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/master by this push:
new 0ac4b12 ATLAS-4324: FS entity created for 'load data inpath' Part 2
0ac4b12 is described below
commit 0ac4b12be7c38ba34951da6bdbe82d23968cce13
Author: Ashutosh Mestry <[email protected]>
AuthorDate: Fri Jun 4 13:25:04 2021 -0700
ATLAS-4324: FS entity created for 'load data inpath' Part 2
---
.../java/org/apache/atlas/hive/hook/HiveHook.java | 2 +-
.../atlas/hive/hook/utils/ActiveEntityFilter.java | 24 ++++-
.../atlas/hive/hook/utils/HiveDDLEntityFilter.java | 26 ++---
.../hive/hook/utils/ActiveEntityFilterTest.java | 54 +++--------
.../src/test/resources/json/hs2-create-db-v2.json | 3 +-
...reate-db-with-no-pathentities-to-retain-v2.json | 30 ------
...2-create-db-with-no-pathentities-to-retain.json | 73 --------------
.../test/resources/json/hs2-create-process-v2.json | 3 -
.../test/resources/json/hs2-create-table-v2.json | 2 -
.../test/resources/json/hs2-load-inpath-v2.json | 108 ++++++++++++++++++---
...d-inpath-with-no-pathentities-to-retain-v2.json | 4 -
...load-inpath-with-no-pathentities-to-retain.json | 18 ----
.../src/test/resources/json/hs2-load-inpath.json | 108 ++++++++++++++++++---
.../test/resources/json/hs2-table-rename-v2.json | 6 +-
14 files changed, 238 insertions(+), 223 deletions(-)
diff --git
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
index 94ef225..37084d5 100644
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
@@ -70,7 +70,7 @@ public class HiveHook extends AtlasHook implements
ExecuteWithHookContext {
public static final String HOOK_HIVE_TABLE_PRUNE_PATTERN
= CONF_PREFIX + "hive_table.prune.pattern";
public static final String HOOK_HIVE_TABLE_CACHE_SIZE
= CONF_PREFIX + "hive_table.cache.size";
public static final String HOOK_HIVE_IGNORE_DDL_OPERATIONS
= CONF_PREFIX + "hs2.ignore.ddl.operations";
- public static final String HOOK_HIVE_FILTER_ENTITY_TYPES_TO_RETAIN
= CONF_PREFIX + "hs2.filter.entity.types.to.retain";
+ public static final String
HOOK_HIVE_FILTER_ENTITY_ADDITIONAL_TYPES_TO_RETAIN = CONF_PREFIX +
"hs2.filter.entity.additional.types.to.retain";
public static final String DEFAULT_HOST_NAME = "localhost";
private static final Map<String, HiveOperation> OPERATION_MAP = new
HashMap<>();
diff --git
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/utils/ActiveEntityFilter.java
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/utils/ActiveEntityFilter.java
index 737c637..0b0d4d6 100644
---
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/utils/ActiveEntityFilter.java
+++
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/utils/ActiveEntityFilter.java
@@ -18,11 +18,13 @@
package org.apache.atlas.hive.hook.utils;
import com.google.common.annotations.VisibleForTesting;
+import org.apache.atlas.hive.hook.HiveHook;
import org.apache.atlas.model.notification.HookNotification;
import org.apache.commons.configuration.Configuration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.Arrays;
import java.util.List;
import static
org.apache.atlas.hive.hook.HiveHook.HOOK_HIVE_IGNORE_DDL_OPERATIONS;
@@ -34,16 +36,32 @@ public class ActiveEntityFilter {
public static void init(Configuration configuration) {
boolean skipDdlOperations =
configuration.getBoolean(HOOK_HIVE_IGNORE_DDL_OPERATIONS, false);
- init(skipDdlOperations);
+ List<String> additionalTypesToRetain =
getConfiguredTypesToRetainForDDLEntityFilter(configuration);
+
+ init(skipDdlOperations, additionalTypesToRetain);
LOG.info("atlas.hook.hive.ignore.ddl.operations={} - {}",
skipDdlOperations, entityFilter.getClass().getSimpleName());
}
@VisibleForTesting
- static void init(boolean lineageOnlyFilter) {
- entityFilter = lineageOnlyFilter ? new HiveDDLEntityFilter() : new
PassthroughFilter();
+ static void init(boolean lineageOnlyFilter, List<String>
additionalTypesToRetain) {
+ entityFilter = lineageOnlyFilter ? new
HiveDDLEntityFilter(additionalTypesToRetain) : new PassthroughFilter();
}
public static List<HookNotification> apply(List<HookNotification>
incoming) {
return entityFilter.apply(incoming);
}
+
+ private static List<String>
getConfiguredTypesToRetainForDDLEntityFilter(Configuration configuration) {
+ try {
+ if
(configuration.containsKey(HiveHook.HOOK_HIVE_FILTER_ENTITY_ADDITIONAL_TYPES_TO_RETAIN))
{
+ String[] configuredTypes =
configuration.getStringArray(HiveHook.HOOK_HIVE_FILTER_ENTITY_ADDITIONAL_TYPES_TO_RETAIN);
+
+ return Arrays.asList(configuredTypes);
+ }
+ } catch (Exception e) {
+ LOG.error("Failed to load application properties", e);
+ }
+
+ return null;
+ }
}
diff --git
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/utils/HiveDDLEntityFilter.java
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/utils/HiveDDLEntityFilter.java
index 9163c47..3ce3942 100644
---
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/utils/HiveDDLEntityFilter.java
+++
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/utils/HiveDDLEntityFilter.java
@@ -18,8 +18,6 @@
package org.apache.atlas.hive.hook.utils;
import com.google.common.annotations.VisibleForTesting;
-import org.apache.atlas.ApplicationProperties;
-import org.apache.atlas.hive.hook.HiveHook;
import org.apache.atlas.hive.hook.events.BaseHiveEvent;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasObjectId;
@@ -43,7 +41,7 @@ import java.util.stream.Collectors;
public class HiveDDLEntityFilter implements EntityFilter {
private static final Logger LOG =
LoggerFactory.getLogger(HiveDDLEntityFilter.class);
- private static final Set<String> defaultPathTypes = new HashSet<String>()
{{
+ private static final Set<String> defaultPathTypesToRetain = new
HashSet<String>() {{
add(AtlasPathExtractorUtil.HDFS_TYPE_PATH);
add(AtlasPathExtractorUtil.ADLS_GEN2_DIRECTORY);
add(AtlasPathExtractorUtil.GCS_VIRTUAL_DIR);
@@ -57,20 +55,16 @@ public class HiveDDLEntityFilter implements EntityFilter {
add(BaseHiveEvent.HIVE_TYPE_COLUMN_LINEAGE);
add(BaseHiveEvent.HIVE_DB_DDL);
add(BaseHiveEvent.HIVE_TABLE_DDL);
- addAll(defaultPathTypes);
- addAll(getConfiguredTypesToRetain());
+ addAll(defaultPathTypesToRetain);
}};
- private static List<String> getConfiguredTypesToRetain() {
- String[] configuredTypesToRetain = null;
-
- try {
- configuredTypesToRetain =
ApplicationProperties.get().getStringArray(HiveHook.HOOK_HIVE_FILTER_ENTITY_TYPES_TO_RETAIN);
- } catch (Exception e) {
- LOG.error("Failed to load application properties", e);
+ public HiveDDLEntityFilter(List<String> additionalTypesToRetain) {
+ if (CollectionUtils.isEmpty(additionalTypesToRetain)) {
+ return;
}
- return configuredTypesToRetain != null ?
Arrays.asList(configuredTypesToRetain) : new ArrayList<>();
+ typesToRetain.addAll(additionalTypesToRetain);
+ LOG.info("Types retained: {}", typesToRetain.toArray());
}
public List<HookNotification> apply(List<HookNotification> incoming) {
@@ -197,7 +191,7 @@ public class HiveDDLEntityFilter implements EntityFilter {
AtlasObjectId oid = (AtlasObjectId) o;
String typeName = oid.getTypeName();
- if (oid.getUniqueAttributes() != null &&
!defaultPathTypes.contains(typeName)) {
+ if (oid.getUniqueAttributes() != null &&
!typesToRetain.contains(typeName)) {
oid.setGuid(null);
}
} else {
@@ -208,8 +202,8 @@ public class HiveDDLEntityFilter implements EntityFilter {
String typeName = hm.containsKey(AtlasObjectId.KEY_TYPENAME) ?
(String) hm.get(AtlasObjectId.KEY_TYPENAME) : null;
- if (hm.containsKey(BaseHiveEvent.ATTRIBUTE_UNIQUE_ATTRIBUTES) &&
!defaultPathTypes.contains(typeName)) {
- hm.put(BaseHiveEvent.ATTRIBUTE_GUID, null);
+ if (hm.containsKey(BaseHiveEvent.ATTRIBUTE_UNIQUE_ATTRIBUTES) &&
!typesToRetain.contains(typeName)) {
+ hm.remove(BaseHiveEvent.ATTRIBUTE_GUID);
}
}
}
diff --git
a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/utils/ActiveEntityFilterTest.java
b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/utils/ActiveEntityFilterTest.java
index 4dde1dc..a201214 100644
---
a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/utils/ActiveEntityFilterTest.java
+++
b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/utils/ActiveEntityFilterTest.java
@@ -21,13 +21,13 @@ import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.notification.HookNotification;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.utils.TestResourceFileUtils;
-import org.apache.commons.collections.MapUtils;
import org.testng.Assert;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
@@ -35,16 +35,17 @@ import java.util.Map;
import java.util.Set;
import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertNotEquals;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.fail;
public class ActiveEntityFilterTest {
- private static String FILE_SUFFIX = "-v2";
+ private static String FILE_SUFFIX_ACTUAL_RESULTS = "-v2";
+ private static String ADDITIONAL_TYPE_HDFS_PATH = "hdfs_path";
@BeforeClass
public void setup() {
- ActiveEntityFilter.init(true);
+ ActiveEntityFilter.init(true, Arrays.asList(new
String[]{ADDITIONAL_TYPE_HDFS_PATH}));
}
@Test
@@ -56,14 +57,12 @@ public class ActiveEntityFilterTest {
assertMessageFromFile("hs2-alter-view");
assertMessageFromFile("hs2-drop-table");
assertAtlasEntitiesWithExtInfoFromFile("hs2-create-process");
- assertAtlasEntitiesWithExtInfoFromFile("hs2-load-inpath");
-
assertAtlasEntitiesWithExtInfoFromFile("hs2-create-db-with-no-pathentities-to-retain",
false);
-
assertAtlasEntitiesWithExtInfoFromFile("hs2-load-inpath-with-no-pathentities-to-retain",
false);
+ assertMessageFromFile("hs2-load-inpath");
}
private void assertMessageFromFile(String msgFile) throws IOException {
List incoming = loadList(msgFile);
- List expected = loadList(msgFile + FILE_SUFFIX);
+ List expected = loadList(msgFile + FILE_SUFFIX_ACTUAL_RESULTS);
int expectedSize = expected.size();
List<HookNotification> actual =
ActiveEntityFilter.apply((List<HookNotification>) incoming);
@@ -139,48 +138,27 @@ public class ActiveEntityFilterTest {
}
private void assertAtlasEntitiesWithExtInfoFromFile(String entityFile)
throws IOException {
- assertAtlasEntitiesWithExtInfoFromFile(entityFile, true);
- }
-
- private void assertAtlasEntitiesWithExtInfoFromFile(String entityFile,
boolean retainPathEntities) throws IOException {
AtlasEntity.AtlasEntitiesWithExtInfo incoming =
TestResourceFileUtils.readObjectFromJson("", entityFile,
AtlasEntity.AtlasEntitiesWithExtInfo.class);
- AtlasEntity.AtlasEntitiesWithExtInfo expected =
TestResourceFileUtils.readObjectFromJson("", entityFile + FILE_SUFFIX,
AtlasEntity.AtlasEntitiesWithExtInfo.class);
+ AtlasEntity.AtlasEntitiesWithExtInfo expected =
TestResourceFileUtils.readObjectFromJson("", entityFile +
FILE_SUFFIX_ACTUAL_RESULTS, AtlasEntity.AtlasEntitiesWithExtInfo.class);
- HiveDDLEntityFilter hiveLineageEntityFilter = new
HiveDDLEntityFilter();
+ HiveDDLEntityFilter hiveLineageEntityFilter = new
HiveDDLEntityFilter(null);
AtlasEntity.AtlasEntitiesWithExtInfo actual =
hiveLineageEntityFilter.apply(incoming);
-
- if (retainPathEntities) {
- assertAtlasEntitiesWithExtInfo(actual, expected);
- } else {
- assertAtlasEntitiesWithNoPathEntitiesToRetain(actual, expected);
- }
+ assertAtlasEntitiesWithExtInfo(actual, expected);
}
private void
assertAtlasEntitiesWithExtInfo(AtlasEntity.AtlasEntitiesWithExtInfo actual,
AtlasEntity.AtlasEntitiesWithExtInfo expected) {
assertNotNull(actual);
assertNotNull(expected);
- if (expected.getEntities() != null && actual.getEntities() != null) {
- assertEquals(actual.getEntities().size(),
expected.getEntities().size());
- assertEntity(actual.getEntities(), expected.getEntities());
- }
-
- assertEquals(MapUtils.isEmpty(actual.getReferredEntities()),
MapUtils.isEmpty(expected.getReferredEntities()));
- if (expected.getReferredEntities() != null &&
actual.getReferredEntities() != null) {
- assertEntity(actual.getReferredEntities(),
expected.getReferredEntities());
- }
- }
-
- private void
assertAtlasEntitiesWithNoPathEntitiesToRetain(AtlasEntity.AtlasEntitiesWithExtInfo
actual, AtlasEntity.AtlasEntitiesWithExtInfo expected) {
- assertNotNull(actual);
- assertNotNull(expected);
+ assertEquals(actual.getEntities().size(),
expected.getEntities().size());
+ assertEntity(actual.getEntities(), expected.getEntities());
- if (expected.getEntities() != null && actual.getEntities() != null) {
- assertNotEquals(actual.getEntities().size(),
expected.getEntities().size());
+ if (expected.getReferredEntities() == null &&
actual.getReferredEntities() != null) {
+ fail("expected.getReferredEntities() == null, but
expected.getReferredEntities() != null");
}
if (expected.getReferredEntities() != null &&
actual.getReferredEntities() != null) {
- assertNotEquals(actual.getReferredEntities().size(),
expected.getReferredEntities().size());
+ assertEntity(actual.getReferredEntities(),
expected.getReferredEntities());
}
}
@@ -225,7 +203,7 @@ public class ActiveEntityFilterTest {
String actualJson = AtlasType.toJson(actualEntity);
String expectedJson = AtlasType.toJson(expectedEntity);
- assertEquals(AtlasType.fromJson(actualJson,
LinkedHashMap.class), AtlasType.fromJson(expectedJson, LinkedHashMap.class));
+ Assert.assertEquals(actualJson, expectedJson, "Actual: " +
actualJson);
}
}
}
diff --git a/addons/hive-bridge/src/test/resources/json/hs2-create-db-v2.json
b/addons/hive-bridge/src/test/resources/json/hs2-create-db-v2.json
index 881ee10..28d3b6b 100644
--- a/addons/hive-bridge/src/test/resources/json/hs2-create-db-v2.json
+++ b/addons/hive-bridge/src/test/resources/json/hs2-create-db-v2.json
@@ -11,7 +11,7 @@
"name": "create database cadb02",
"userName": "hive"
},
- "guid": "-14530890092409411",
+ "guid": "-14529329955589449",
"isIncomplete": false,
"provenanceType": 0,
"version": 0,
@@ -40,7 +40,6 @@
"version": 0,
"relationshipAttributes": {
"hiveDb": {
- "guid": "-14529329955589448",
"typeName": "hive_db",
"uniqueAttributes": {
"qualifiedName": "cadb02@cm"
diff --git
a/addons/hive-bridge/src/test/resources/json/hs2-create-db-with-no-pathentities-to-retain-v2.json
b/addons/hive-bridge/src/test/resources/json/hs2-create-db-with-no-pathentities-to-retain-v2.json
deleted file mode 100644
index 42553b5..0000000
---
a/addons/hive-bridge/src/test/resources/json/hs2-create-db-with-no-pathentities-to-retain-v2.json
+++ /dev/null
@@ -1,30 +0,0 @@
-{
- "referredEntities": {},
- "entities": [
- {
- "typeName": "hive_db_ddl",
- "attributes": {
- "serviceType": "hive",
- "qualifiedName": "cadb02@cm:1616450673617",
- "execTime": 1616450673617,
- "queryText": "create database cadb02",
- "name": "create database cadb02",
- "userName": "hive"
- },
- "guid": "-14530890092409411",
- "isIncomplete": false,
- "provenanceType": 0,
- "version": 0,
- "relationshipAttributes": {
- "db": {
- "typeName": "hive_db",
- "uniqueAttributes": {
- "qualifiedName": "cadb02@cm"
- },
- "relationshipType": "hive_db_ddl_queries"
- }
- },
- "proxy": false
- }
- ]
-}
diff --git
a/addons/hive-bridge/src/test/resources/json/hs2-create-db-with-no-pathentities-to-retain.json
b/addons/hive-bridge/src/test/resources/json/hs2-create-db-with-no-pathentities-to-retain.json
deleted file mode 100644
index a5b810f..0000000
---
a/addons/hive-bridge/src/test/resources/json/hs2-create-db-with-no-pathentities-to-retain.json
+++ /dev/null
@@ -1,73 +0,0 @@
-{
- "referredEntities": {},
- "entities": [
- {
- "typeName": "hive_db",
- "attributes": {
- "owner": "hive",
- "ownerType": "USER",
- "managedLocation": null,
- "qualifiedName": "cadb02@cm",
- "clusterName": "cm",
- "name": "cadb02",
- "location":
"hdfs://ve0126.halxg.cloudera.com:8020/warehouse/tablespace/external/hive/cadb02.db",
- "parameters": {}
- },
- "guid": "-14529329955589448",
- "isIncomplete": false,
- "provenanceType": 0,
- "version": 0,
- "proxy": false
- },
- {
- "typeName": "hive_db_ddl",
- "attributes": {
- "serviceType": "hive",
- "qualifiedName": "cadb02@cm:1616450673617",
- "execTime": 1616450673617,
- "queryText": "create database cadb02",
- "name": "create database cadb02",
- "userName": "hive"
- },
- "guid": "-14529329955589449",
- "isIncomplete": false,
- "provenanceType": 0,
- "version": 0,
- "relationshipAttributes": {
- "db": {
- "guid": "-14529329955589448",
- "typeName": "hive_db",
- "uniqueAttributes": {
- "qualifiedName": "cadb02@cm"
- },
- "relationshipType": "hive_db_ddl_queries"
- }
- },
- "proxy": false
- },
- {
- "typeName": "hdfs_path",
- "attributes": {
- "path":
"hdfs://ve0126.halxg.cloudera.com:8020/warehouse/tablespace/external/hive/cadb02.db",
- "qualifiedName":
"hdfs://ve0126.halxg.cloudera.com:8020/warehouse/tablespace/external/hive/cadb02.db@cm",
- "clusterName": "cm",
- "name": "/warehouse/tablespace/external/hive/cadb02.db"
- },
- "guid": "-14529329955589450",
- "isIncomplete": false,
- "provenanceType": 0,
- "version": 0,
- "relationshipAttributes": {
- "hiveDb": {
- "guid": "-14529329955589448",
- "typeName": "hive_db",
- "uniqueAttributes": {
- "qualifiedName": "cadb02@cm"
- },
- "relationshipType": "hive_db_location"
- }
- },
- "proxy": false
- }
- ]
-}
diff --git
a/addons/hive-bridge/src/test/resources/json/hs2-create-process-v2.json
b/addons/hive-bridge/src/test/resources/json/hs2-create-process-v2.json
index 8e55b72..9291cde 100644
--- a/addons/hive-bridge/src/test/resources/json/hs2-create-process-v2.json
+++ b/addons/hive-bridge/src/test/resources/json/hs2-create-process-v2.json
@@ -17,7 +17,6 @@
"version": 0,
"relationshipAttributes": {
"table": {
- "guid": "-44808597128610",
"typeName": "hive_table",
"uniqueAttributes": {
"qualifiedName": "cadb202.vw202@primary"
@@ -51,7 +50,6 @@
"relationshipAttributes": {
"outputs": [
{
- "guid": "-44808597128610",
"typeName": "hive_table",
"uniqueAttributes": {
"qualifiedName": "cadb202.vw202@primary"
@@ -112,7 +110,6 @@
"relationshipAttributes": {
"outputs": [
{
- "guid": "-44808597128612",
"typeName": "hive_column",
"uniqueAttributes": {
"qualifiedName": "cadb202.vw202.col202@primary"
diff --git
a/addons/hive-bridge/src/test/resources/json/hs2-create-table-v2.json
b/addons/hive-bridge/src/test/resources/json/hs2-create-table-v2.json
index 801918e..ebf9e51 100644
--- a/addons/hive-bridge/src/test/resources/json/hs2-create-table-v2.json
+++ b/addons/hive-bridge/src/test/resources/json/hs2-create-table-v2.json
@@ -40,7 +40,6 @@
"relationshipAttributes": {
"outputs": [
{
- "guid": "-14529329955589452",
"typeName": "hive_table",
"uniqueAttributes": {
"qualifiedName": "cadb02.hh6@cm"
@@ -103,7 +102,6 @@
"version": 0,
"relationshipAttributes": {
"table": {
- "guid": "-14529329955589452",
"typeName": "hive_table",
"uniqueAttributes": {
"qualifiedName": "cadb02.hh6@cm"
diff --git a/addons/hive-bridge/src/test/resources/json/hs2-load-inpath-v2.json
b/addons/hive-bridge/src/test/resources/json/hs2-load-inpath-v2.json
index dd31aa0..499a9c2 100644
--- a/addons/hive-bridge/src/test/resources/json/hs2-load-inpath-v2.json
+++ b/addons/hive-bridge/src/test/resources/json/hs2-load-inpath-v2.json
@@ -1,18 +1,96 @@
-{
- "referredEntities": {
- "-14529329955589455": {
- "typeName": "hdfs_path",
- "attributes": {
- "path": "hdfs://ve0126.halxg.cloudera.com:8020/tmp/external/hh6.csv",
- "qualifiedName":
"hdfs://ve0126.halxg.cloudera.com:8020/tmp/external/hh6.csv@cm",
- "clusterName": "cm",
- "name": "/tmp/external/hh6.csv"
+[
+ {
+ "type": "ENTITY_CREATE_V2",
+ "user": "hive",
+ "entities": {
+ "referredEntities": {
+ "-98504074851374": {
+ "typeName": "hdfs_path",
+ "attributes": {
+ "path":
"file:/Users/hive/Apache/atlas-wip/addons/hive-bridge/target/load-data-thi5jt1lgj",
+ "qualifiedName":
"file:/users/hive/apache/atlas-wip/addons/hive-bridge/target/load-data-thi5jt1lgj",
+ "clusterName": "primary",
+ "name":
"/Users/hive/Apache/atlas-wip/addons/hive-bridge/target/load-data-thi5jt1lgj"
+ },
+ "guid": "-98504074851374",
+ "isIncomplete": false,
+ "provenanceType": 0,
+ "version": 0,
+ "proxy": false
+ }
},
- "guid": "-14529329955589455",
- "isIncomplete": false,
- "provenanceType": 0,
- "version": 0,
- "proxy": false
+ "entities": [
+ {
+ "typeName": "hive_process",
+ "attributes": {
+ "recentQueries": [
+ "load data local inpath
'file:///users/hive/apache/atlas-wip/addons/hive-bridge/target/load-data-thi5jt1lgj'
into table table_nrx8uoggc0 partition(dt = '2015-01-01')"
+ ],
+ "qualifiedName":
"LOAD->:default.table_nrx8uoggc0@primary:1622738598000",
+ "clusterName": "primary",
+ "name": "LOAD->:default.table_nrx8uoggc0@primary:1622738598000",
+ "queryText": "",
+ "operationType": "LOAD",
+ "startTime": 1622738659471,
+ "queryPlan": "Not Supported",
+ "endTime": 1622738659471,
+ "userName": "",
+ "queryId": ""
+ },
+ "guid": "-98504074851381",
+ "isIncomplete": false,
+ "provenanceType": 0,
+ "version": 0,
+ "relationshipAttributes": {
+ "outputs": [
+ {
+ "typeName": "hive_table",
+ "uniqueAttributes": {
+ "qualifiedName": "default.table_nrx8uoggc0@primary"
+ },
+ "relationshipType": "process_dataset_outputs"
+ }
+ ],
+ "inputs": [
+ {
+ "guid": "-98504074851374",
+ "typeName": "hdfs_path",
+ "uniqueAttributes": {
+ "qualifiedName":
"file:/users/hive/apache/atlas-wip/addons/hive-bridge/target/load-data-thi5jt1lgj"
+ },
+ "relationshipType": "dataset_process_inputs"
+ }
+ ]
+ },
+ "proxy": false
+ },
+ {
+ "typeName": "hive_process_execution",
+ "attributes": {
+ "hostName": "21806.local",
+ "qualifiedName":
"LOAD->:default.table_nrx8uoggc0@primary:1622738598000:1622738658982:1622738659471",
+ "name":
"LOAD->:default.table_nrx8uoggc0@primary:1622738598000:1622738658982:1622738659471",
+ "queryText": "load data local inpath
'file:///users/hive/apache/atlas-wip/addons/hive-bridge/target/load-data-thi5jt1lgj'
into table table_nrx8uoggc0 partition(dt = '2015-01-01')",
+ "startTime": 1622738658982,
+ "queryPlan": "Not Supported",
+ "endTime": 1622738659471,
+ "userName": "hive",
+ "queryId":
"hive_20210603094308_ef789483-7de1-462b-ac74-bb0ebe7aeedf"
+ },
+ "guid": "-98504074851382",
+ "isIncomplete": false,
+ "provenanceType": 0,
+ "version": 0,
+ "relationshipAttributes": {
+ "process": {
+ "guid": "-98504074851381",
+ "typeName": "hive_process",
+ "relationshipType": "hive_process_process_executions"
+ }
+ },
+ "proxy": false
+ }
+ ]
}
}
-}
\ No newline at end of file
+]
\ No newline at end of file
diff --git
a/addons/hive-bridge/src/test/resources/json/hs2-load-inpath-with-no-pathentities-to-retain-v2.json
b/addons/hive-bridge/src/test/resources/json/hs2-load-inpath-with-no-pathentities-to-retain-v2.json
deleted file mode 100644
index 7f90d19..0000000
---
a/addons/hive-bridge/src/test/resources/json/hs2-load-inpath-with-no-pathentities-to-retain-v2.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
- "referredEntities": {
- }
-}
\ No newline at end of file
diff --git
a/addons/hive-bridge/src/test/resources/json/hs2-load-inpath-with-no-pathentities-to-retain.json
b/addons/hive-bridge/src/test/resources/json/hs2-load-inpath-with-no-pathentities-to-retain.json
deleted file mode 100644
index dd31aa0..0000000
---
a/addons/hive-bridge/src/test/resources/json/hs2-load-inpath-with-no-pathentities-to-retain.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
- "referredEntities": {
- "-14529329955589455": {
- "typeName": "hdfs_path",
- "attributes": {
- "path": "hdfs://ve0126.halxg.cloudera.com:8020/tmp/external/hh6.csv",
- "qualifiedName":
"hdfs://ve0126.halxg.cloudera.com:8020/tmp/external/hh6.csv@cm",
- "clusterName": "cm",
- "name": "/tmp/external/hh6.csv"
- },
- "guid": "-14529329955589455",
- "isIncomplete": false,
- "provenanceType": 0,
- "version": 0,
- "proxy": false
- }
- }
-}
\ No newline at end of file
diff --git a/addons/hive-bridge/src/test/resources/json/hs2-load-inpath.json
b/addons/hive-bridge/src/test/resources/json/hs2-load-inpath.json
index dd31aa0..499a9c2 100644
--- a/addons/hive-bridge/src/test/resources/json/hs2-load-inpath.json
+++ b/addons/hive-bridge/src/test/resources/json/hs2-load-inpath.json
@@ -1,18 +1,96 @@
-{
- "referredEntities": {
- "-14529329955589455": {
- "typeName": "hdfs_path",
- "attributes": {
- "path": "hdfs://ve0126.halxg.cloudera.com:8020/tmp/external/hh6.csv",
- "qualifiedName":
"hdfs://ve0126.halxg.cloudera.com:8020/tmp/external/hh6.csv@cm",
- "clusterName": "cm",
- "name": "/tmp/external/hh6.csv"
+[
+ {
+ "type": "ENTITY_CREATE_V2",
+ "user": "hive",
+ "entities": {
+ "referredEntities": {
+ "-98504074851374": {
+ "typeName": "hdfs_path",
+ "attributes": {
+ "path":
"file:/Users/hive/Apache/atlas-wip/addons/hive-bridge/target/load-data-thi5jt1lgj",
+ "qualifiedName":
"file:/users/hive/apache/atlas-wip/addons/hive-bridge/target/load-data-thi5jt1lgj",
+ "clusterName": "primary",
+ "name":
"/Users/hive/Apache/atlas-wip/addons/hive-bridge/target/load-data-thi5jt1lgj"
+ },
+ "guid": "-98504074851374",
+ "isIncomplete": false,
+ "provenanceType": 0,
+ "version": 0,
+ "proxy": false
+ }
},
- "guid": "-14529329955589455",
- "isIncomplete": false,
- "provenanceType": 0,
- "version": 0,
- "proxy": false
+ "entities": [
+ {
+ "typeName": "hive_process",
+ "attributes": {
+ "recentQueries": [
+ "load data local inpath
'file:///users/hive/apache/atlas-wip/addons/hive-bridge/target/load-data-thi5jt1lgj'
into table table_nrx8uoggc0 partition(dt = '2015-01-01')"
+ ],
+ "qualifiedName":
"LOAD->:default.table_nrx8uoggc0@primary:1622738598000",
+ "clusterName": "primary",
+ "name": "LOAD->:default.table_nrx8uoggc0@primary:1622738598000",
+ "queryText": "",
+ "operationType": "LOAD",
+ "startTime": 1622738659471,
+ "queryPlan": "Not Supported",
+ "endTime": 1622738659471,
+ "userName": "",
+ "queryId": ""
+ },
+ "guid": "-98504074851381",
+ "isIncomplete": false,
+ "provenanceType": 0,
+ "version": 0,
+ "relationshipAttributes": {
+ "outputs": [
+ {
+ "typeName": "hive_table",
+ "uniqueAttributes": {
+ "qualifiedName": "default.table_nrx8uoggc0@primary"
+ },
+ "relationshipType": "process_dataset_outputs"
+ }
+ ],
+ "inputs": [
+ {
+ "guid": "-98504074851374",
+ "typeName": "hdfs_path",
+ "uniqueAttributes": {
+ "qualifiedName":
"file:/users/hive/apache/atlas-wip/addons/hive-bridge/target/load-data-thi5jt1lgj"
+ },
+ "relationshipType": "dataset_process_inputs"
+ }
+ ]
+ },
+ "proxy": false
+ },
+ {
+ "typeName": "hive_process_execution",
+ "attributes": {
+ "hostName": "21806.local",
+ "qualifiedName":
"LOAD->:default.table_nrx8uoggc0@primary:1622738598000:1622738658982:1622738659471",
+ "name":
"LOAD->:default.table_nrx8uoggc0@primary:1622738598000:1622738658982:1622738659471",
+ "queryText": "load data local inpath
'file:///users/hive/apache/atlas-wip/addons/hive-bridge/target/load-data-thi5jt1lgj'
into table table_nrx8uoggc0 partition(dt = '2015-01-01')",
+ "startTime": 1622738658982,
+ "queryPlan": "Not Supported",
+ "endTime": 1622738659471,
+ "userName": "hive",
+ "queryId":
"hive_20210603094308_ef789483-7de1-462b-ac74-bb0ebe7aeedf"
+ },
+ "guid": "-98504074851382",
+ "isIncomplete": false,
+ "provenanceType": 0,
+ "version": 0,
+ "relationshipAttributes": {
+ "process": {
+ "guid": "-98504074851381",
+ "typeName": "hive_process",
+ "relationshipType": "hive_process_process_executions"
+ }
+ },
+ "proxy": false
+ }
+ ]
}
}
-}
\ No newline at end of file
+]
\ No newline at end of file
diff --git
a/addons/hive-bridge/src/test/resources/json/hs2-table-rename-v2.json
b/addons/hive-bridge/src/test/resources/json/hs2-table-rename-v2.json
index f2b2bf7..f133e7f 100644
--- a/addons/hive-bridge/src/test/resources/json/hs2-table-rename-v2.json
+++ b/addons/hive-bridge/src/test/resources/json/hs2-table-rename-v2.json
@@ -8,13 +8,13 @@
"typeName": "hive_table_ddl",
"attributes": {
"serviceType": "hive",
- "qualifiedName": "cadb02.hh6_renamed@cm:1616452234545",
- "execTime": 1616452234545,
+ "qualifiedName": "cadb02.hh6_renamed@cm:1616450674247",
+ "execTime": 1616450674247,
"queryText": "ALTER TABLE hh6 RENAME TO hh6_renamed",
"name": "ALTER TABLE hh6 RENAME TO hh6_renamed",
"userName": "hive"
},
- "guid": "-14530890092409428",
+ "guid": "-14529329955589467",
"isIncomplete": false,
"provenanceType": 0,
"version": 0,
