This is an automated email from the ASF dual-hosted git repository.

sarath pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git


The following commit(s) were added to refs/heads/master by this push:
     new 4e6339d  ATLAS-3197: Disable creation of hive process, hive process 
execution and ddl query entities for HMS hook
4e6339d is described below

commit 4e6339d8721e23bec7aabaafd28412497df2bb17
Author: Le Ma <[email protected]>
AuthorDate: Thu May 9 16:40:40 2019 -0700

    ATLAS-3197: Disable creation of hive process, hive process execution and 
ddl query entities for HMS hook
---
 .../atlas/hive/hook/events/AlterTableRename.java   |  8 ++-
 .../atlas/hive/hook/events/CreateHiveProcess.java  | 10 +++-
 .../apache/atlas/hive/hook/events/CreateTable.java | 11 ++--
 .../org/apache/atlas/hive/hook/HiveHookIT.java     | 68 +++++++++++++++++-----
 .../atlas/hive/hook/HiveMetastoreHookIT.java       | 34 ++++++++---
 5 files changed, 98 insertions(+), 33 deletions(-)

diff --git 
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/AlterTableRename.java
 
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/AlterTableRename.java
index 67ea527..927ddc0 100644
--- 
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/AlterTableRename.java
+++ 
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/AlterTableRename.java
@@ -148,10 +148,12 @@ public class AlterTableRename extends BaseHiveEvent {
         ret.add(new EntityPartialUpdateRequestV2(getUserName(), oldTableId, 
renamedTableEntity));
 
         // partial update relationship attribute ddl
-        AtlasEntity ddlEntity = 
createHiveDDLEntity(renamedTableEntity.getEntity(), true);
+        if (!context.isMetastoreHook()) {
+            AtlasEntity ddlEntity = 
createHiveDDLEntity(renamedTableEntity.getEntity(), true);
 
-        if (ddlEntity != null) {
-            ret.add(new HookNotification.EntityCreateRequestV2(getUserName(), 
new AtlasEntitiesWithExtInfo(ddlEntity)));
+            if (ddlEntity != null) {
+                ret.add(new 
HookNotification.EntityCreateRequestV2(getUserName(), new 
AtlasEntitiesWithExtInfo(ddlEntity)));
+            }
         }
 
         context.removeFromKnownTable(oldTableQualifiedName);
diff --git 
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateHiveProcess.java
 
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateHiveProcess.java
index d1511c0..38b53b6 100644
--- 
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateHiveProcess.java
+++ 
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateHiveProcess.java
@@ -108,6 +108,14 @@ public class CreateHiveProcess extends BaseHiveEvent {
                     if (entity != null) {
                         outputs.add(entity);
                     }
+
+                    if (entity != null && !context.isMetastoreHook()) {
+                        AtlasEntity ddlEntity = createHiveDDLEntity(entity);
+
+                        if (ddlEntity != null) {
+                            ret.addEntity(ddlEntity);
+                        }
+                    }
                 }
             }
 
@@ -121,7 +129,7 @@ public class CreateHiveProcess extends BaseHiveEvent {
                 }
             }
 
-            if (!skipProcess) {
+            if (!skipProcess && !context.isMetastoreHook()) {
                 AtlasEntity process = getHiveProcessEntity(inputs, outputs);
 
                 ret.addEntity(process);
diff --git 
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateTable.java
 
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateTable.java
index ef4e687..aedb155 100644
--- 
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateTable.java
+++ 
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateTable.java
@@ -117,7 +117,7 @@ public class CreateTable extends BaseHiveEvent {
         if (table != null) {
             AtlasEntity tblEntity = toTableEntity(table, ret);
 
-            if (tblEntity != null) {
+            if (tblEntity != null && !context.isMetastoreHook()) {
                 if (isHBaseStore(table)) {
                     // This create lineage to HBase table in case of Hive on 
HBase
                     AtlasEntity hbaseTableEntity = 
toReferencedHBaseTable(table, ret);
@@ -134,7 +134,6 @@ public class CreateTable extends BaseHiveEvent {
 
                         AtlasEntity processExecution = 
getHiveProcessExecutionEntity(processEntity);
                         ret.addEntity(processExecution);
-
                     }
                 } else {
                     if (EXTERNAL_TABLE.equals(table.getTableType())) {
@@ -148,12 +147,12 @@ public class CreateTable extends BaseHiveEvent {
                         ret.addEntity(processExecution);
                     }
                 }
-            }
 
-            AtlasEntity tableDDLEntity = createHiveDDLEntity(tblEntity);
+                AtlasEntity tableDDLEntity = createHiveDDLEntity(tblEntity);
 
-            if (tableDDLEntity != null) {
-                ret.addEntity(tableDDLEntity);
+                if (tableDDLEntity != null) {
+                    ret.addEntity(tableDDLEntity);
+                }
             }
         }
     }
diff --git 
a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java 
b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
index 28c600e..d19470f 100755
--- 
a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
+++ 
b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
@@ -128,7 +128,7 @@ public class HiveHookIT extends HiveITBase {
         String      tableId   = assertTableIsRegistered(dbName, tableName);
         String      colId     = 
assertColumnIsRegistered(HiveMetaStoreBridge.getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME,
 dbName, tableName), colName)); //there is only one instance of column 
registered
         AtlasEntity colEntity = 
atlasClientV2.getEntityByGuid(colId).getEntity();
-        AtlasEntity tblEntity = 
atlasClientV2.getEntityByGuid(colId).getEntity();
+        AtlasEntity tblEntity = 
atlasClientV2.getEntityByGuid(tableId).getEntity();
 
         Assert.assertEquals(colEntity.getAttribute(ATTRIBUTE_QUALIFIED_NAME), 
String.format("%s.%s.%s@%s", dbName.toLowerCase(), tableName.toLowerCase(), 
colName.toLowerCase(), CLUSTER_NAME));
         Assert.assertNotNull(colEntity.getAttribute(ATTRIBUTE_TABLE));
@@ -195,14 +195,19 @@ public class HiveHookIT extends HiveITBase {
     //ATLAS-1321: Disable problematic tests. Need to revisit and fix them later
     @Test(enabled = false)
     public void testCreateExternalTable() throws Exception {
-        String tableName = tableName();
-        String colName   = columnName();
-        String pFile     = createTestDFSPath("parentPath");
-        String query     = String.format("create EXTERNAL table %s.%s(%s, %s) 
location '%s'", DEFAULT_DB , tableName , colName + " int", "name string",  
pFile);
+        String tableName     = tableName();
+        String colName       = columnName();
+        String pFile         = createTestDFSPath("parentPath");
+        String query         = String.format("create EXTERNAL table %s.%s(%s, 
%s) location '%s'", DEFAULT_DB , tableName , colName + " int", "name string",  
pFile);
 
         runCommand(query);
 
-        assertTableIsRegistered(DEFAULT_DB, tableName, null, true);
+        String tblId         = assertTableIsRegistered(DEFAULT_DB, tableName, 
null, true);
+        AtlasEntity tblEnity = 
atlasClientV2.getEntityByGuid(tblId).getEntity();
+        List ddlList         = (List) 
tblEnity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES);
+
+        assertNotNull(ddlList);
+        assertEquals(ddlList.size(), 1);
 
         String processId = 
assertEntityIsRegistered(HiveDataTypes.HIVE_PROCESS.getName(), 
ATTRIBUTE_QUALIFIED_NAME, getTableProcessQualifiedName(DEFAULT_DB, tableName), 
null);
 
@@ -381,15 +386,20 @@ public class HiveHookIT extends HiveITBase {
                 BaseHiveEvent.ATTRIBUTE_PROCESS));
         Assert.assertEquals(process.getGuid(), processEntity1.getGuid());
 
-        String           drpquery         = String.format("drop table %s ", 
ctasTableName);
+        String           dropQuery         = String.format("drop table %s ", 
ctasTableName);
 
-        runCommandWithDelay(drpquery, 100);
+        runCommandWithDelay(dropQuery, 5000);
 
         assertTableIsNotRegistered(DEFAULT_DB, ctasTableName);
 
         runCommand(query);
 
-        assertTableIsRegistered(DEFAULT_DB, ctasTableName);
+        String tblId          = assertTableIsRegistered(DEFAULT_DB, 
ctasTableName);
+        AtlasEntity tblEntity = 
atlasClientV2.getEntityByGuid(tblId).getEntity();
+        List ddlList          = (List) 
tblEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES);
+
+        assertNotNull(ddlList);
+        assertEquals(ddlList.size(), 1);
 
         outputs =  getOutputs(ctasTableName, Entity.Type.TABLE);
 
@@ -423,6 +433,13 @@ public class HiveHookIT extends HiveITBase {
         Assert.assertEquals(process1.getGuid(), processEntity1.getGuid());
         Assert.assertEquals(numberOfProcessExecutions(processEntity1), 1);
         assertTableIsRegistered(DEFAULT_DB, viewName);
+
+        String viewId          = assertTableIsRegistered(DEFAULT_DB, viewName);
+        AtlasEntity viewEntity = 
atlasClientV2.getEntityByGuid(viewId).getEntity();
+        List ddlQueries        = (List) 
viewEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES);
+
+        Assert.assertNotNull(ddlQueries);
+        Assert.assertEquals(ddlQueries.size(), 1);
     }
 
     @Test
@@ -476,8 +493,15 @@ public class HiveHookIT extends HiveITBase {
         Assert.assertEquals(processEntity1.getGuid(), 
processEntity2.getGuid());
 
         String table2Id = assertTableIsRegistered(DEFAULT_DB, table2Name);
+        String viewId2  = assertTableIsRegistered(DEFAULT_DB, viewName);
 
-        Assert.assertEquals(assertTableIsRegistered(DEFAULT_DB, viewName), 
viewId);
+        Assert.assertEquals(viewId2, viewId);
+
+        AtlasEntity viewEntity = 
atlasClientV2.getEntityByGuid(viewId2).getEntity();
+        List ddlQueries        = (List) 
viewEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES);
+
+        Assert.assertNotNull(ddlQueries);
+        Assert.assertEquals(ddlQueries.size(), 2);
 
         datasetName = HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, 
DEFAULT_DB, viewName);
 
@@ -745,7 +769,13 @@ public class HiveHookIT extends HiveITBase {
         Assert.assertEquals(process.getGuid(), hiveProcess.getGuid());
         Assert.assertEquals(numberOfProcessExecutions(hiveProcess), 1);
 
-        assertTableIsRegistered(DEFAULT_DB, tableName);
+        String tblId          = assertTableIsRegistered(DEFAULT_DB, tableName);
+
+        AtlasEntity tblEntity = 
atlasClientV2.getEntityByGuid(tblId).getEntity();
+        List ddlQueries       = (List) 
tblEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES);
+
+        Assert.assertNotNull(ddlQueries);
+        Assert.assertEquals(ddlQueries.size(), 1);
     }
 
     @Test
@@ -797,7 +827,13 @@ public class HiveHookIT extends HiveITBase {
 
         runCommandWithDelay(query, 1000);
 
-        assertTableIsRegistered(DEFAULT_DB, tableName);
+        String tblId          = assertTableIsRegistered(DEFAULT_DB, tableName);
+
+        AtlasEntity tblEntity = 
atlasClientV2.getEntityByGuid(tblId).getEntity();
+        List ddlQueries       = (List) 
tblEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES);
+
+        Assert.assertNotNull(ddlQueries);
+        Assert.assertEquals(ddlQueries.size(), 1);
 
         Set<WriteEntity> p3Outputs = new LinkedHashSet<WriteEntity>() {{
             addAll(getOutputs(pFile2, Entity.Type.DFS_DIR));
@@ -1308,9 +1344,9 @@ public class HiveHookIT extends HiveITBase {
         assertEquals(columns.size(), 1);
         assertEquals(columns.get(0).getAttribute(NAME), "name");
 
-        String tblId = assertTableIsRegistered(DEFAULT_DB, tableName);
+        String tblId          = assertTableIsRegistered(DEFAULT_DB, tableName);
         AtlasEntity tblEntity = 
atlasClientV2.getEntityByGuid(tblId).getEntity();
-        List ddlQueries = (List) 
tblEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES);
+        List ddlQueries       = (List) 
tblEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES);
 
         Assert.assertNotNull(ddlQueries);
         Assert.assertEquals(ddlQueries.size(), 2);
@@ -1430,7 +1466,7 @@ public class HiveHookIT extends HiveITBase {
         List        ddlQueries3 = (List) 
tblEntity3.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES);
 
         Assert.assertNotNull(ddlQueries3);
-        Assert.assertEquals(ddlQueries3.size(), 4);
+        Assert.assertEquals(ddlQueries3.size(), 5);
 
         //Change col position again
         oldColName = "name4";
@@ -1466,7 +1502,7 @@ public class HiveHookIT extends HiveITBase {
         List        ddlQueries4 = (List) 
tblEntity4.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES);
 
         Assert.assertNotNull(ddlQueries4);
-        Assert.assertEquals(ddlQueries4.size(), 5);
+        Assert.assertEquals(ddlQueries4.size(), 6);
     }
 
     /**
diff --git 
a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveMetastoreHookIT.java
 
b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveMetastoreHookIT.java
index fdd65eb..a07e193 100755
--- 
a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveMetastoreHookIT.java
+++ 
b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveMetastoreHookIT.java
@@ -26,9 +26,10 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.testng.annotations.Test;
 
-import java.util.Collection;
+import java.util.List;
 import java.util.Map;
 
+import static 
org.apache.atlas.hive.hook.events.BaseHiveEvent.ATTRIBUTE_DDL_QUERIES;
 import static org.apache.atlas.model.instance.AtlasEntity.Status.ACTIVE;
 import static org.apache.atlas.model.instance.AtlasEntity.Status.DELETED;
 import static org.testng.AssertJUnit.*;
@@ -42,10 +43,10 @@ public class HiveMetastoreHookIT extends HiveITBase {
         String query  = "CREATE DATABASE " + dbName;
 
         runCommand(query);
-        String dbId = assertDatabaseIsRegistered(dbName);
-
+        String dbId          = assertDatabaseIsRegistered(dbName);
         AtlasEntity dbEntity = getAtlasEntity(dbId);
-        assertNotNull(dbEntity);
+
+        assertEquals(((List) 
dbEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES)).size(), 0);
     }
 
     @Test (priority = 2)
@@ -66,6 +67,7 @@ public class HiveMetastoreHookIT extends HiveITBase {
         dbEntity = getAtlasEntity(dbId);
         Map parameters = (Map) dbEntity.getAttribute("parameters");
 
+        assertEquals(((List) 
dbEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES)).size(), 0);
         assertNotNull(parameters);
         assertEquals(2, parameters.size());
 
@@ -75,6 +77,8 @@ public class HiveMetastoreHookIT extends HiveITBase {
 
         dbEntity = getAtlasEntity(dbId);
 
+
+        assertEquals(((List) 
dbEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES)).size(), 0);
         assertEquals(dbEntity.getAttribute("owner"), "hive");
         assertEquals(dbEntity.getAttribute("ownerType"), "USER");
 
@@ -86,6 +90,8 @@ public class HiveMetastoreHookIT extends HiveITBase {
 
         dbEntity = getAtlasEntity(dbId);
 
+        assertEquals(((List) 
dbEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES)).size(), 0);
+
         String location = (String) dbEntity.getAttribute("location");
         assertEquals(location, hdfsPath);
     }
@@ -146,9 +152,12 @@ public class HiveMetastoreHookIT extends HiveITBase {
         String dbId = assertDatabaseIsRegistered(dbName);
         assertEquals(getAtlasEntity(dbId).getStatus(), ACTIVE);
 
-        String tableName = tableName();
+        String tableName      = tableName();
         runCommand("CREATE TABLE " + dbName + "." + tableName + " (name 
string, age int, dob date)");
-        String tblId = assertTableIsRegistered(dbName, tableName);
+        String tblId          = assertTableIsRegistered(dbName, tableName);
+        AtlasEntity tblEntity = getAtlasEntity(tblId);
+
+        assertEquals(((List) 
tblEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES)).size(), 0);
         assertEquals(getAtlasEntity(tblId).getStatus(), ACTIVE);
     }
 
@@ -167,8 +176,13 @@ public class HiveMetastoreHookIT extends HiveITBase {
         assertEquals(getAtlasEntity(tblId).getStatus(), ACTIVE);
 
         String viewName = tableName();
+
         runCommand("CREATE VIEW " + dbName + "." + viewName + " AS SELECT * 
FROM " + dbName + "." + tableName);
-        tblId = assertTableIsRegistered(dbName, viewName);
+
+        tblId                 = assertTableIsRegistered(dbName, viewName);
+        AtlasEntity tblEntity = getAtlasEntity(tblId);
+
+        assertEquals(((List) 
tblEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES)).size(), 0);
         assertEquals(getAtlasEntity(tblId).getStatus(), ACTIVE);
     }
 
@@ -180,11 +194,13 @@ public class HiveMetastoreHookIT extends HiveITBase {
         runCommand(query);
         String dbId = assertDatabaseIsRegistered(dbName);
         assertEquals(getAtlasEntity(dbId).getStatus(), ACTIVE);
+        assertEquals(((List) 
getAtlasEntity(dbId).getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES)).size(), 
0);
 
         String tableName = tableName();
         runCommand("CREATE TABLE " + dbName + "." + tableName + " (name 
string, age int, dob date)");
         String tblId = assertTableIsRegistered(dbName, tableName);
         assertEquals(getAtlasEntity(tblId).getStatus(), ACTIVE);
+        assertEquals(((List) 
getAtlasEntity(tblId).getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES)).size(), 
0);
 
         // SET TBLPROPERTIES
         query = "ALTER TABLE " + dbName + "." + tableName + " SET 
TBLPROPERTIES (\"prop1\"=\"val1\", \"prop2\"=\"val2\", \"prop3\"=\"val3\")";
@@ -216,6 +232,7 @@ public class HiveMetastoreHookIT extends HiveITBase {
         assertEquals(serdeAttrs.get("serializationLib"), 
"org.apache.hadoop.hive.ql.io.orc.OrcSerde");
         assertEquals(((Map) serdeAttrs.get("parameters")).get("prop1"), 
"val1");
         assertEquals(((Map) serdeAttrs.get("parameters")).get("prop2"), 
"val2");
+        assertEquals(((List) 
tableEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES)).size(), 0);
     }
 
     @Test (priority = 8)
@@ -240,6 +257,8 @@ public class HiveMetastoreHookIT extends HiveITBase {
         AtlasEntityWithExtInfo tableEntityWithExtInfo = 
getAtlasEntityWithExtInfo(tblId);
         AtlasEntity            tableEntity            = 
tableEntityWithExtInfo.getEntity();
 
+        assertEquals(((List) 
tableEntity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES)).size(), 0);
+
         // validate table rename in table entity
         assertEquals(newTableName, tableEntity.getAttribute("name"));
         assertTrue(((String) 
tableEntity.getAttribute("qualifiedName")).contains(newTableName));
@@ -284,6 +303,7 @@ public class HiveMetastoreHookIT extends HiveITBase {
         AtlasEntity col2Entity = getAtlasEntity(col2Id);
         assertEquals(col2Entity.getAttribute("name"), "col22");
         assertEquals(col2Entity.getAttribute("type"), "string");
+        assertEquals(((List) 
getAtlasEntity(tblId).getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES)).size(), 
0);
     }
 
     @Test (priority = 10)

Reply via email to