Repository: incubator-atlas Updated Branches: refs/heads/master 9d1040b7c -> 6f241db4a
ATLAS-877 CreateTime of an entity(for example hive table) changes for every update (sumasai) Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/6f241db4 Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/6f241db4 Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/6f241db4 Branch: refs/heads/master Commit: 6f241db4aeeba896c00f0b447fd269f0cee804da Parents: 9d1040b Author: Suma Shivaprasad <[email protected]> Authored: Fri Jun 17 11:08:50 2016 -0700 Committer: Suma Shivaprasad <[email protected]> Committed: Fri Jun 17 11:08:50 2016 -0700 ---------------------------------------------------------------------- .../org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java | 8 ++++---- .../test/java/org/apache/atlas/hive/hook/HiveHookIT.java | 9 +++++++-- release-log.txt | 1 + 3 files changed, 12 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6f241db4/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java index c1940a6..c956a32 100755 --- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java +++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java @@ -44,7 +44,6 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Order; import org.apache.hadoop.hive.metastore.api.SerDeInfo; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; -import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Table; @@ -73,7 +72,7 @@ public class HiveMetaStoreBridge { public static final String TEMP_TABLE_PREFIX = "_temp-"; private final String clusterName; - public static final int MILLIS_CONVERT_FACTOR = 1000; + public static final long MILLIS_CONVERT_FACTOR = 1000; public static final String ATLAS_ENDPOINT = "atlas.rest.address"; @@ -403,9 +402,10 @@ public class HiveMetaStoreBridge { tableReference.set(HiveDataModelGenerator.OWNER, hiveTable.getOwner()); Date createDate = new Date(); - if (hiveTable.getMetadata().getProperty(hive_metastoreConstants.DDL_TIME) != null){ + if (hiveTable.getTTable() != null){ try { - createDate = new Date(Long.parseLong(hiveTable.getMetadata().getProperty(hive_metastoreConstants.DDL_TIME)) * MILLIS_CONVERT_FACTOR); + createDate = new Date(hiveTable.getTTable().getCreateTime() * MILLIS_CONVERT_FACTOR); + LOG.debug("Setting create time to {} ", createDate); tableReference.set(HiveDataModelGenerator.CREATE_TIME, createDate); } catch(NumberFormatException ne) { LOG.error("Error while updating createTime for the table {} ", hiveTable.getCompleteName(), ne); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6f241db4/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java index f5904d6..e7fbf71 100755 --- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java +++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java @@ -711,7 +711,11 @@ public class HiveHookIT { String tableName = createTable(true); final String newDBName = createDatabase(); - assertTableIsRegistered(DEFAULT_DB, tableName); + String tableId = assertTableIsRegistered(DEFAULT_DB, tableName); + Referenceable tableEntity = atlasClient.getEntity(tableId); + final String createTime = (String)tableEntity.get(HiveDataModelGenerator.CREATE_TIME); + Assert.assertNotNull(createTime); + String columnGuid = assertColumnIsRegistered(HiveMetaStoreBridge.getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, tableName), NAME)); String sdGuid = assertSDIsRegistered(HiveMetaStoreBridge.getStorageDescQFName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, tableName)), null); assertDatabaseIsRegistered(newDBName); @@ -728,7 +732,7 @@ public class HiveHookIT { final String newTableName = tableName(); String query = String.format("alter table %s rename to %s", DEFAULT_DB + "." + tableName, newDBName + "." + newTableName); - runCommand(query); + runCommandWithDelay(query, 1000); String newColGuid = assertColumnIsRegistered(HiveMetaStoreBridge.getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, newDBName, newTableName), NAME)); Assert.assertEquals(newColGuid, columnGuid); @@ -750,6 +754,7 @@ public class HiveHookIT { Referenceable sd = ((Referenceable) entity.get(HiveDataModelGenerator.STORAGE_DESC)); String location = (String) sd.get(HiveDataModelGenerator.LOCATION); assertTrue(location.contains(newTableName)); + Assert.assertEquals(entity.get(HiveDataModelGenerator.CREATE_TIME), createTime); } }); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6f241db4/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index e4256bf..b033e99 100644 --- a/release-log.txt +++ b/release-log.txt @@ -24,6 +24,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags) ALL CHANGES: +ATLAS-877 CreateTime of an entity(for example hive table) changes for every update (sumasai) ATLAS-642 import-hive should create the lineage for external tables (svimal2106 via sumasai) ATLAS-901 Log messages that cannot be sent to Kafka to a specific log configuration (yhemanth) ATLAS-911 Get entity by unique attribute doesn't enforce type (shwethags)
