Repository: incubator-atlas Updated Branches: refs/heads/master 40ee94921 -> 922a83c9a
ATLAS-532 Change Data types of all timestamps in Hive model(currently long)(sumasai via yhemanth) Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/922a83c9 Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/922a83c9 Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/922a83c9 Branch: refs/heads/master Commit: 922a83c9a10e857d54855463225e9a5c375bc2b9 Parents: 40ee949 Author: Suma Shivaprasad <[email protected]> Authored: Mon Apr 18 11:57:22 2016 -0700 Committer: Suma Shivaprasad <[email protected]> Committed: Mon Apr 18 11:57:22 2016 -0700 ---------------------------------------------------------------------- .../atlas/hive/bridge/HiveMetaStoreBridge.java | 21 +++++++++++-- .../org/apache/atlas/hive/hook/HiveHook.java | 5 +-- .../hive/model/HiveDataModelGenerator.java | 11 ++++--- .../org/apache/atlas/hive/hook/HiveHookIT.java | 33 +++++++++++++++++++- release-log.txt | 5 +-- 5 files changed, 63 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/922a83c9/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java index 6eb5194..a28b4ac 100755 --- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java +++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java @@ -50,6 +50,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.ArrayList; +import java.util.Date; import java.util.List; /** @@ -63,8 +64,9 @@ public class HiveMetaStoreBridge { public static final String DESCRIPTION_ATTR = "description"; public static final String TABLE_TYPE_ATTR = "tableType"; public static final String SEARCH_ENTRY_GUID_ATTR = "__guid"; - public static final String LAST_ACCESS_TIME_ATTR = "lastAccessTime"; + private final String clusterName; + public static final int MILLIS_CONVERT_FACTOR = 1000; public static final String ATLAS_ENDPOINT = "atlas.rest.address"; @@ -303,8 +305,21 @@ public class HiveMetaStoreBridge { tableReference.set(HiveDataModelGenerator.TABLE_NAME, hiveTable.getTableName().toLowerCase()); tableReference.set("owner", hiveTable.getOwner()); - tableReference.set("createTime", hiveTable.getMetadata().getProperty(hive_metastoreConstants.DDL_TIME)); - tableReference.set("lastAccessTime", hiveTable.getLastAccessTime()); + Date createDate = new Date(); + if (hiveTable.getMetadata().getProperty(hive_metastoreConstants.DDL_TIME) != null){ + try { + createDate = new Date(Long.parseLong(hiveTable.getMetadata().getProperty(hive_metastoreConstants.DDL_TIME)) * MILLIS_CONVERT_FACTOR); + tableReference.set(HiveDataModelGenerator.CREATE_TIME, createDate); + } catch(NumberFormatException ne) { + LOG.error("Error while updating createTime for the table {} ", hiveTable.getCompleteName(), ne); + } + } + + Date lastAccessTime = createDate; + if ( hiveTable.getLastAccessTime() > 0) { + lastAccessTime = new Date(hiveTable.getLastAccessTime() * MILLIS_CONVERT_FACTOR); + } + tableReference.set(HiveDataModelGenerator.LAST_ACCESS_TIME, lastAccessTime); tableReference.set("retention", hiveTable.getRetention()); tableReference.set(HiveDataModelGenerator.COMMENT, hiveTable.getParameters().get(HiveDataModelGenerator.COMMENT)); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/922a83c9/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java index 4ee15d6..749294f 100755 --- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java +++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java @@ -53,6 +53,7 @@ import org.slf4j.LoggerFactory; import java.net.MalformedURLException; import java.util.ArrayList; +import java.util.Date; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; @@ -619,12 +620,12 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext { } processReferenceable.set("name", queryStr); processReferenceable.set("operationType", hiveEvent.getOperation().getOperationName()); - processReferenceable.set("startTime", hiveEvent.getQueryStartTime()); + processReferenceable.set("startTime", new Date(hiveEvent.getQueryStartTime())); processReferenceable.set("userName", hiveEvent.getUser()); processReferenceable.set("queryText", queryStr); processReferenceable.set("queryId", hiveEvent.getQueryId()); processReferenceable.set("queryPlan", hiveEvent.getJsonPlan()); - processReferenceable.set("endTime", System.currentTimeMillis()); + processReferenceable.set("endTime", new Date(System.currentTimeMillis())); //TODO set queryGraph return processReferenceable; } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/922a83c9/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java index ebeabb6..f099e39 100755 --- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java +++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java @@ -78,6 +78,9 @@ public class HiveDataModelGenerator { public static final String STORAGE_DESC_OUTPUT_FMT = "outputFormat"; public static final String OWNER = "owner"; + public static final String CREATE_TIME = "createTime"; + public static final String LAST_ACCESS_TIME = "lastAccessTime"; + public HiveDataModelGenerator() { classTypeDefinitions = new HashMap<>(); enumTypeDefinitionMap = new HashMap<>(); @@ -233,9 +236,9 @@ public class HiveDataModelGenerator { null), new AttributeDefinition(DB, HiveDataTypes.HIVE_DB.getName(), Multiplicity.REQUIRED, false, null), new AttributeDefinition(OWNER, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null), - new AttributeDefinition("createTime", DataTypes.LONG_TYPE.getName(), Multiplicity.OPTIONAL, false, + new AttributeDefinition(CREATE_TIME, DataTypes.DATE_TYPE.getName(), Multiplicity.OPTIONAL, false, null), - new AttributeDefinition("lastAccessTime", DataTypes.LONG_TYPE.getName(), Multiplicity.OPTIONAL, false, + new AttributeDefinition(LAST_ACCESS_TIME, DataTypes.DATE_TYPE.getName(), Multiplicity.OPTIONAL, false, null), new AttributeDefinition(COMMENT, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null), new AttributeDefinition("retention", DataTypes.INT_TYPE.getName(), Multiplicity.OPTIONAL, false, null), @@ -263,8 +266,8 @@ public class HiveDataModelGenerator { private void createProcessClass() throws AtlasException { AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{ - new AttributeDefinition("startTime", DataTypes.LONG_TYPE.getName(), Multiplicity.REQUIRED, false, null), - new AttributeDefinition("endTime", DataTypes.LONG_TYPE.getName(), Multiplicity.REQUIRED, false, null), + new AttributeDefinition("startTime", DataTypes.DATE_TYPE.getName(), Multiplicity.REQUIRED, false, null), + new AttributeDefinition("endTime", DataTypes.DATE_TYPE.getName(), Multiplicity.REQUIRED, false, null), new AttributeDefinition("userName", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null), new AttributeDefinition("operationType", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/922a83c9/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java index 5fda0d3..4c7ac70 100755 --- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java +++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java @@ -31,6 +31,7 @@ import org.apache.atlas.hive.model.HiveDataTypes; import org.apache.atlas.typesystem.Referenceable; import org.apache.atlas.typesystem.Struct; import org.apache.atlas.typesystem.persistence.Id; +import org.apache.atlas.typesystem.types.TypeSystem; import org.apache.atlas.utils.ParamChecker; import org.apache.commons.configuration.Configuration; import org.apache.commons.lang.RandomStringUtils; @@ -38,8 +39,10 @@ import org.apache.commons.lang.StringUtils; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.TableType; +import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.ql.Driver; import org.apache.hadoop.hive.ql.hooks.Entity; +import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; import org.apache.hadoop.hive.ql.session.SessionState; import org.codehaus.jettison.json.JSONObject; @@ -49,6 +52,8 @@ import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; import java.io.File; +import java.text.ParseException; +import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -66,6 +71,7 @@ public class HiveHookIT { public static final String DEFAULT_DB = "default"; private Driver driver; private AtlasClient atlasClient; + private HiveMetaStoreBridge hiveMetaStoreBridge; private SessionState ss; private static final String INPUTS = AtlasClient.PROCESS_ATTRIBUTE_INPUTS; @@ -87,7 +93,7 @@ public class HiveHookIT { Configuration configuration = ApplicationProperties.get(); atlasClient = new AtlasClient(configuration.getString(HiveMetaStoreBridge.ATLAS_ENDPOINT, DGI_URL)); - HiveMetaStoreBridge hiveMetaStoreBridge = new HiveMetaStoreBridge(conf, atlasClient); + hiveMetaStoreBridge = new HiveMetaStoreBridge(conf, atlasClient); hiveMetaStoreBridge.registerHiveDataModel(); } @@ -189,6 +195,12 @@ public class HiveHookIT { Assert.assertEquals(tableRef.get(HiveDataModelGenerator.NAME), entityName); Assert.assertEquals(tableRef.get(HiveDataModelGenerator.NAME), "default." + tableName.toLowerCase() + "@" + CLUSTER_NAME); + Table t = hiveMetaStoreBridge.hiveClient.getTable(DEFAULT_DB, tableName); + long createTime = Long.parseLong(t.getMetadata().getProperty(hive_metastoreConstants.DDL_TIME)) * HiveMetaStoreBridge.MILLIS_CONVERT_FACTOR; + + verifyTimestamps(tableRef, HiveDataModelGenerator.CREATE_TIME, createTime); + verifyTimestamps(tableRef, HiveDataModelGenerator.LAST_ACCESS_TIME, createTime); + final Referenceable sdRef = (Referenceable) tableRef.get("sd"); Assert.assertEquals(sdRef.get(HiveDataModelGenerator.STORAGE_IS_STORED_AS_SUB_DIRS), false); @@ -196,6 +208,21 @@ public class HiveHookIT { assertDatabaseIsRegistered(DEFAULT_DB); } + private void verifyTimestamps(Referenceable ref, String property, long expectedTime) throws ParseException { + //Verify timestamps. + String createTimeStr = (String) ref.get(property); + Date createDate = TypeSystem.getInstance().getDateFormat().parse(createTimeStr); + Assert.assertNotNull(createTimeStr); + + if (expectedTime > 0) { + Assert.assertEquals(expectedTime, createDate.getTime()); + } + } + + private void verifyTimestamps(Referenceable ref, String property) throws ParseException { + verifyTimestamps(ref, property, 0); + } + @Test public void testCreateExternalTable() throws Exception { String tableName = tableName(); @@ -208,6 +235,10 @@ public class HiveHookIT { String tableId = assertTableIsRegistered(dbName, tableName); Referenceable processReference = validateProcess(query, 1, 1); + + verifyTimestamps(processReference, "startTime"); + verifyTimestamps(processReference, "endTime"); + validateHDFSPaths(processReference, pFile, INPUTS); validateOutputTables(processReference, tableId); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/922a83c9/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 61c6e8b..326e685 100644 --- a/release-log.txt +++ b/release-log.txt @@ -3,13 +3,14 @@ Apache Atlas Release Notes --trunk - unreleased INCOMPATIBLE CHANGES: +ATLAS-532 Change Data types of all timestamps in Hive model(currently long)(sumasai via yhemanth) ATLAS-622 Introduce soft delete (shwethags) ATLAS-494 UI Authentication (nixonrodrigues via shwethags) ATLAS-621 Introduce entity state in Id object (shwethags) ATLAS-474 Server does not start if the type is updated with same super type class information (dkantor via shwethags) ATLAS-479 Add description for different types during create time (guptaneeru via shwethags) -ATLAS-521 Support Alter Table column commands (suma.shivaprasad via shwethags) ATLAS-500 UI: Search Default (sanjayp via shwethags) +ATLAS-525 Drop support for partitions, select query lineage, roles, principals, resource, hive_type...(sumasai via shwethags) ATLAS-483 Remove client.properties (tbeerbower via shwethags) ATLAS-349 SSL - Atlas SSL connection has weak/unsafe Ciphers suites (ndjouhr via shwethags) ATLAS-409 Atlas will not import avro tables with schema read from a file ([email protected] via shwethags) @@ -32,7 +33,6 @@ ATLAS-571 Modify Atlas client for necessary changes in context of HA (yhemanth v ATLAS-620 Disable hbase based entity audit (shwethags) ATLAS-618 Fix assembly for hdfs-module (sumasai via yhemanth) ATLAS-573 Inherited attributes disappear from entities after server restart (dkantor via sumasai) -ATLAS-525 Drop support for partitions, select query lineage, roles, principals, resource, hive_type...(sumasai via shwethags) ATLAS-599 HDFS Path Model (sumasai via yhemanth) ATLAS-553 Entity mutation - Fix issue with reordering of elements in array<class> with composite references (sumasai via shwethags) ATLAS-513 Admin support for HA (yhemanth via sumasai) @@ -46,6 +46,7 @@ ATLAS-539 Store for entity audit events (shwethags) ATLAS-523 Support alter view (sumasai via shwethags) ATLAS-555 Tag creation from UI fails due to missing description attribute (guptaneeru via shwethags) ATLAS-522 Support Alter table commands (sumasai via shwethags) +ATLAS-521 Support Alter Table column commands (sumasai via shwethags) ATLAS-512 Decouple currently integrating components from availability of Atlas service for raising metadata events ( yhemanth via sumasai) ATLAS-537 Falcon hook failing when tried to submit a process which creates a hive table ( shwethags via sumasai) ATLAS-476 Update type attribute with Reserved characters updated the original type as unknown (yhemanth via shwethags)
