-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/73169/
-----------------------------------------------------------
(Updated Feb. 3, 2021, 6:04 p.m.)
Review request for atlas, Ashutosh Mestry, Deep Singh, Madhan Neethiraj,
Radhika Kundam, and Sarath Subramanian.
Bugs: ATLAS-4123
https://issues.apache.org/jira/browse/ATLAS-4123
Repository: atlas
Description
-------
ATLAS-4123: Lineage missing in case of CREATE MATERIALIZED VIEW query at Hive
Diffs
-----
addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
e48967d63
Diff: https://reviews.apache.org/r/73169/diff/1/
Testing (updated)
-------
Atlas hook topic message from the hive hook before fix:
=======================================================
{
"version": {
"version": "1.0.0",
"versionParts": [
1
]
},
"msgCompressionKind": "NONE",
"msgSplitIdx": 1,
"msgSplitCount": 1,
"msgSourceIP": "172.27.10.2",
"msgCreatedBy": "hive",
"msgCreationTime": 1612311707304,
"message": {
"type": "ENTITY_CREATE_V2",
"user": "hive",
"entities": {
"referredEntities": {
"-73285185733726514": {
"typeName": "hive_storagedesc",
"attributes": {
"qualifiedName": "default.test_mv_ctas_2@cm_storage",
"storedAsSubDirectories": false,
"location":
"hdfs://secure-2.secure.root.hwx.site:8020/warehouse/tablespace/managed/hive/test_mv_ctas_2",
"compressed": false,
"inputFormat":
"org.apache.hadoop.hive.ql.io.orc.OrcInputFormat",
"parameters": {},
"outputFormat":
"org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat",
"serdeInfo": {
"typeName": "hive_serde",
"attributes": {
"serializationLib":
"org.apache.hadoop.hive.ql.io.orc.OrcSerde",
"name": null,
"parameters": {}
}
},
"numBuckets": -1
},
"guid": "-73285185733726514",
"isIncomplete": false,
"provenanceType": 0,
"version": 0,
"relationshipAttributes": {
"table": {
"guid": "-73285185733726513",
"typeName": "hive_table",
"uniqueAttributes": {
"qualifiedName": "default.test_mv_ctas_2@cm"
},
"relationshipType": "hive_table_storagedesc"
}
},
"proxy": false
},
"-73285185733726515": {
"typeName": "hive_column",
"attributes": {
"owner": "admin",
"qualifiedName": "default.test_mv_ctas_2.id@cm",
"name": "id",
"comment": null,
"position": 0,
"type": "int"
},
"guid": "-73285185733726515",
"isIncomplete": false,
"provenanceType": 0,
"version": 0,
"relationshipAttributes": {
"table": {
"guid": "-73285185733726513",
"typeName": "hive_table",
"uniqueAttributes": {
"qualifiedName": "default.test_mv_ctas_2@cm"
},
"relationshipType": "hive_table_columns"
}
},
"proxy": false
}
},
"entities": [
{
"typeName": "hive_table",
"attributes": {
"owner": "admin",
"viewExpandedText": "select `test_1`.`id` from
`default`.`test_1`",
"tableType": "MATERIALIZED_VIEW",
"temporary": false,
"lastAccessTime": 1612311707000,
"createTime": 1612311707000,
"qualifiedName": "default.test_mv_ctas_2@cm",
"name": "test_mv_ctas_2",
"comment": null,
"parameters": {
"transient_lastDdlTime": "1612311707",
"bucketing_version": "2",
"transactional_properties": "default",
"transactional": "true"
},
"retention": 0,
"viewOriginalText": "select * from test_1"
},
"guid": "-73285185733726513",
"isIncomplete": false,
"provenanceType": 0,
"version": 0,
"relationshipAttributes": {
"sd": {
"guid": "-73285185733726514",
"typeName": "hive_storagedesc",
"uniqueAttributes": {
"qualifiedName":
"default.test_mv_ctas_2@cm_storage"
},
"relationshipType": "hive_table_storagedesc"
},
"columns": [
{
"guid": "-73285185733726515",
"typeName": "hive_column",
"uniqueAttributes": {
"qualifiedName":
"default.test_mv_ctas_2.id@cm"
},
"relationshipType": "hive_table_columns"
}
],
"partitionKeys": [],
"db": {
"typeName": "hive_db",
"uniqueAttributes": {
"qualifiedName": "default@cm"
},
"relationshipType": "hive_table_db"
}
},
"proxy": false
}
]
}
}
}
Atlas hook topic message from the hive hook after fix:
=======================================================
{
"version": {
"version": "1.0.0",
"versionParts": [
1
]
},
"msgCompressionKind": "NONE",
"msgSplitIdx": 1,
"msgSplitCount": 1,
"msgSourceIP": "172.27.10.2",
"msgCreatedBy": "hive",
"msgCreationTime": 1612317050425,
"message": {
"type": "ENTITY_CREATE_V2",
"user": "hive",
"entities": {
"referredEntities": {
"-73293806469890554": {
"typeName": "hive_column",
"attributes": {
"owner": "admin",
"qualifiedName": "default.test_mv_ctas_1.id@cm",
"name": "id",
"comment": null,
"position": 0,
"type": "int"
},
"guid": "-73293806469890554",
"isIncomplete": false,
"provenanceType": 0,
"version": 0,
"relationshipAttributes": {
"table": {
"guid": "-73293806469890552",
"typeName": "hive_table",
"uniqueAttributes": {
"qualifiedName": "default.test_mv_ctas_1@cm"
},
"relationshipType": "hive_table_columns"
}
},
"proxy": false
},
"-73293806469890553": {
"typeName": "hive_storagedesc",
"attributes": {
"qualifiedName": "default.test_mv_ctas_1@cm_storage",
"storedAsSubDirectories": false,
"location":
"hdfs://secure-2.secure.root.hwx.site:8020/warehouse/tablespace/managed/hive/test_mv_ctas_1",
"compressed": false,
"inputFormat":
"org.apache.hadoop.hive.ql.io.orc.OrcInputFormat",
"parameters": {},
"outputFormat":
"org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat",
"serdeInfo": {
"typeName": "hive_serde",
"attributes": {
"serializationLib":
"org.apache.hadoop.hive.ql.io.orc.OrcSerde",
"name": null,
"parameters": {}
}
},
"numBuckets": -1
},
"guid": "-73293806469890553",
"isIncomplete": false,
"provenanceType": 0,
"version": 0,
"relationshipAttributes": {
"table": {
"guid": "-73293806469890552",
"typeName": "hive_table",
"uniqueAttributes": {
"qualifiedName": "default.test_mv_ctas_1@cm"
},
"relationshipType": "hive_table_storagedesc"
}
},
"proxy": false
},
"-73293806469890551": {
"typeName": "hive_db",
"attributes": {
"owner": "public",
"ownerType": "ROLE",
"managedLocation": null,
"qualifiedName": "default@cm",
"clusterName": "cm",
"name": "default",
"description": "Default Hive database",
"location":
"hdfs://secure-2.secure.root.hwx.site:8020/warehouse/tablespace/external/hive",
"parameters": {}
},
"guid": "-73293806469890551",
"isIncomplete": false,
"provenanceType": 0,
"version": 0,
"proxy": false
}
},
"entities": [
{
"typeName": "hive_table",
"attributes": {
"owner": "admin",
"viewExpandedText": "select `test_1`.`id` from
`default`.`test_1`",
"tableType": "MATERIALIZED_VIEW",
"temporary": false,
"lastAccessTime": 1612317049000,
"createTime": 1612317049000,
"qualifiedName": "default.test_mv_ctas_1@cm",
"name": "test_mv_ctas_1",
"comment": null,
"parameters": {
"transient_lastDdlTime": "1612317049",
"bucketing_version": "2",
"transactional_properties": "default",
"transactional": "true"
},
"retention": 0,
"viewOriginalText": "select * from test_1"
},
"guid": "-73293806469890552",
"isIncomplete": false,
"provenanceType": 0,
"version": 0,
"relationshipAttributes": {
"sd": {
"guid": "-73293806469890553",
"typeName": "hive_storagedesc",
"uniqueAttributes": {
"qualifiedName":
"default.test_mv_ctas_1@cm_storage"
},
"relationshipType": "hive_table_storagedesc"
},
"columns": [
{
"guid": "-73293806469890554",
"typeName": "hive_column",
"uniqueAttributes": {
"qualifiedName":
"default.test_mv_ctas_1.id@cm"
},
"relationshipType": "hive_table_columns"
}
],
"partitionKeys": [],
"db": {
"guid": "-73293806469890551",
"typeName": "hive_db",
"uniqueAttributes": {
"qualifiedName": "default@cm"
},
"relationshipType": "hive_table_db"
}
},
"proxy": false
}
]
}
}
}
{
"version": {
"version": "1.0.0",
"versionParts": [
1
]
},
"msgCompressionKind": "NONE",
"msgSplitIdx": 1,
"msgSplitCount": 1,
"msgSourceIP": "172.27.10.2",
"msgCreatedBy": "hive",
"msgCreationTime": 1612317051395,
"message": {
"type": "ENTITY_CREATE_V2",
"user": "admin",
"entities": {
"referredEntities": {
"-73293807317085531": {
"typeName": "hive_column",
"attributes": {
"owner": "admin",
"qualifiedName": "default.test_mv_ctas_1.id@cm",
"name": "id",
"comment": null,
"position": 0,
"type": "int"
},
"guid": "-73293807317085531",
"isIncomplete": false,
"provenanceType": 0,
"version": 0,
"relationshipAttributes": {
"table": {
"guid": "-73293807317085529",
"typeName": "hive_table",
"uniqueAttributes": {
"qualifiedName": "default.test_mv_ctas_1@cm"
},
"relationshipType": "hive_table_columns"
}
},
"proxy": false
},
"-73293807317085530": {
"typeName": "hive_storagedesc",
"attributes": {
"qualifiedName": "default.test_mv_ctas_1@cm_storage",
"storedAsSubDirectories": false,
"location":
"hdfs://secure-2.secure.root.hwx.site:8020/warehouse/tablespace/managed/hive/test_mv_ctas_1",
"compressed": false,
"inputFormat":
"org.apache.hadoop.hive.ql.io.orc.OrcInputFormat",
"parameters": {},
"outputFormat":
"org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat",
"serdeInfo": {
"typeName": "hive_serde",
"attributes": {
"serializationLib":
"org.apache.hadoop.hive.ql.io.orc.OrcSerde",
"name": null,
"parameters": {}
}
},
"numBuckets": -1
},
"guid": "-73293807317085530",
"isIncomplete": false,
"provenanceType": 0,
"version": 0,
"relationshipAttributes": {
"table": {
"guid": "-73293807317085529",
"typeName": "hive_table",
"uniqueAttributes": {
"qualifiedName": "default.test_mv_ctas_1@cm"
},
"relationshipType": "hive_table_storagedesc"
}
},
"proxy": false
},
"-73293807317085529": {
"typeName": "hive_table",
"attributes": {
"owner": "admin",
"viewExpandedText": "select `test_1`.`id` from
`default`.`test_1`",
"tableType": "MATERIALIZED_VIEW",
"temporary": false,
"lastAccessTime": 1612317049000,
"createTime": 1612317049000,
"qualifiedName": "default.test_mv_ctas_1@cm",
"name": "test_mv_ctas_1",
"comment": null,
"parameters": {
"totalSize": "0",
"rawDataSize": "0",
"numRows": "0",
"transactional_properties": "default",
"COLUMN_STATS_ACCURATE":
"{\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"id\":\"true\"}}",
"numFiles": "0",
"transient_lastDdlTime": "1612317050",
"bucketing_version": "2",
"numFilesErasureCoded": "0",
"transactional": "true"
},
"retention": 0,
"viewOriginalText": "select * from test_1"
},
"guid": "-73293807317085529",
"isIncomplete": false,
"provenanceType": 0,
"version": 0,
"relationshipAttributes": {
"sd": {
"guid": "-73293807317085530",
"typeName": "hive_storagedesc",
"uniqueAttributes": {
"qualifiedName":
"default.test_mv_ctas_1@cm_storage"
},
"relationshipType": "hive_table_storagedesc"
},
"columns": [
{
"guid": "-73293807317085531",
"typeName": "hive_column",
"uniqueAttributes": {
"qualifiedName":
"default.test_mv_ctas_1.id@cm"
},
"relationshipType": "hive_table_columns"
}
],
"partitionKeys": [],
"db": {
"guid": "-73293807317085525",
"typeName": "hive_db",
"uniqueAttributes": {
"qualifiedName": "default@cm"
},
"relationshipType": "hive_table_db"
}
},
"proxy": false
},
"-73293807317085528": {
"typeName": "hive_column",
"attributes": {
"owner": "admin",
"qualifiedName": "default.test_1.id@cm",
"name": "id",
"comment": null,
"position": 0,
"type": "int"
},
"guid": "-73293807317085528",
"isIncomplete": false,
"provenanceType": 0,
"version": 0,
"relationshipAttributes": {
"table": {
"guid": "-73293807317085526",
"typeName": "hive_table",
"uniqueAttributes": {
"qualifiedName": "default.test_1@cm"
},
"relationshipType": "hive_table_columns"
}
},
"proxy": false
},
"-73293807317085527": {
"typeName": "hive_storagedesc",
"attributes": {
"qualifiedName": "default.test_1@cm_storage",
"storedAsSubDirectories": false,
"location":
"hdfs://secure-2.secure.root.hwx.site:8020/warehouse/tablespace/managed/hive/test_1",
"compressed": false,
"inputFormat":
"org.apache.hadoop.hive.ql.io.orc.OrcInputFormat",
"parameters": {},
"outputFormat":
"org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat",
"serdeInfo": {
"typeName": "hive_serde",
"attributes": {
"serializationLib":
"org.apache.hadoop.hive.ql.io.orc.OrcSerde",
"name": null,
"parameters": {}
}
},
"numBuckets": -1
},
"guid": "-73293807317085527",
"isIncomplete": false,
"provenanceType": 0,
"version": 0,
"relationshipAttributes": {
"table": {
"guid": "-73293807317085526",
"typeName": "hive_table",
"uniqueAttributes": {
"qualifiedName": "default.test_1@cm"
},
"relationshipType": "hive_table_storagedesc"
}
},
"proxy": false
},
"-73293807317085526": {
"typeName": "hive_table",
"attributes": {
"owner": "admin",
"tableType": "MANAGED_TABLE",
"temporary": false,
"lastAccessTime": 1612314520000,
"createTime": 1612314520000,
"qualifiedName": "default.test_1@cm",
"name": "test_1",
"comment": null,
"parameters": {
"totalSize": "0",
"rawDataSize": "0",
"numRows": "0",
"transactional_properties": "default",
"COLUMN_STATS_ACCURATE":
"{\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"id\":\"true\"}}",
"numFiles": "0",
"transient_lastDdlTime": "1612314520",
"bucketing_version": "2",
"numFilesErasureCoded": "0",
"transactional": "true"
},
"retention": 0
},
"guid": "-73293807317085526",
"isIncomplete": false,
"provenanceType": 0,
"version": 0,
"relationshipAttributes": {
"sd": {
"guid": "-73293807317085527",
"typeName": "hive_storagedesc",
"uniqueAttributes": {
"qualifiedName": "default.test_1@cm_storage"
},
"relationshipType": "hive_table_storagedesc"
},
"columns": [
{
"guid": "-73293807317085528",
"typeName": "hive_column",
"uniqueAttributes": {
"qualifiedName": "default.test_1.id@cm"
},
"relationshipType": "hive_table_columns"
}
],
"partitionKeys": [],
"db": {
"guid": "-73293807317085525",
"typeName": "hive_db",
"uniqueAttributes": {
"qualifiedName": "default@cm"
},
"relationshipType": "hive_table_db"
}
},
"proxy": false
},
"-73293807317085525": {
"typeName": "hive_db",
"attributes": {
"owner": "public",
"ownerType": "ROLE",
"managedLocation":
"hdfs://secure-2.secure.root.hwx.site:8020/warehouse/tablespace/managed/hive",
"qualifiedName": "default@cm",
"clusterName": "cm",
"name": "default",
"description": "Default Hive database",
"location":
"hdfs://secure-2.secure.root.hwx.site:8020/warehouse/tablespace/external/hive",
"parameters": {}
},
"guid": "-73293807317085525",
"isIncomplete": false,
"provenanceType": 0,
"version": 0,
"proxy": false
}
},
"entities": [
{
"typeName": "hive_process",
"attributes": {
"recentQueries": [
"create materialized view test_mv_ctas_1 as select
* from test_1"
],
"qualifiedName":
"CREATE_MATERIALIZED_VIEW:default.test_1@cm:1612314520000->:default@cm:default.test_mv_ctas_1@cm:1612317049000",
"clusterName": "cm",
"name":
"CREATE_MATERIALIZED_VIEW:default.test_1@cm:1612314520000->:default@cm:default.test_mv_ctas_1@cm:1612317049000",
"queryText": "",
"operationType": "CREATE_MATERIALIZED_VIEW",
"startTime": 1612317051380,
"queryPlan": "Not Supported",
"endTime": 1612317051380,
"userName": "",
"queryId": ""
},
"guid": "-73293807317085532",
"isIncomplete": false,
"provenanceType": 0,
"version": 0,
"relationshipAttributes": {
"outputs": [
{
"guid": "-73293807317085529",
"typeName": "hive_table",
"uniqueAttributes": {
"qualifiedName": "default.test_mv_ctas_1@cm"
},
"relationshipType": "process_dataset_outputs"
}
],
"inputs": [
{
"guid": "-73293807317085526",
"typeName": "hive_table",
"uniqueAttributes": {
"qualifiedName": "default.test_1@cm"
},
"relationshipType": "dataset_process_inputs"
}
]
},
"proxy": false
},
{
"typeName": "hive_process_execution",
"attributes": {
"hostName": "secure-2.secure.root.hwx.site",
"qualifiedName":
"CREATE_MATERIALIZED_VIEW:default.test_1@cm:1612314520000->:default@cm:default.test_mv_ctas_1@cm:1612317049000:1612317032474:1612317051380",
"name":
"CREATE_MATERIALIZED_VIEW:default.test_1@cm:1612314520000->:default@cm:default.test_mv_ctas_1@cm:1612317049000:1612317032474:1612317051380",
"queryText": "create materialized view test_mv_ctas_1
as select * from test_1",
"startTime": 1612317032474,
"queryPlan": "Not Supported",
"endTime": 1612317051380,
"userName": "admin",
"queryId":
"hive_20210203015032_7bef0f08-bddf-4992-b761-97aa261be104"
},
"guid": "-73293807317085533",
"isIncomplete": false,
"provenanceType": 0,
"version": 0,
"relationshipAttributes": {
"process": {
"guid": "-73293807317085532",
"typeName": "hive_process",
"relationshipType":
"hive_process_process_executions"
}
},
"proxy": false
}
]
}
}
}
Thanks,
Sidharth Mishra
