[ https://issues.apache.org/jira/browse/ATLAS-4635?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Dharshana M Krishnamoorthy updated ATLAS-4635: ---------------------------------------------- Description: Source information is missing for impala messages {code:java} {"source":{"version":"2.1.0.7.1.8.0-723"},"version":{"version":"1.0.0","versionParts":[1]},"msgCompressionKind":"NONE","msgSplitIdx":1,"msgSplitCount":1,"msgSourceIP":"...","msgCreatedBy":"...","msgCreationTime":1657759634684,"spooled":false,"message":...} {code} This is a message from ATLAS_SPARK_HOOK, the source infor of the message is missing here Expected: "source":\{"source": "spark", "version":"2.1.0.7.1.8.0-723"}, but source is missing It is available for other services: HMS: {code:java} {"source":{"version":"2.1.0.7.1.8.0-723","source":"hive_metastore"},"version":{"version":"1.0.0","versionParts":[1]},"msgCompressionKind":"NONE","msgSplitIdx":1,"msgSplitCount":1,"msgSourceIP":"...","msgCreatedBy":"...","msgCreationTime":1657700529795,"spooled":false,"message":{ {code} {color:#009100}...{color} {code:java} }{code} HBase: {code:java} {"source":{"version":"2.1.0.7.1.8.0-723","source":"hbase"},"version":{"version":"1.0.0","versionParts":[1]},"msgCompressionKind":"NONE","msgSplitIdx":1,"msgSplitCount":1,"msgSourceIP":"...","msgCreatedBy":"...","msgCreationTime":1657700897665,"spooled":false,"message":{ {code} {color:#009100}...{color} {code:java} }{code} Impala: {code:java} {"source":{"version":"2.1.0.7.1.8.0-723","source":"impala"},"version":{"version":"1.0.0","versionParts":[1]},"msgCompressionKind":"NONE","msgSplitIdx":1,"msgSplitCount":1,"msgSourceIP":"...","msgCreatedBy":"...","msgCreationTime":1657695780022,"spooled":false,"message":{"...}}{code} h4. was: Source information is missing for impala messages {code:java} {"source":{"version":"2.1.0.7.1.8.0-723"},"version":{"version":"1.0.0","versionParts":[1]},"msgCompressionKind":"NONE","msgSplitIdx":1,"msgSplitCount":1,"msgSourceIP":"172.27.134.1","msgCreatedBy":"hrt_qa","msgCreationTime":1657759634684,"spooled":false,"message":{"type":"ENTITY_CREATE_V2","user":"hrt...@root.hwx.site","entities":{"entities":[{"typeName":"spark_column_lineage","attributes":{"outputs":[{"typeName":"hive_column","uniqueAttributes":{"qualifiedName":"default.tab_1_2235274282_ctas.col1@cm"}}],"qualifiedName":"default.tab_1_2235274282_ctas@cm:1657759634668:col1","inputs":[{"typeName":"hive_column","uniqueAttributes":{"qualifiedName":"default.tab_1_2235274282.col1@cm"}}],"name":"default.tab_1_2235274282_ctas@cm:1657759634668:col1"},"guid":"-215346008271268","isIncomplete":false,"provenanceType":0,"version":0,"relationshipAttributes":{"process":{"typeName":"spark_process","uniqueAttributes":{"qualifiedName":"application_1657738076117_0064-execution-12"}}},"proxy":false}]}}} {code} This is a message from ATLAS_SPARK_HOOK, the source infor of the message is missing here Expected: "source":\{"source": "spark", "version":"2.1.0.7.1.8.0-723"}, but source is missing It is available for other services: HMS: {code:java} {"source":{"version":"2.1.0.7.1.8.0-723","source":"hive_metastore"},"version":{"version":"1.0.0","versionParts":[1]},"msgCompressionKind":"NONE","msgSplitIdx":1,"msgSplitCount":1,"msgSourceIP":"172.27.10.134","msgCreatedBy":"hive","msgCreationTime":1657700529795,"spooled":false,"message":{"type":"ENTITY_DELETE_V2","user":"hue","entities":[{"typeName":"hive_db","uniqueAttributes":{"qualifiedName":"cloudera_manager_metastore_canary_test_catalog_hive_hivemetastore_1:default@cm"}}]}}{code} HBase: {code:java} {"source":{"version":"2.1.0.7.1.8.0-723","source":"hbase"},"version":{"version":"1.0.0","versionParts":[1]},"msgCompressionKind":"NONE","msgSplitIdx":1,"msgSplitCount":1,"msgSourceIP":"172.27.72.195","msgCreatedBy":"hbase","msgCreationTime":1657700897665,"spooled":false,"message":{"type":"ENTITY_CREATE_V2","user":"hrt_qa","entities":{"entities":[{"typeName":"hbase_namespace","attributes":{"owner":"hrt_qa","modifiedTime":1657700897665,"createTime":1657700897665,"qualifiedName":"ns_zsptr@cm","clusterName":"cm","name":"ns_zsptr","description":"ns_zsptr","parameters":null},"guid":"-3685335942958094","isIncomplete":false,"provenanceType":0,"version":0,"proxy":false}]}}}{code} Impala: {code:java} {"source":{"version":"2.1.0.7.1.8.0-723","source":"impala"},"version":{"version":"1.0.0","versionParts":[1]},"msgCompressionKind":"NONE","msgSplitIdx":1,"msgSplitCount":1,"msgSourceIP":"172.27.187.129","msgCreatedBy":"impala","msgCreationTime":1657695780022,"spooled":false,"message":{"type":"ENTITY_CREATE_V2","user":"hrt...@qe-infra-ad.cloudera.com","entities":{"referredEntities":{},"entities":[{"typeName":"impala_process","attributes":{"outputs":[{"typeName":"hive_table","uniqueAttributes":{"qualifiedName":"db_mzuh.tbl_mzuh_2@cm"}}],"recentQueries":["insert into db_mzuh.tbl_mzuh_2 select * from (select tbl_mzuh_0_col_0 from db_mzuh.tbl_mzuh_0) as a join (select tbl_mzuh_1_col_1 from db_mzuh.tbl_mzuh_1) as b"],"qualifiedName":"QUERY:db_mzuh.tbl_mzuh_0@cm:1657695719000:db_mzuh.tbl_mzuh_1@cm:1657695725000->:INSERT:db_mzuh.tbl_mzuh_2@cm:1657695732000","inputs":[{"typeName":"hive_table","uniqueAttributes":{"qualifiedName":"db_mzuh.tbl_mzuh_0@cm"}},{"typeName":"hive_table","uniqueAttributes":{"qualifiedName":"db_mzuh.tbl_mzuh_1@cm"}}],"name":"insert into db_mzuh.tbl_mzuh_2 select * from (select tbl_mzuh_0_col_0 from db_mzuh.tbl_mzuh_0) as a join (select tbl_mzuh_1_col_1 from db_mzuh.tbl_mzuh_1) as b","queryText":"","operationType":"QUERY","startTime":"","queryPlan":"Not Supported","endTime":"","userName":"","queryId":""},"guid":"-38791214447462036","isIncomplete":false,"provenanceType":0,"version":0,"proxy":false},{"typeName":"impala_process_execution","attributes":{"hostName":"quasar-ytcooi-5.quasar-ytcooi.root.hwx.site","qualifiedName":"QUERY:db_mzuh.tbl_mzuh_0@cm:1657695719000:db_mzuh.tbl_mzuh_1@cm:1657695725000->:INSERT:db_mzuh.tbl_mzuh_2@cm:1657695732000:1657695772000:1657695780000","name":"insert into db_mzuh.tbl_mzuh_2 select * from (select tbl_mzuh_0_col_0 from db_mzuh.tbl_mzuh_0) as a join (select tbl_mzuh_1_col_1 from db_mzuh.tbl_mzuh_1) as b:1657695772000","queryText":"insert into db_mzuh.tbl_mzuh_2 select * from (select tbl_mzuh_0_col_0 from db_mzuh.tbl_mzuh_0) as a join (select tbl_mzuh_1_col_1 from db_mzuh.tbl_mzuh_1) as b","startTime":1657695772000,"queryPlan":"Not Supported","endTime":1657695780000,"userName":"hrt...@qe-infra-ad.cloudera.com","queryId":"d24c529cfdd4b6bc:e9a60a0d00000000"},"guid":"-38791214447462037","isIncomplete":false,"provenanceType":0,"version":0,"relationshipAttributes":{"process":{"guid":"-38791214447462036","typeName":"impala_process"}},"proxy":false},{"typeName":"impala_column_lineage","attributes":{"outputs":[{"typeName":"hive_column","uniqueAttributes":{"qualifiedName":"db_mzuh.tbl_mzuh_2.tbl_mzuh_2_col_0@cm"}}],"dependencyType":"PROJECTION","qualifiedName":"QUERY:db_mzuh.tbl_mzuh_0@cm:1657695719000:db_mzuh.tbl_mzuh_1@cm:1657695725000->:INSERT:db_mzuh.tbl_mzuh_2@cm:1657695732000:tbl_mzuh_2_col_0","inputs":[{"typeName":"hive_column","uniqueAttributes":{"qualifiedName":"db_mzuh.tbl_mzuh_0.tbl_mzuh_0_col_0@cm"}}],"query":{"guid":"-38791214447462036","typeName":"impala_process","uniqueAttributes":{"qualifiedName":"QUERY:db_mzuh.tbl_mzuh_0@cm:1657695719000:db_mzuh.tbl_mzuh_1@cm:1657695725000->:INSERT:db_mzuh.tbl_mzuh_2@cm:1657695732000"}},"name":"QUERY:db_mzuh.tbl_mzuh_0@cm:1657695719000:db_mzuh.tbl_mzuh_1@cm:1657695725000->:INSERT:db_mzuh.tbl_mzuh_2@cm:1657695732000:tbl_mzuh_2_col_0"},"guid":"-38791214447462038","isIncomplete":false,"provenanceType":0,"version":0,"proxy":false},{"typeName":"impala_column_lineage","attributes":{"outputs":[{"typeName":"hive_column","uniqueAttributes":{"qualifiedName":"db_mzuh.tbl_mzuh_2.tbl_mzuh_2_col_1@cm"}}],"dependencyType":"PROJECTION","qualifiedName":"QUERY:db_mzuh.tbl_mzuh_0@cm:1657695719000:db_mzuh.tbl_mzuh_1@cm:1657695725000->:INSERT:db_mzuh.tbl_mzuh_2@cm:1657695732000:tbl_mzuh_2_col_1","inputs":[{"typeName":"hive_column","uniqueAttributes":{"qualifiedName":"db_mzuh.tbl_mzuh_1.tbl_mzuh_1_col_1@cm"}}],"query":{"guid":"-38791214447462036","typeName":"impala_process","uniqueAttributes":{"qualifiedName":"QUERY:db_mzuh.tbl_mzuh_0@cm:1657695719000:db_mzuh.tbl_mzuh_1@cm:1657695725000->:INSERT:db_mzuh.tbl_mzuh_2@cm:1657695732000"}},"name":"QUERY:db_mzuh.tbl_mzuh_0@cm:1657695719000:db_mzuh.tbl_mzuh_1@cm:1657695725000->:INSERT:db_mzuh.tbl_mzuh_2@cm:1657695732000:tbl_mzuh_2_col_1"},"guid":"-38791214447462039","isIncomplete":false,"provenanceType":0,"version":0,"proxy":false}]}}}{code} h4. > Source information is missing in the Hook message of Spark entries > ------------------------------------------------------------------ > > Key: ATLAS-4635 > URL: https://issues.apache.org/jira/browse/ATLAS-4635 > Project: Atlas > Issue Type: Bug > Components: atlas-core > Reporter: Dharshana M Krishnamoorthy > Priority: Major > > Source information is missing for impala messages > {code:java} > {"source":{"version":"2.1.0.7.1.8.0-723"},"version":{"version":"1.0.0","versionParts":[1]},"msgCompressionKind":"NONE","msgSplitIdx":1,"msgSplitCount":1,"msgSourceIP":"...","msgCreatedBy":"...","msgCreationTime":1657759634684,"spooled":false,"message":...} > {code} > This is a message from ATLAS_SPARK_HOOK, the source infor of the message is > missing here > Expected: > "source":\{"source": "spark", "version":"2.1.0.7.1.8.0-723"}, > but source is missing > It is available for other services: > HMS: > {code:java} > > {"source":{"version":"2.1.0.7.1.8.0-723","source":"hive_metastore"},"version":{"version":"1.0.0","versionParts":[1]},"msgCompressionKind":"NONE","msgSplitIdx":1,"msgSplitCount":1,"msgSourceIP":"...","msgCreatedBy":"...","msgCreationTime":1657700529795,"spooled":false,"message":{ > {code} > {color:#009100}...{color} > {code:java} > }{code} > HBase: > {code:java} > > {"source":{"version":"2.1.0.7.1.8.0-723","source":"hbase"},"version":{"version":"1.0.0","versionParts":[1]},"msgCompressionKind":"NONE","msgSplitIdx":1,"msgSplitCount":1,"msgSourceIP":"...","msgCreatedBy":"...","msgCreationTime":1657700897665,"spooled":false,"message":{ > {code} > {color:#009100}...{color} > {code:java} > }{code} > Impala: > {code:java} > > {"source":{"version":"2.1.0.7.1.8.0-723","source":"impala"},"version":{"version":"1.0.0","versionParts":[1]},"msgCompressionKind":"NONE","msgSplitIdx":1,"msgSplitCount":1,"msgSourceIP":"...","msgCreatedBy":"...","msgCreationTime":1657695780022,"spooled":false,"message":{"...}}{code} > h4. -- This message was sent by Atlassian Jira (v8.20.10#820010)