Ayub Pathan created ATLAS-2202:
----------------------------------

             Summary: Duplicate kafka notifications observed for external table
                 Key: ATLAS-2202
                 URL: https://issues.apache.org/jira/browse/ATLAS-2202
             Project: Atlas
          Issue Type: Bug
          Components: atlas-intg
    Affects Versions: 0.8.2
            Reporter: Ayub Pathan
             Fix For: 0.8.2



Steps to reproduce: Create external table and observe the kafka notifications.
* This notification includes hive_db and external hive_table notification
{noformat}
{"msgSourceIP":"172.27.39.68","msgCreatedBy":"hrt_qa","msgCreationTime":1507714828002,"message":{"entities":[{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Reference","id":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-1706538104304055","version":0,"typeName":"hive_db","state":"ACTIVE"},"typeName":"hive_db","values":{"name":"default","location":"hdfs://ctr-e134-1499953498516-214922-01-000003.hwx.site:8020/apps/hive/warehouse","description":"Default
 Hive 
database","ownerType":2,"qualifiedName":"default@cl1","owner":"public","clusterName":"cl1","parameters":{}},"traitNames":[],"traits":{},"systemAttributes":{}},{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Reference","id":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-1706538104304054","version":0,"typeName":"hive_table","state":"ACTIVE"},"typeName":"hive_table","values":{"tableType":"EXTERNAL_TABLE","name":"externaltable_2","createTime":"2017-10-11T09:40:27.000Z","temporary":false,"db":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Reference","id":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-1706538104304055","version":0,"typeName":"hive_db","state":"ACTIVE"},"typeName":"hive_db","values":{"name":"default","location":"hdfs://ctr-e134-1499953498516-214922-01-000003.hwx.site:8020/apps/hive/warehouse","description":"Default
 Hive 
database","ownerType":2,"qualifiedName":"default@cl1","owner":"public","clusterName":"cl1","parameters":{}},"traitNames":[],"traits":{},"systemAttributes":{}},"retention":0,"qualifiedName":"default.externaltable_2@cl1","columns":[{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Reference","id":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-1706538104304052","version":0,"typeName":"hive_column","state":"ACTIVE"},"typeName":"hive_column","values":{"name":"id","qualifiedName":"default.externaltable_2.id@cl1","position":0,"owner":"hrt_qa","type":"int","table":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-1706538104304054","version":0,"typeName":"hive_table","state":"ACTIVE"}},"traitNames":[],"traits":{},"systemAttributes":{}},{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Reference","id":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-1706538104304051","version":0,"typeName":"hive_column","state":"ACTIVE"},"typeName":"hive_column","values":{"name":"name","qualifiedName":"default.externaltable_2.name@cl1","position":1,"owner":"hrt_qa","type":"string","table":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-1706538104304054","version":0,"typeName":"hive_table","state":"ACTIVE"}},"traitNames":[],"traits":{},"systemAttributes":{}}],"lastAccessTime":"2017-10-11T09:40:27.000Z","owner":"hrt_qa","sd":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Reference","id":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-1706538104304053","version":0,"typeName":"hive_storagedesc","state":"ACTIVE"},"typeName":"hive_storagedesc","values":{"location":"hdfs://ctr-e134-1499953498516-214922-01-000003.hwx.site:8020/apps/hive/warehouse/externaltable_2","serdeInfo":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Struct","typeName":"hive_serde","values":{"serializationLib":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe","parameters":{"serialization.format":"1"}}},"qualifiedName":"default.externaltable_2@cl1_storage","outputFormat":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat","compressed":false,"numBuckets":-1,"inputFormat":"org.apache.hadoop.mapred.TextInputFormat","parameters":{},"storedAsSubDirectories":false,"table":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-1706538104304054","version":0,"typeName":"hive_table","state":"ACTIVE"}},"traitNames":[],"traits":{},"systemAttributes":{}},"parameters":{"rawDataSize":"0","numFiles":"0","transient_lastDdlTime":"1507714827","totalSize":"0","EXTERNAL":"TRUE","COLUMN_STATS_ACCURATE":"{\"BASIC_STATS\":\"true\"}","numRows":"0"},"partitionKeys":[]},"traitNames":[],"traits":{},"systemAttributes":{}}],"type":"ENTITY_FULL_UPDATE","user":"hrt_qa"},"version":{"version":"1.0.0"},"msgCompressionKind":"NONE","msgSplitIdx":1,"msgSplitCount":1}
{noformat}


* Duplicate notification for external hive table creation, which is already 
done as part of above notification.
{noformat}
{"msgSourceIP":"172.27.39.68","msgCreatedBy":"hrt_qa","msgCreationTime":1507714828004,"message":{"entities":[{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Reference","id":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-1706538104304054","version":0,"typeName":"hive_table","state":"ACTIVE"},"typeName":"hive_table","values":{"tableType":"EXTERNAL_TABLE","name":"externaltable_2","createTime":"2017-10-11T09:40:27.000Z","temporary":false,"db":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Reference","id":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-1706538104304055","version":0,"typeName":"hive_db","state":"ACTIVE"},"typeName":"hive_db","values":{"name":"default","location":"hdfs://ctr-e134-1499953498516-214922-01-000003.hwx.site:8020/apps/hive/warehouse","description":"Default
 Hive 
database","ownerType":2,"qualifiedName":"default@cl1","owner":"public","clusterName":"cl1","parameters":{}},"traitNames":[],"traits":{},"systemAttributes":{}},"retention":0,"qualifiedName":"default.externaltable_2@cl1","columns":[{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Reference","id":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-1706538104304052","version":0,"typeName":"hive_column","state":"ACTIVE"},"typeName":"hive_column","values":{"name":"id","qualifiedName":"default.externaltable_2.id@cl1","position":0,"owner":"hrt_qa","type":"int","table":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-1706538104304054","version":0,"typeName":"hive_table","state":"ACTIVE"}},"traitNames":[],"traits":{},"systemAttributes":{}},{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Reference","id":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-1706538104304051","version":0,"typeName":"hive_column","state":"ACTIVE"},"typeName":"hive_column","values":{"name":"name","qualifiedName":"default.externaltable_2.name@cl1","position":1,"owner":"hrt_qa","type":"string","table":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-1706538104304054","version":0,"typeName":"hive_table","state":"ACTIVE"}},"traitNames":[],"traits":{},"systemAttributes":{}}],"lastAccessTime":"2017-10-11T09:40:27.000Z","owner":"hrt_qa","sd":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Reference","id":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-1706538104304053","version":0,"typeName":"hive_storagedesc","state":"ACTIVE"},"typeName":"hive_storagedesc","values":{"location":"hdfs://ctr-e134-1499953498516-214922-01-000003.hwx.site:8020/apps/hive/warehouse/externaltable_2","serdeInfo":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Struct","typeName":"hive_serde","values":{"serializationLib":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe","parameters":{"serialization.format":"1"}}},"qualifiedName":"default.externaltable_2@cl1_storage","outputFormat":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat","compressed":false,"numBuckets":-1,"inputFormat":"org.apache.hadoop.mapred.TextInputFormat","parameters":{},"storedAsSubDirectories":false,"table":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-1706538104304054","version":0,"typeName":"hive_table","state":"ACTIVE"}},"traitNames":[],"traits":{},"systemAttributes":{}},"parameters":{"rawDataSize":"0","numFiles":"0","transient_lastDdlTime":"1507714827","totalSize":"0","EXTERNAL":"TRUE","COLUMN_STATS_ACCURATE":"{\"BASIC_STATS\":\"true\"}","numRows":"0"},"partitionKeys":[]},"traitNames":[],"traits":{},"systemAttributes":{}}],"type":"ENTITY_FULL_UPDATE","user":"hrt_qa"},"version":{"version":"1.0.0"},"msgCompressionKind":"NONE","msgSplitIdx":1,"msgSplitCount":1}
{noformat}

* Hive process creation notification
{noformat}
{"msgSourceIP":"172.27.39.68","msgCreatedBy":"hrt_qa","msgCreationTime":1507714828005,"message":{"entities":[{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Reference","id":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-1706538104304049","version":0,"typeName":"hive_process","state":"ACTIVE"},"typeName":"hive_process","values":{"queryId":"hive_20171011094027_8888d280-af97-41ff-8ffe-cf56cb47213f","name":"create
 external table `externaltable_2` (id int, name 
string)","startTime":"2017-10-11T09:40:27.495Z","queryPlan":"Not 
Supported","operationType":"CREATETABLE","outputs":[{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Reference","id":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-1706538104304054","version":0,"typeName":"hive_table","state":"ACTIVE"},"typeName":"hive_table","values":{"tableType":"EXTERNAL_TABLE","name":"externaltable_2","createTime":"2017-10-11T09:40:27.000Z","temporary":false,"db":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Reference","id":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-1706538104304055","version":0,"typeName":"hive_db","state":"ACTIVE"},"typeName":"hive_db","values":{"name":"default","location":"hdfs://ctr-e134-1499953498516-214922-01-000003.hwx.site:8020/apps/hive/warehouse","description":"Default
 Hive 
database","ownerType":2,"qualifiedName":"default@cl1","owner":"public","clusterName":"cl1","parameters":{}},"traitNames":[],"traits":{},"systemAttributes":{}},"retention":0,"qualifiedName":"default.externaltable_2@cl1","columns":[{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Reference","id":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-1706538104304052","version":0,"typeName":"hive_column","state":"ACTIVE"},"typeName":"hive_column","values":{"name":"id","qualifiedName":"default.externaltable_2.id@cl1","position":0,"owner":"hrt_qa","type":"int","table":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-1706538104304054","version":0,"typeName":"hive_table","state":"ACTIVE"}},"traitNames":[],"traits":{},"systemAttributes":{}},{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Reference","id":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-1706538104304051","version":0,"typeName":"hive_column","state":"ACTIVE"},"typeName":"hive_column","values":{"name":"name","qualifiedName":"default.externaltable_2.name@cl1","position":1,"owner":"hrt_qa","type":"string","table":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-1706538104304054","version":0,"typeName":"hive_table","state":"ACTIVE"}},"traitNames":[],"traits":{},"systemAttributes":{}}],"lastAccessTime":"2017-10-11T09:40:27.000Z","owner":"hrt_qa","sd":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Reference","id":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-1706538104304053","version":0,"typeName":"hive_storagedesc","state":"ACTIVE"},"typeName":"hive_storagedesc","values":{"location":"hdfs://ctr-e134-1499953498516-214922-01-000003.hwx.site:8020/apps/hive/warehouse/externaltable_2","serdeInfo":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Struct","typeName":"hive_serde","values":{"serializationLib":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe","parameters":{"serialization.format":"1"}}},"qualifiedName":"default.externaltable_2@cl1_storage","outputFormat":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat","compressed":false,"numBuckets":-1,"inputFormat":"org.apache.hadoop.mapred.TextInputFormat","parameters":{},"storedAsSubDirectories":false,"table":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-1706538104304054","version":0,"typeName":"hive_table","state":"ACTIVE"}},"traitNames":[],"traits":{},"systemAttributes":{}},"parameters":{"rawDataSize":"0","numFiles":"0","transient_lastDdlTime":"1507714827","totalSize":"0","EXTERNAL":"TRUE","COLUMN_STATS_ACCURATE":"{\"BASIC_STATS\":\"true\"}","numRows":"0"},"partitionKeys":[]},"traitNames":[],"traits":{},"systemAttributes":{}}],"endTime":"2017-10-11T09:40:28.002Z","recentQueries":["create
 external table `externaltable_2` (id int, name 
string)"],"inputs":[{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Reference","id":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-1706538104304050","version":0,"typeName":"hdfs_path","state":"ACTIVE"},"typeName":"hdfs_path","values":{"qualifiedName":"hdfs://ctr-e134-1499953498516-214922-01-000003.hwx.site:8020/apps/hive/warehouse/externaltable_2","name":"/apps/hive/warehouse/externaltable_2","clusterName":"cl1","path":"hdfs://ctr-e134-1499953498516-214922-01-000003.hwx.site:8020/apps/hive/warehouse/externaltable_2"},"traitNames":[],"traits":{},"systemAttributes":{}}],"qualifiedName":"default.externaltable_2@cl1:1507714827000","queryText":"create
 external table `externaltable_2` (id int, name 
string)","clusterName":"cl1","userName":"hrt_qa"},"traitNames":[],"traits":{},"systemAttributes":{}}],"type":"ENTITY_FULL_UPDATE","user":"hrt_qa"},"version":{"version":"1.0.0"},"msgCompressionKind":"NONE","msgSplitIdx":1,"msgSplitCount":1}
{noformat}


This regression is introduced likely by the ATLAS-2198 fix.



--
This message was sent by Atlassian JIRA
(v6.4.14#64029)

Reply via email to