Disha Talreja created ATLAS-4674:
------------------------------------
Summary: Regression: Classification tagging is not sending
appropriate notification to ATLAS_ENTITIES
Key: ATLAS-4674
URL: https://issues.apache.org/jira/browse/ATLAS-4674
Project: Atlas
Issue Type: Bug
Components: atlas-core
Reporter: Disha Talreja
Assignee: Disha Talreja
When we add/delete a classification from an entity, we send notification to
ATLAS_ENTITIES so ranger tag sync works appropriately to facilitate Tag-based
policy. This functionally seems to be broken and is a regression.
On further investigation, it is seen that the issue is caused by deferred
action for classification propagation as the right notifications are sent when
deferred action is disabled.
Considering the following exampe:
1. Create two table as shown below at Hive:
=====================================
create table parent_table(ssn varchar(100));
create table child_table as select * from parent_table;
2. Created two classification: "PII" and "sensitive"
3. Tagged PII to parent_table and notification received at ATLAS_ENTITIES topic:
-----------------------------------------------------------------------------------------------------------
{
"source": {},
"version": {
"version": "1.0.0",
"versionParts": [
1
]
},
"msgCompressionKind": "NONE",
"msgSplitIdx": 1,
"msgSplitCount": 1,
"msgSourceIP": "172.27.40.5",
"msgCreatedBy": "",
"msgCreationTime": 1662660308744,
"spooled": false,
"message": {
"type": "ENTITY_NOTIFICATION_V2",
"entity": {
"typeName": "hive_table",
"attributes": {
"owner": "admin",
"createTime": 1662602695000,
"qualifiedName": "default.parent_table@cm",
"name": "parent_table" },
"guid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f",
"status": "ACTIVE",
"displayText": "parent_table",
"classificationNames": [
"PII" ],
"classifications": [
{
"typeName": "PII",
"entityGuid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f",
"entityStatus": "ACTIVE",
"propagate": true,
"validityPeriods": [],
"removePropagationsOnEntityDelete": false }
],
"isIncomplete": false },
"operationType": "CLASSIFICATION_ADD",
"eventTime": 1662660307360
}
}
{
"source": {},
"version": {
"version": "1.0.0",
"versionParts": [
1
]
},
"msgCompressionKind": "NONE",
"msgSplitIdx": 1,
"msgSplitCount": 1,
"msgSourceIP": "172.27.40.5",
"msgCreatedBy": "",
"msgCreationTime": 1662660309419,
"spooled": false,
"message": {
"type": "ENTITY_NOTIFICATION_V2",
"entity": {
"typeName": "hive_process",
"attributes": {
"qualifiedName": "default.child_table@cm:1662602726000",
"clusterName": "cm",
"name": "default.child_table@cm:1662602726000" },
"guid": "0e180d04-79ed-4cab-bbc3-76af4b876a5b",
"status": "ACTIVE",
"displayText": "default.child_table@cm:1662602726000",
"classificationNames": [
"PII" ],
"classifications": [
{
"typeName": "PII",
"entityGuid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f",
"entityStatus": "ACTIVE",
"propagate": true,
"validityPeriods": [],
"removePropagationsOnEntityDelete": false }
],
"isIncomplete": false },
"operationType": "CLASSIFICATION_ADD",
"eventTime": 1662660308935
}
}
{
"source": {},
"version": {
"version": "1.0.0",
"versionParts": [
1
]
},
"msgCompressionKind": "NONE",
"msgSplitIdx": 1,
"msgSplitCount": 1,
"msgSourceIP": "172.27.40.5",
"msgCreatedBy": "",
"msgCreationTime": 1662660309419,
"spooled": false,
"message": {
"type": "ENTITY_NOTIFICATION_V2",
"entity": {
"typeName": "hive_table",
"attributes": {
"owner": "admin",
"createTime": 1662602726000,
"qualifiedName": "default.child_table@cm",
"name": "child_table" },
"guid": "117bc1c5-748c-4cbd-a86f-8369b7167b1f",
"status": "ACTIVE",
"displayText": "child_table",
"classificationNames": [
"PII" ],
"classifications": [
{
"typeName": "PII",
"entityGuid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f",
"entityStatus": "ACTIVE",
"propagate": true,
"validityPeriods": [],
"removePropagationsOnEntityDelete": false }
],
"isIncomplete": false },
"operationType": "CLASSIFICATION_ADD",
"eventTime": 1662660308935
}
}
4. Tagged another classification "sensitive" to parent_table and the
notification at ATLAS_ENTITIES topic:
-----------------------------------------------------------------------------------------------------
{
"source": {},
"version": {
"version": "1.0.0",
"versionParts": [
1
]
},
"msgCompressionKind": "NONE",
"msgSplitIdx": 1,
"msgSplitCount": 1,
"msgSourceIP": "172.27.40.5",
"msgCreatedBy": "",
"msgCreationTime": 1662660417995,
"spooled": false,
"message": {
"type": "ENTITY_NOTIFICATION_V2",
"entity": {
"typeName": "hive_table",
"attributes": {
"owner": "admin",
"createTime": 1662602695000,
"qualifiedName": "default.parent_table@cm",
"name": "parent_table" },
"guid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f",
"status": "ACTIVE",
"displayText": "parent_table",
"classificationNames": [
"PII",
"sensitive" ],
"classifications": [
{
"typeName": "PII",
"entityGuid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f",
"entityStatus": "ACTIVE",
"propagate": true,
"validityPeriods": [],
"removePropagationsOnEntityDelete": false },
{
"typeName": "sensitive",
"entityGuid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f",
"entityStatus": "ACTIVE",
"propagate": true,
"validityPeriods": [],
"removePropagationsOnEntityDelete": false }
],
"isIncomplete": false },
"operationType": "CLASSIFICATION_ADD",
"eventTime": 1662660417115
}
}
{
"source": {},
"version": {
"version": "1.0.0",
"versionParts": [
1
]
},
"msgCompressionKind": "NONE",
"msgSplitIdx": 1,
"msgSplitCount": 1,
"msgSourceIP": "172.27.40.5",
"msgCreatedBy": "",
"msgCreationTime": 1662660418085,
"spooled": false,
"message": {
"type": "ENTITY_NOTIFICATION_V2",
"entity": {
"typeName": "hive_process",
"attributes": {
"qualifiedName": "default.child_table@cm:1662602726000",
"clusterName": "cm",
"name": "default.child_table@cm:1662602726000" },
"guid": "0e180d04-79ed-4cab-bbc3-76af4b876a5b",
"status": "ACTIVE",
"displayText": "default.child_table@cm:1662602726000",
"classificationNames": [
"PII" ],
"classifications": [
{
"typeName": "PII",
"entityGuid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f",
"entityStatus": "ACTIVE",
"propagate": true,
"validityPeriods": [],
"removePropagationsOnEntityDelete": false }
],
"isIncomplete": false },
"operationType": "CLASSIFICATION_ADD",
"eventTime": 1662660308935
}
}
{
"source": {},
"version": {
"version": "1.0.0",
"versionParts": [
1
]
},
"msgCompressionKind": "NONE",
"msgSplitIdx": 1,
"msgSplitCount": 1,
"msgSourceIP": "172.27.40.5",
"msgCreatedBy": "",
"msgCreationTime": 1662660418085,
"spooled": false,
"message": {
"type": "ENTITY_NOTIFICATION_V2",
"entity": {
"typeName": "hive_table",
"attributes": {
"owner": "admin",
"createTime": 1662602726000,
"qualifiedName": "default.child_table@cm",
"name": "child_table" },
"guid": "117bc1c5-748c-4cbd-a86f-8369b7167b1f",
"status": "ACTIVE",
"displayText": "child_table",
"classificationNames": [
"PII" ],
"classifications": [
{
"typeName": "PII",
"entityGuid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f",
"entityStatus": "ACTIVE",
"propagate": true,
"validityPeriods": [],
"removePropagationsOnEntityDelete": false }
],
"isIncomplete": false },
"operationType": "CLASSIFICATION_ADD",
"eventTime": 1662660308935
}
}
5. Removed "PII" tag from parent_table and the notification:
------------------------------------------------------------
{
"source": {},
"version": {
"version": "1.0.0",
"versionParts": [
1
]
},
"msgCompressionKind": "NONE",
"msgSplitIdx": 1,
"msgSplitCount": 1,
"msgSourceIP": "172.27.40.5",
"msgCreatedBy": "",
"msgCreationTime": 1662660456406,
"spooled": false,
"message": {
"type": "ENTITY_NOTIFICATION_V2",
"entity": {
"typeName": "hive_table",
"attributes": {
"owner": "admin",
"createTime": 1662602695000,
"qualifiedName": "default.parent_table@cm",
"name": "parent_table" },
"guid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f",
"status": "ACTIVE",
"displayText": "parent_table",
"classificationNames": [
"sensitive" ],
"classifications": [
{
"typeName": "sensitive",
"entityGuid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f",
"entityStatus": "ACTIVE",
"propagate": true,
"validityPeriods": [],
"removePropagationsOnEntityDelete": false }
],
"isIncomplete": false },
"operationType": "CLASSIFICATION_DELETE",
"eventTime": 1662660456206
}
}
{
"source": {},
"version": {
"version": "1.0.0",
"versionParts": [
1
]
},
"msgCompressionKind": "NONE",
"msgSplitIdx": 1,
"msgSplitCount": 1,
"msgSourceIP": "172.27.40.5",
"msgCreatedBy": "",
"msgCreationTime": 1662660456586,
"spooled": false,
"message": {
"type": "ENTITY_NOTIFICATION_V2",
"entity": {
"typeName": "hive_table",
"attributes": {
"owner": "admin",
"createTime": 1662602726000,
"qualifiedName": "default.child_table@cm",
"name": "child_table" },
"guid": "117bc1c5-748c-4cbd-a86f-8369b7167b1f",
"status": "ACTIVE",
"displayText": "child_table",
"classificationNames": [
"PII" ],
"classifications": [
{
"typeName": "PII",
"entityGuid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f",
"entityStatus": "ACTIVE",
"propagate": true,
"validityPeriods": [],
"removePropagationsOnEntityDelete": false }
],
"isIncomplete": false },
"operationType": "CLASSIFICATION_DELETE",
"eventTime": 1662660308935
}
}
{
"source": {},
"version": {
"version": "1.0.0",
"versionParts": [
1
]
},
"msgCompressionKind": "NONE",
"msgSplitIdx": 1,
"msgSplitCount": 1,
"msgSourceIP": "172.27.40.5",
"msgCreatedBy": "",
"msgCreationTime": 1662660456587,
"spooled": false,
"message": {
"type": "ENTITY_NOTIFICATION_V2",
"entity": {
"typeName": "hive_process",
"attributes": {
"qualifiedName": "default.child_table@cm:1662602726000",
"clusterName": "cm",
"name": "default.child_table@cm:1662602726000" },
"guid": "0e180d04-79ed-4cab-bbc3-76af4b876a5b",
"status": "ACTIVE",
"displayText": "default.child_table@cm:1662602726000",
"classificationNames": [
"PII" ],
"classifications": [
{
"typeName": "PII",
"entityGuid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f",
"entityStatus": "ACTIVE",
"propagate": true,
"validityPeriods": [],
"removePropagationsOnEntityDelete": false }
],
"isIncomplete": false },
"operationType": "CLASSIFICATION_DELETE",
"eventTime": 1662660308935
}
}
>From the above notification messages, it is clear that the functionality is
>broken as the messages received are incorrect in case of the second tag
>(sensitive) being attached and on classification delete.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
