[
https://issues.apache.org/jira/browse/ATLAS-4674?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Disha Talreja updated ATLAS-4674:
---------------------------------
Description:
When we add/delete a classification from an entity, we send notification to
ATLAS_ENTITIES so ranger tag sync works appropriately to facilitate Tag-based
policy. This functionally seems to be broken and is a regression.
On further investigation, it is seen that the issue is caused by deferred
action for classification propagation as the right notifications are sent when
deferred action is disabled.
Considering the following example:
1. Create two table as shown below at Hive:
=====================================
create table parent_table(ssn varchar(100));
create table child_table as select * from parent_table;
2. Created two classification: "PII" and "sensitive"
3. Tagged PII to parent_table and notification received at ATLAS_ENTITIES topic:
-----------------------------------------------------------------------------------------------------------
{code:java}
{
"source": {},
"version":
{ "version": "1.0.0", "versionParts": [ 1 ] }
,
"msgCompressionKind": "NONE",
"msgSplitIdx": 1,
"msgSplitCount": 1,
"msgSourceIP": "172.27.40.5",
"msgCreatedBy": "",
"msgCreationTime": 1662660308744,
"spooled": false,
"message": {
"type": "ENTITY_NOTIFICATION_V2",
"entity": {
"typeName": "hive_table",
"attributes":
{ "owner": "admin", "createTime": 1662602695000,
"qualifiedName": "default.parent_table@cm", "name": "parent_table"
}
,
"guid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f",
"status": "ACTIVE",
"displayText": "parent_table",
"classificationNames": [
"PII" ],
"classifications": [
{ "typeName": "PII", "entityGuid":
"04ff79d7-b1ff-4354-ba75-a35b28d2f53f", "entityStatus": "ACTIVE",
"propagate": true, "validityPeriods": [],
"removePropagationsOnEntityDelete": false }
],
"isIncomplete": false },
"operationType": "CLASSIFICATION_ADD",
"eventTime": 1662660307360
}
}
{
"source": {},
"version":
{ "version": "1.0.0", "versionParts": [ 1 ] }
,
"msgCompressionKind": "NONE",
"msgSplitIdx": 1,
"msgSplitCount": 1,
"msgSourceIP": "172.27.40.5",
"msgCreatedBy": "",
"msgCreationTime": 1662660309419,
"spooled": false,
"message": {
"type": "ENTITY_NOTIFICATION_V2",
"entity": {
"typeName": "hive_process",
"attributes":
{ "qualifiedName": "default.child_table@cm:1662602726000",
"clusterName": "cm", "name": "default.child_table@cm:1662602726000"
}
,
"guid": "0e180d04-79ed-4cab-bbc3-76af4b876a5b",
"status": "ACTIVE",
"displayText": "default.child_table@cm:1662602726000",
"classificationNames": [
"PII" ],
"classifications": [
{ "typeName": "PII", "entityGuid":
"04ff79d7-b1ff-4354-ba75-a35b28d2f53f", "entityStatus": "ACTIVE",
"propagate": true, "validityPeriods": [],
"removePropagationsOnEntityDelete": false }
],
"isIncomplete": false },
"operationType": "CLASSIFICATION_ADD",
"eventTime": 1662660308935
}
}
{
"source": {},
"version":
{ "version": "1.0.0", "versionParts": [ 1 ] }
,
"msgCompressionKind": "NONE",
"msgSplitIdx": 1,
"msgSplitCount": 1,
"msgSourceIP": "172.27.40.5",
"msgCreatedBy": "",
"msgCreationTime": 1662660309419,
"spooled": false,
"message": {
"type": "ENTITY_NOTIFICATION_V2",
"entity": {
"typeName": "hive_table",
"attributes":
{ "owner": "admin", "createTime": 1662602726000,
"qualifiedName": "default.child_table@cm", "name": "child_table" }
,
"guid": "117bc1c5-748c-4cbd-a86f-8369b7167b1f",
"status": "ACTIVE",
"displayText": "child_table",
"classificationNames": [
"PII" ],
"classifications": [
{ "typeName": "PII", "entityGuid":
"04ff79d7-b1ff-4354-ba75-a35b28d2f53f", "entityStatus": "ACTIVE",
"propagate": true, "validityPeriods": [],
"removePropagationsOnEntityDelete": false }
],
"isIncomplete": false },
"operationType": "CLASSIFICATION_ADD",
"eventTime": 1662660308935
}
}
{code}
4. Tagged another classification "sensitive" to parent_table and the
notification at ATLAS_ENTITIES topic:
-----------------------------------------------------------------------------------------------------
{code:java}
{
"source": {},
"version":
{ "version": "1.0.0", "versionParts": [ 1 ] }
,
"msgCompressionKind": "NONE",
"msgSplitIdx": 1,
"msgSplitCount": 1,
"msgSourceIP": "172.27.40.5",
"msgCreatedBy": "",
"msgCreationTime": 1662660417995,
"spooled": false,
"message": {
"type": "ENTITY_NOTIFICATION_V2",
"entity": {
"typeName": "hive_table",
"attributes":
{ "owner": "admin", "createTime": 1662602695000,
"qualifiedName": "default.parent_table@cm", "name": "parent_table"
}
,
"guid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f",
"status": "ACTIVE",
"displayText": "parent_table",
"classificationNames": [
"PII",
"sensitive" ],
"classifications": [
{ "typeName": "PII", "entityGuid":
"04ff79d7-b1ff-4354-ba75-a35b28d2f53f", "entityStatus": "ACTIVE",
"propagate": true, "validityPeriods": [],
"removePropagationsOnEntityDelete": false }
,
{ "typeName": "sensitive", "entityGuid":
"04ff79d7-b1ff-4354-ba75-a35b28d2f53f", "entityStatus": "ACTIVE",
"propagate": true, "validityPeriods": [],
"removePropagationsOnEntityDelete": false }
],
"isIncomplete": false },
"operationType": "CLASSIFICATION_ADD",
"eventTime": 1662660417115
}
}
{
"source": {},
"version":
{ "version": "1.0.0", "versionParts": [ 1 ] }
,
"msgCompressionKind": "NONE",
"msgSplitIdx": 1,
"msgSplitCount": 1,
"msgSourceIP": "172.27.40.5",
"msgCreatedBy": "",
"msgCreationTime": 1662660418085,
"spooled": false,
"message": {
"type": "ENTITY_NOTIFICATION_V2",
"entity": {
"typeName": "hive_process",
"attributes":
{ "qualifiedName": "default.child_table@cm:1662602726000",
"clusterName": "cm", "name": "default.child_table@cm:1662602726000"
}
,
"guid": "0e180d04-79ed-4cab-bbc3-76af4b876a5b",
"status": "ACTIVE",
"displayText": "default.child_table@cm:1662602726000",
"classificationNames": [
"PII" ],
"classifications": [
{ "typeName": "PII", "entityGuid":
"04ff79d7-b1ff-4354-ba75-a35b28d2f53f", "entityStatus": "ACTIVE",
"propagate": true, "validityPeriods": [],
"removePropagationsOnEntityDelete": false }
],
"isIncomplete": false },
"operationType": "CLASSIFICATION_ADD",
"eventTime": 1662660308935
}
}
{
"source": {},
"version":
{ "version": "1.0.0", "versionParts": [ 1 ] }
,
"msgCompressionKind": "NONE",
"msgSplitIdx": 1,
"msgSplitCount": 1,
"msgSourceIP": "172.27.40.5",
"msgCreatedBy": "",
"msgCreationTime": 1662660418085,
"spooled": false,
"message": {
"type": "ENTITY_NOTIFICATION_V2",
"entity": {
"typeName": "hive_table",
"attributes":
{ "owner": "admin", "createTime": 1662602726000,
"qualifiedName": "default.child_table@cm", "name": "child_table" }
,
"guid": "117bc1c5-748c-4cbd-a86f-8369b7167b1f",
"status": "ACTIVE",
"displayText": "child_table",
"classificationNames": [
"PII" ],
"classifications": [
{ "typeName": "PII", "entityGuid":
"04ff79d7-b1ff-4354-ba75-a35b28d2f53f", "entityStatus": "ACTIVE",
"propagate": true, "validityPeriods": [],
"removePropagationsOnEntityDelete": false }
],
"isIncomplete": false },
"operationType": "CLASSIFICATION_ADD",
"eventTime": 1662660308935
}
}
{code}
5. Removed "PII" tag from parent_table and the notification:
------------------------------------------------------------
{code:java}
{
"source": {},
"version":
{ "version": "1.0.0", "versionParts": [ 1 ] }
,
"msgCompressionKind": "NONE",
"msgSplitIdx": 1,
"msgSplitCount": 1,
"msgSourceIP": "172.27.40.5",
"msgCreatedBy": "",
"msgCreationTime": 1662660456406,
"spooled": false,
"message": {
"type": "ENTITY_NOTIFICATION_V2",
"entity": {
"typeName": "hive_table",
"attributes":
{ "owner": "admin", "createTime": 1662602695000,
"qualifiedName": "default.parent_table@cm", "name": "parent_table"
}
,
"guid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f",
"status": "ACTIVE",
"displayText": "parent_table",
"classificationNames": [
"sensitive" ],
"classifications": [
{ "typeName": "sensitive", "entityGuid":
"04ff79d7-b1ff-4354-ba75-a35b28d2f53f", "entityStatus": "ACTIVE",
"propagate": true, "validityPeriods": [],
"removePropagationsOnEntityDelete": false }
],
"isIncomplete": false },
"operationType": "CLASSIFICATION_DELETE",
"eventTime": 1662660456206
}
}
{
"source": {},
"version":
{ "version": "1.0.0", "versionParts": [ 1 ] }
,
"msgCompressionKind": "NONE",
"msgSplitIdx": 1,
"msgSplitCount": 1,
"msgSourceIP": "172.27.40.5",
"msgCreatedBy": "",
"msgCreationTime": 1662660456586,
"spooled": false,
"message": {
"type": "ENTITY_NOTIFICATION_V2",
"entity": {
"typeName": "hive_table",
"attributes":
{ "owner": "admin", "createTime": 1662602726000,
"qualifiedName": "default.child_table@cm", "name": "child_table" }
,
"guid": "117bc1c5-748c-4cbd-a86f-8369b7167b1f",
"status": "ACTIVE",
"displayText": "child_table",
"classificationNames": [
"PII" ],
"classifications": [
{ "typeName": "PII", "entityGuid":
"04ff79d7-b1ff-4354-ba75-a35b28d2f53f", "entityStatus": "ACTIVE",
"propagate": true, "validityPeriods": [],
"removePropagationsOnEntityDelete": false }
],
"isIncomplete": false },
"operationType": "CLASSIFICATION_DELETE",
"eventTime": 1662660308935
}
}
{
"source": {},
"version":
{ "version": "1.0.0", "versionParts": [ 1 ] }
,
"msgCompressionKind": "NONE",
"msgSplitIdx": 1,
"msgSplitCount": 1,
"msgSourceIP": "172.27.40.5",
"msgCreatedBy": "",
"msgCreationTime": 1662660456587,
"spooled": false,
"message": {
"type": "ENTITY_NOTIFICATION_V2",
"entity": {
"typeName": "hive_process",
"attributes":
{ "qualifiedName": "default.child_table@cm:1662602726000",
"clusterName": "cm", "name": "default.child_table@cm:1662602726000"
}
,
"guid": "0e180d04-79ed-4cab-bbc3-76af4b876a5b",
"status": "ACTIVE",
"displayText": "default.child_table@cm:1662602726000",
"classificationNames": [
"PII" ],
"classifications": [
{ "typeName": "PII", "entityGuid":
"04ff79d7-b1ff-4354-ba75-a35b28d2f53f", "entityStatus": "ACTIVE",
"propagate": true, "validityPeriods": [],
"removePropagationsOnEntityDelete": false }
],
"isIncomplete": false },
"operationType": "CLASSIFICATION_DELETE",
"eventTime": 1662660308935
}
}
{code}
>From the above notification messages, it is clear that the functionality is
>broken as the messages received are incorrect in case of the second tag
>(sensitive) being attached and on classification delete.
was:
When we add/delete a classification from an entity, we send notification to
ATLAS_ENTITIES so ranger tag sync works appropriately to facilitate Tag-based
policy. This functionally seems to be broken and is a regression.
On further investigation, it is seen that the issue is caused by deferred
action for classification propagation as the right notifications are sent when
deferred action is disabled.
Considering the following exampe:
1. Create two table as shown below at Hive:
=====================================
create table parent_table(ssn varchar(100));
create table child_table as select * from parent_table;
2. Created two classification: "PII" and "sensitive"
3. Tagged PII to parent_table and notification received at ATLAS_ENTITIES topic:
-----------------------------------------------------------------------------------------------------------
{
"source": {},
"version": {
"version": "1.0.0",
"versionParts": [
1
]
},
"msgCompressionKind": "NONE",
"msgSplitIdx": 1,
"msgSplitCount": 1,
"msgSourceIP": "172.27.40.5",
"msgCreatedBy": "",
"msgCreationTime": 1662660308744,
"spooled": false,
"message": {
"type": "ENTITY_NOTIFICATION_V2",
"entity": {
"typeName": "hive_table",
"attributes": {
"owner": "admin",
"createTime": 1662602695000,
"qualifiedName": "default.parent_table@cm",
"name": "parent_table" },
"guid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f",
"status": "ACTIVE",
"displayText": "parent_table",
"classificationNames": [
"PII" ],
"classifications": [
{
"typeName": "PII",
"entityGuid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f",
"entityStatus": "ACTIVE",
"propagate": true,
"validityPeriods": [],
"removePropagationsOnEntityDelete": false }
],
"isIncomplete": false },
"operationType": "CLASSIFICATION_ADD",
"eventTime": 1662660307360
}
}
{
"source": {},
"version": {
"version": "1.0.0",
"versionParts": [
1
]
},
"msgCompressionKind": "NONE",
"msgSplitIdx": 1,
"msgSplitCount": 1,
"msgSourceIP": "172.27.40.5",
"msgCreatedBy": "",
"msgCreationTime": 1662660309419,
"spooled": false,
"message": {
"type": "ENTITY_NOTIFICATION_V2",
"entity": {
"typeName": "hive_process",
"attributes": {
"qualifiedName": "default.child_table@cm:1662602726000",
"clusterName": "cm",
"name": "default.child_table@cm:1662602726000" },
"guid": "0e180d04-79ed-4cab-bbc3-76af4b876a5b",
"status": "ACTIVE",
"displayText": "default.child_table@cm:1662602726000",
"classificationNames": [
"PII" ],
"classifications": [
{
"typeName": "PII",
"entityGuid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f",
"entityStatus": "ACTIVE",
"propagate": true,
"validityPeriods": [],
"removePropagationsOnEntityDelete": false }
],
"isIncomplete": false },
"operationType": "CLASSIFICATION_ADD",
"eventTime": 1662660308935
}
}
{
"source": {},
"version": {
"version": "1.0.0",
"versionParts": [
1
]
},
"msgCompressionKind": "NONE",
"msgSplitIdx": 1,
"msgSplitCount": 1,
"msgSourceIP": "172.27.40.5",
"msgCreatedBy": "",
"msgCreationTime": 1662660309419,
"spooled": false,
"message": {
"type": "ENTITY_NOTIFICATION_V2",
"entity": {
"typeName": "hive_table",
"attributes": {
"owner": "admin",
"createTime": 1662602726000,
"qualifiedName": "default.child_table@cm",
"name": "child_table" },
"guid": "117bc1c5-748c-4cbd-a86f-8369b7167b1f",
"status": "ACTIVE",
"displayText": "child_table",
"classificationNames": [
"PII" ],
"classifications": [
{
"typeName": "PII",
"entityGuid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f",
"entityStatus": "ACTIVE",
"propagate": true,
"validityPeriods": [],
"removePropagationsOnEntityDelete": false }
],
"isIncomplete": false },
"operationType": "CLASSIFICATION_ADD",
"eventTime": 1662660308935
}
}
4. Tagged another classification "sensitive" to parent_table and the
notification at ATLAS_ENTITIES topic:
-----------------------------------------------------------------------------------------------------
{
"source": {},
"version": {
"version": "1.0.0",
"versionParts": [
1
]
},
"msgCompressionKind": "NONE",
"msgSplitIdx": 1,
"msgSplitCount": 1,
"msgSourceIP": "172.27.40.5",
"msgCreatedBy": "",
"msgCreationTime": 1662660417995,
"spooled": false,
"message": {
"type": "ENTITY_NOTIFICATION_V2",
"entity": {
"typeName": "hive_table",
"attributes": {
"owner": "admin",
"createTime": 1662602695000,
"qualifiedName": "default.parent_table@cm",
"name": "parent_table" },
"guid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f",
"status": "ACTIVE",
"displayText": "parent_table",
"classificationNames": [
"PII",
"sensitive" ],
"classifications": [
{
"typeName": "PII",
"entityGuid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f",
"entityStatus": "ACTIVE",
"propagate": true,
"validityPeriods": [],
"removePropagationsOnEntityDelete": false },
{
"typeName": "sensitive",
"entityGuid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f",
"entityStatus": "ACTIVE",
"propagate": true,
"validityPeriods": [],
"removePropagationsOnEntityDelete": false }
],
"isIncomplete": false },
"operationType": "CLASSIFICATION_ADD",
"eventTime": 1662660417115
}
}
{
"source": {},
"version": {
"version": "1.0.0",
"versionParts": [
1
]
},
"msgCompressionKind": "NONE",
"msgSplitIdx": 1,
"msgSplitCount": 1,
"msgSourceIP": "172.27.40.5",
"msgCreatedBy": "",
"msgCreationTime": 1662660418085,
"spooled": false,
"message": {
"type": "ENTITY_NOTIFICATION_V2",
"entity": {
"typeName": "hive_process",
"attributes": {
"qualifiedName": "default.child_table@cm:1662602726000",
"clusterName": "cm",
"name": "default.child_table@cm:1662602726000" },
"guid": "0e180d04-79ed-4cab-bbc3-76af4b876a5b",
"status": "ACTIVE",
"displayText": "default.child_table@cm:1662602726000",
"classificationNames": [
"PII" ],
"classifications": [
{
"typeName": "PII",
"entityGuid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f",
"entityStatus": "ACTIVE",
"propagate": true,
"validityPeriods": [],
"removePropagationsOnEntityDelete": false }
],
"isIncomplete": false },
"operationType": "CLASSIFICATION_ADD",
"eventTime": 1662660308935
}
}
{
"source": {},
"version": {
"version": "1.0.0",
"versionParts": [
1
]
},
"msgCompressionKind": "NONE",
"msgSplitIdx": 1,
"msgSplitCount": 1,
"msgSourceIP": "172.27.40.5",
"msgCreatedBy": "",
"msgCreationTime": 1662660418085,
"spooled": false,
"message": {
"type": "ENTITY_NOTIFICATION_V2",
"entity": {
"typeName": "hive_table",
"attributes": {
"owner": "admin",
"createTime": 1662602726000,
"qualifiedName": "default.child_table@cm",
"name": "child_table" },
"guid": "117bc1c5-748c-4cbd-a86f-8369b7167b1f",
"status": "ACTIVE",
"displayText": "child_table",
"classificationNames": [
"PII" ],
"classifications": [
{
"typeName": "PII",
"entityGuid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f",
"entityStatus": "ACTIVE",
"propagate": true,
"validityPeriods": [],
"removePropagationsOnEntityDelete": false }
],
"isIncomplete": false },
"operationType": "CLASSIFICATION_ADD",
"eventTime": 1662660308935
}
}
5. Removed "PII" tag from parent_table and the notification:
------------------------------------------------------------
{
"source": {},
"version": {
"version": "1.0.0",
"versionParts": [
1
]
},
"msgCompressionKind": "NONE",
"msgSplitIdx": 1,
"msgSplitCount": 1,
"msgSourceIP": "172.27.40.5",
"msgCreatedBy": "",
"msgCreationTime": 1662660456406,
"spooled": false,
"message": {
"type": "ENTITY_NOTIFICATION_V2",
"entity": {
"typeName": "hive_table",
"attributes": {
"owner": "admin",
"createTime": 1662602695000,
"qualifiedName": "default.parent_table@cm",
"name": "parent_table" },
"guid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f",
"status": "ACTIVE",
"displayText": "parent_table",
"classificationNames": [
"sensitive" ],
"classifications": [
{
"typeName": "sensitive",
"entityGuid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f",
"entityStatus": "ACTIVE",
"propagate": true,
"validityPeriods": [],
"removePropagationsOnEntityDelete": false }
],
"isIncomplete": false },
"operationType": "CLASSIFICATION_DELETE",
"eventTime": 1662660456206
}
}
{
"source": {},
"version": {
"version": "1.0.0",
"versionParts": [
1
]
},
"msgCompressionKind": "NONE",
"msgSplitIdx": 1,
"msgSplitCount": 1,
"msgSourceIP": "172.27.40.5",
"msgCreatedBy": "",
"msgCreationTime": 1662660456586,
"spooled": false,
"message": {
"type": "ENTITY_NOTIFICATION_V2",
"entity": {
"typeName": "hive_table",
"attributes": {
"owner": "admin",
"createTime": 1662602726000,
"qualifiedName": "default.child_table@cm",
"name": "child_table" },
"guid": "117bc1c5-748c-4cbd-a86f-8369b7167b1f",
"status": "ACTIVE",
"displayText": "child_table",
"classificationNames": [
"PII" ],
"classifications": [
{
"typeName": "PII",
"entityGuid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f",
"entityStatus": "ACTIVE",
"propagate": true,
"validityPeriods": [],
"removePropagationsOnEntityDelete": false }
],
"isIncomplete": false },
"operationType": "CLASSIFICATION_DELETE",
"eventTime": 1662660308935
}
}
{
"source": {},
"version": {
"version": "1.0.0",
"versionParts": [
1
]
},
"msgCompressionKind": "NONE",
"msgSplitIdx": 1,
"msgSplitCount": 1,
"msgSourceIP": "172.27.40.5",
"msgCreatedBy": "",
"msgCreationTime": 1662660456587,
"spooled": false,
"message": {
"type": "ENTITY_NOTIFICATION_V2",
"entity": {
"typeName": "hive_process",
"attributes": {
"qualifiedName": "default.child_table@cm:1662602726000",
"clusterName": "cm",
"name": "default.child_table@cm:1662602726000" },
"guid": "0e180d04-79ed-4cab-bbc3-76af4b876a5b",
"status": "ACTIVE",
"displayText": "default.child_table@cm:1662602726000",
"classificationNames": [
"PII" ],
"classifications": [
{
"typeName": "PII",
"entityGuid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f",
"entityStatus": "ACTIVE",
"propagate": true,
"validityPeriods": [],
"removePropagationsOnEntityDelete": false }
],
"isIncomplete": false },
"operationType": "CLASSIFICATION_DELETE",
"eventTime": 1662660308935
}
}
>From the above notification messages, it is clear that the functionality is
>broken as the messages received are incorrect in case of the second tag
>(sensitive) being attached and on classification delete.
> Regression: Classification tagging is not sending appropriate notification to
> ATLAS_ENTITIES
> --------------------------------------------------------------------------------------------
>
> Key: ATLAS-4674
> URL: https://issues.apache.org/jira/browse/ATLAS-4674
> Project: Atlas
> Issue Type: Bug
> Components: atlas-core
> Reporter: Disha Talreja
> Assignee: Disha Talreja
> Priority: Major
>
> When we add/delete a classification from an entity, we send notification to
> ATLAS_ENTITIES so ranger tag sync works appropriately to facilitate Tag-based
> policy. This functionally seems to be broken and is a regression.
> On further investigation, it is seen that the issue is caused by deferred
> action for classification propagation as the right notifications are sent
> when deferred action is disabled.
> Considering the following example:
> 1. Create two table as shown below at Hive:
> =====================================
> create table parent_table(ssn varchar(100));
> create table child_table as select * from parent_table;
> 2. Created two classification: "PII" and "sensitive"
>
> 3. Tagged PII to parent_table and notification received at ATLAS_ENTITIES
> topic:
> -----------------------------------------------------------------------------------------------------------
> {code:java}
> {
> "source": {},
> "version":
> { "version": "1.0.0", "versionParts": [ 1 ] }
> ,
> "msgCompressionKind": "NONE",
> "msgSplitIdx": 1,
> "msgSplitCount": 1,
> "msgSourceIP": "172.27.40.5",
> "msgCreatedBy": "",
> "msgCreationTime": 1662660308744,
> "spooled": false,
> "message": {
> "type": "ENTITY_NOTIFICATION_V2",
> "entity": {
> "typeName": "hive_table",
> "attributes":
> { "owner": "admin", "createTime": 1662602695000,
> "qualifiedName": "default.parent_table@cm", "name": "parent_table"
> }
> ,
> "guid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f",
> "status": "ACTIVE",
> "displayText": "parent_table",
> "classificationNames": [
> "PII" ],
> "classifications": [
>
> { "typeName": "PII", "entityGuid":
> "04ff79d7-b1ff-4354-ba75-a35b28d2f53f", "entityStatus": "ACTIVE",
> "propagate": true, "validityPeriods": [],
> "removePropagationsOnEntityDelete": false }
> ],
> "isIncomplete": false },
> "operationType": "CLASSIFICATION_ADD",
> "eventTime": 1662660307360
> }
> }
> {
> "source": {},
> "version":
> { "version": "1.0.0", "versionParts": [ 1 ] }
> ,
> "msgCompressionKind": "NONE",
> "msgSplitIdx": 1,
> "msgSplitCount": 1,
> "msgSourceIP": "172.27.40.5",
> "msgCreatedBy": "",
> "msgCreationTime": 1662660309419,
> "spooled": false,
> "message": {
> "type": "ENTITY_NOTIFICATION_V2",
> "entity": {
> "typeName": "hive_process",
> "attributes":
> { "qualifiedName": "default.child_table@cm:1662602726000",
> "clusterName": "cm", "name": "default.child_table@cm:1662602726000"
> }
> ,
> "guid": "0e180d04-79ed-4cab-bbc3-76af4b876a5b",
> "status": "ACTIVE",
> "displayText": "default.child_table@cm:1662602726000",
> "classificationNames": [
> "PII" ],
> "classifications": [
>
> { "typeName": "PII", "entityGuid":
> "04ff79d7-b1ff-4354-ba75-a35b28d2f53f", "entityStatus": "ACTIVE",
> "propagate": true, "validityPeriods": [],
> "removePropagationsOnEntityDelete": false }
> ],
> "isIncomplete": false },
> "operationType": "CLASSIFICATION_ADD",
> "eventTime": 1662660308935
> }
> }
> {
> "source": {},
> "version":
> { "version": "1.0.0", "versionParts": [ 1 ] }
> ,
> "msgCompressionKind": "NONE",
> "msgSplitIdx": 1,
> "msgSplitCount": 1,
> "msgSourceIP": "172.27.40.5",
> "msgCreatedBy": "",
> "msgCreationTime": 1662660309419,
> "spooled": false,
> "message": {
> "type": "ENTITY_NOTIFICATION_V2",
> "entity": {
> "typeName": "hive_table",
> "attributes":
> { "owner": "admin", "createTime": 1662602726000,
> "qualifiedName": "default.child_table@cm", "name": "child_table"
> }
> ,
> "guid": "117bc1c5-748c-4cbd-a86f-8369b7167b1f",
> "status": "ACTIVE",
> "displayText": "child_table",
> "classificationNames": [
> "PII" ],
> "classifications": [
>
> { "typeName": "PII", "entityGuid":
> "04ff79d7-b1ff-4354-ba75-a35b28d2f53f", "entityStatus": "ACTIVE",
> "propagate": true, "validityPeriods": [],
> "removePropagationsOnEntityDelete": false }
> ],
> "isIncomplete": false },
> "operationType": "CLASSIFICATION_ADD",
> "eventTime": 1662660308935
> }
> }
> {code}
> 4. Tagged another classification "sensitive" to parent_table and the
> notification at ATLAS_ENTITIES topic:
> -----------------------------------------------------------------------------------------------------
> {code:java}
> {
> "source": {},
> "version":
> { "version": "1.0.0", "versionParts": [ 1 ] }
> ,
> "msgCompressionKind": "NONE",
> "msgSplitIdx": 1,
> "msgSplitCount": 1,
> "msgSourceIP": "172.27.40.5",
> "msgCreatedBy": "",
> "msgCreationTime": 1662660417995,
> "spooled": false,
> "message": {
> "type": "ENTITY_NOTIFICATION_V2",
> "entity": {
> "typeName": "hive_table",
> "attributes":
> { "owner": "admin", "createTime": 1662602695000,
> "qualifiedName": "default.parent_table@cm", "name": "parent_table"
> }
> ,
> "guid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f",
> "status": "ACTIVE",
> "displayText": "parent_table",
> "classificationNames": [
> "PII",
> "sensitive" ],
> "classifications": [
>
> { "typeName": "PII", "entityGuid":
> "04ff79d7-b1ff-4354-ba75-a35b28d2f53f", "entityStatus": "ACTIVE",
> "propagate": true, "validityPeriods": [],
> "removePropagationsOnEntityDelete": false }
> ,
>
> { "typeName": "sensitive", "entityGuid":
> "04ff79d7-b1ff-4354-ba75-a35b28d2f53f", "entityStatus": "ACTIVE",
> "propagate": true, "validityPeriods": [],
> "removePropagationsOnEntityDelete": false }
> ],
> "isIncomplete": false },
> "operationType": "CLASSIFICATION_ADD",
> "eventTime": 1662660417115
> }
> }
> {
> "source": {},
> "version":
> { "version": "1.0.0", "versionParts": [ 1 ] }
> ,
> "msgCompressionKind": "NONE",
> "msgSplitIdx": 1,
> "msgSplitCount": 1,
> "msgSourceIP": "172.27.40.5",
> "msgCreatedBy": "",
> "msgCreationTime": 1662660418085,
> "spooled": false,
> "message": {
> "type": "ENTITY_NOTIFICATION_V2",
> "entity": {
> "typeName": "hive_process",
> "attributes":
> { "qualifiedName": "default.child_table@cm:1662602726000",
> "clusterName": "cm", "name": "default.child_table@cm:1662602726000"
> }
> ,
> "guid": "0e180d04-79ed-4cab-bbc3-76af4b876a5b",
> "status": "ACTIVE",
> "displayText": "default.child_table@cm:1662602726000",
> "classificationNames": [
> "PII" ],
> "classifications": [
>
> { "typeName": "PII", "entityGuid":
> "04ff79d7-b1ff-4354-ba75-a35b28d2f53f", "entityStatus": "ACTIVE",
> "propagate": true, "validityPeriods": [],
> "removePropagationsOnEntityDelete": false }
> ],
> "isIncomplete": false },
> "operationType": "CLASSIFICATION_ADD",
> "eventTime": 1662660308935
> }
> }
> {
> "source": {},
> "version":
> { "version": "1.0.0", "versionParts": [ 1 ] }
> ,
> "msgCompressionKind": "NONE",
> "msgSplitIdx": 1,
> "msgSplitCount": 1,
> "msgSourceIP": "172.27.40.5",
> "msgCreatedBy": "",
> "msgCreationTime": 1662660418085,
> "spooled": false,
> "message": {
> "type": "ENTITY_NOTIFICATION_V2",
> "entity": {
> "typeName": "hive_table",
> "attributes":
> { "owner": "admin", "createTime": 1662602726000,
> "qualifiedName": "default.child_table@cm", "name": "child_table"
> }
> ,
> "guid": "117bc1c5-748c-4cbd-a86f-8369b7167b1f",
> "status": "ACTIVE",
> "displayText": "child_table",
> "classificationNames": [
> "PII" ],
> "classifications": [
>
> { "typeName": "PII", "entityGuid":
> "04ff79d7-b1ff-4354-ba75-a35b28d2f53f", "entityStatus": "ACTIVE",
> "propagate": true, "validityPeriods": [],
> "removePropagationsOnEntityDelete": false }
> ],
> "isIncomplete": false },
> "operationType": "CLASSIFICATION_ADD",
> "eventTime": 1662660308935
> }
> }
> {code}
> 5. Removed "PII" tag from parent_table and the notification:
> ------------------------------------------------------------
> {code:java}
> {
> "source": {},
> "version":
> { "version": "1.0.0", "versionParts": [ 1 ] }
> ,
> "msgCompressionKind": "NONE",
> "msgSplitIdx": 1,
> "msgSplitCount": 1,
> "msgSourceIP": "172.27.40.5",
> "msgCreatedBy": "",
> "msgCreationTime": 1662660456406,
> "spooled": false,
> "message": {
> "type": "ENTITY_NOTIFICATION_V2",
> "entity": {
> "typeName": "hive_table",
> "attributes":
> { "owner": "admin", "createTime": 1662602695000,
> "qualifiedName": "default.parent_table@cm", "name": "parent_table"
> }
> ,
> "guid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f",
> "status": "ACTIVE",
> "displayText": "parent_table",
> "classificationNames": [
> "sensitive" ],
> "classifications": [
>
> { "typeName": "sensitive", "entityGuid":
> "04ff79d7-b1ff-4354-ba75-a35b28d2f53f", "entityStatus": "ACTIVE",
> "propagate": true, "validityPeriods": [],
> "removePropagationsOnEntityDelete": false }
> ],
> "isIncomplete": false },
> "operationType": "CLASSIFICATION_DELETE",
> "eventTime": 1662660456206
> }
> }
> {
> "source": {},
> "version":
> { "version": "1.0.0", "versionParts": [ 1 ] }
> ,
> "msgCompressionKind": "NONE",
> "msgSplitIdx": 1,
> "msgSplitCount": 1,
> "msgSourceIP": "172.27.40.5",
> "msgCreatedBy": "",
> "msgCreationTime": 1662660456586,
> "spooled": false,
> "message": {
> "type": "ENTITY_NOTIFICATION_V2",
> "entity": {
> "typeName": "hive_table",
> "attributes":
> { "owner": "admin", "createTime": 1662602726000,
> "qualifiedName": "default.child_table@cm", "name": "child_table"
> }
> ,
> "guid": "117bc1c5-748c-4cbd-a86f-8369b7167b1f",
> "status": "ACTIVE",
> "displayText": "child_table",
> "classificationNames": [
> "PII" ],
> "classifications": [
>
> { "typeName": "PII", "entityGuid":
> "04ff79d7-b1ff-4354-ba75-a35b28d2f53f", "entityStatus": "ACTIVE",
> "propagate": true, "validityPeriods": [],
> "removePropagationsOnEntityDelete": false }
> ],
> "isIncomplete": false },
> "operationType": "CLASSIFICATION_DELETE",
> "eventTime": 1662660308935
> }
> }
> {
> "source": {},
> "version":
> { "version": "1.0.0", "versionParts": [ 1 ] }
> ,
> "msgCompressionKind": "NONE",
> "msgSplitIdx": 1,
> "msgSplitCount": 1,
> "msgSourceIP": "172.27.40.5",
> "msgCreatedBy": "",
> "msgCreationTime": 1662660456587,
> "spooled": false,
> "message": {
> "type": "ENTITY_NOTIFICATION_V2",
> "entity": {
> "typeName": "hive_process",
> "attributes":
> { "qualifiedName": "default.child_table@cm:1662602726000",
> "clusterName": "cm", "name": "default.child_table@cm:1662602726000"
> }
> ,
> "guid": "0e180d04-79ed-4cab-bbc3-76af4b876a5b",
> "status": "ACTIVE",
> "displayText": "default.child_table@cm:1662602726000",
> "classificationNames": [
> "PII" ],
> "classifications": [
>
> { "typeName": "PII", "entityGuid":
> "04ff79d7-b1ff-4354-ba75-a35b28d2f53f", "entityStatus": "ACTIVE",
> "propagate": true, "validityPeriods": [],
> "removePropagationsOnEntityDelete": false }
> ],
> "isIncomplete": false },
> "operationType": "CLASSIFICATION_DELETE",
> "eventTime": 1662660308935
> }
> }
> {code}
> From the above notification messages, it is clear that the functionality is
> broken as the messages received are incorrect in case of the second tag
> (sensitive) being attached and on classification delete.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
