Radhika Kundam created ATLAS-4482:
-------------------------------------
Summary: GCP : Load data in path query throws Referenced entity
-guid is not found and the GCP directory is not created in Atlas
Key: ATLAS-4482
URL: https://issues.apache.org/jira/browse/ATLAS-4482
Project: Atlas
Issue Type: Bug
Affects Versions: 2.3.0
Reporter: Radhika Kundam
Assignee: Radhika Kundam
Following queries have succeeded:
{code:java}
create external table default.hive_table_cloud_load_data_in_path_qiuvr
(student_roll int, student_name string, student_dob date) ROW FORMAT DELIMITED
FIELDS TERMINATED BY ' ' STORED AS TEXTFILE location
'gs://gcp-eng-qe-func-daily/cc-atlas-fmp5pj-dd/warehouse/tablespace/external/hive/hive_table_cloud_load_data_in_path_qiuvr'
load data inpath
'gs://gcp-eng-qe-func-daily/cc-atlas-fmp5pj-dd/hive_table_cloud_load_data_in_path_qiuvr_input_data.txt'
into table hive_table_cloud_load_data_in_path_qiuvr
{code}
In Atlas , could see the following exception:
{code:java}
2021-11-11 10:38:36,236 ERROR - [NotificationHookConsumer thread-0:] ~ graph
rollback due to exception AtlasBaseException:Referenced entity -9503445734599
is not found (GraphTransactionInterceptor:202)
2021-11-11 10:38:36,236 WARN - [NotificationHookConsumer thread-0:] ~ Max
retries exceeded for message
{"version":{"version":"1.0.0","versionParts":[1]},"msgCompressionKind":"NONE","msgSplitIdx":1,"msgSplitCount":1,"msgCreationTime":1636627116236,"spooled":false,"message":{"type":"ENTITY_CREATE_V2","user":"hrt_qa","entities":{"referredEntities":{"-9503445734600":{"typeName":"gcp_storage_virtual_directory","attributes":{"qualifiedName":"gs://gcp-eng-qe-func-daily/cc-atlas-fmp5pj-dd/@cm","name":"cc-atlas-fmp5pj-dd","objectPrefix":"/"},"guid":"-9503445734600","isIncomplete":false,"provenanceType":0,"version":0,"relationshipAttributes":{"parent":{"guid":"-9503445734599","typeName":"gcp_storage_bucket","uniqueAttributes":{"qualifiedName":"gs://gcp-eng-qe-func-daily@cm"},"relationshipType":"gcp_storage_parent_children"}},"proxy":false},"-9503445734601":{"typeName":"gcp_storage_virtual_directory","attributes":{"qualifiedName":"gs://gcp-eng-qe-func-daily/cc-atlas-fmp5pj-dd/hive_table_cloud_load_data_in_path_qiuvr_input_data.txt/@cm","name":"hive_table_cloud_load_data_in_path_qiuvr_input_data.txt","objectPrefix":"/cc-atlas-fmp5pj-dd/"},"guid":"-9503445734601","isIncomplete":false,"provenanceType":0,"version":0,"relationshipAttributes":{"parent":{"guid":"-9503445734600","typeName":"gcp_storage_virtual_directory","uniqueAttributes":{"qualifiedName":"gs://gcp-eng-qe-func-daily/cc-atlas-fmp5pj-dd/@cm"},"relationshipType":"gcp_storage_parent_children"}},"proxy":false}},"entities":[{"typeName":"hive_process_execution","attributes":{"hostName":"atlas-fmp5pj-dw-master1.atlas-vh.l2ov-m7vs.int.cldr.work","qualifiedName":"LOAD:gs:gcp-eng-qe-func-dailycc-atlas-fmp5pj-ddhive_table_cloud_load_data_in_path_qiuvr_input_data.txt->:default.hive_table_cloud_load_data_in_path_qiuvr@cm:1636627083000:1636627112266:1636627115211","name":"LOAD:gs:gcp-eng-qe-func-dailycc-atlas-fmp5pj-ddhive_table_cloud_load_data_in_path_qiuvr_input_data.txt->:default.hive_table_cloud_load_data_in_path_qiuvr@cm:1636627083000:1636627112266:1636627115211","queryText":"load
data inpath
'gs://gcp-eng-qe-func-daily/cc-atlas-fmp5pj-dd/hive_table_cloud_load_data_in_path_qiuvr_input_data.txt'
into table
hive_table_cloud_load_data_in_path_qiuvr","startTime":1636627112266,"queryPlan":"Not
Supported","endTime":1636627115211,"userName":"hrt_qa","queryId":"hive_20211111103832_2983e653-a768-4062-a1bc-0224f5346765"},"guid":"-9503445734609","isIncomplete":false,"provenanceType":0,"version":0,"relationshipAttributes":{"process":{"guid":"-9503445734608","typeName":"hive_process","relationshipType":"hive_process_process_executions"}},"proxy":false},{"typeName":"hive_process","attributes":{"recentQueries":["load
data inpath
'gs://gcp-eng-qe-func-daily/cc-atlas-fmp5pj-dd/hive_table_cloud_load_data_in_path_qiuvr_input_data.txt'
into table
hive_table_cloud_load_data_in_path_qiuvr"],"qualifiedName":"LOAD:gs:gcp-eng-qe-func-dailycc-atlas-fmp5pj-ddhive_table_cloud_load_data_in_path_qiuvr_input_data.txt->:default.hive_table_cloud_load_data_in_path_qiuvr@cm:1636627083000","clusterName":"cm","name":"LOAD:gs:gcp-eng-qe-func-dailycc-atlas-fmp5pj-ddhive_table_cloud_load_data_in_path_qiuvr_input_data.txt->:default.hive_table_cloud_load_data_in_path_qiuvr@cm:1636627083000","queryText":"","operationType":"LOAD","startTime":1636627115211,"queryPlan":"Not
Supported","endTime":1636627115211,"userName":"","queryId":""},"guid":"-9503445734608","isIncomplete":false,"provenanceType":0,"version":0,"relationshipAttributes":{"outputs":[{"typeName":"hive_table","uniqueAttributes":{"qualifiedName":"default.hive_table_cloud_load_data_in_path_qiuvr@cm"},"relationshipType":"process_dataset_outputs"}],"inputs":[{"guid":"-9503445734601","typeName":"gcp_storage_virtual_directory","uniqueAttributes":{"qualifiedName":"gs://gcp-eng-qe-func-daily/cc-atlas-fmp5pj-dd/hive_table_cloud_load_data_in_path_qiuvr_input_data.txt/@cm"},"relationshipType":"dataset_process_inputs"}]},"proxy":false}]}}}
(NotificationHookConsumer$HookConsumer:808)
org.apache.atlas.exception.AtlasBaseException: Referenced entity -9503445734599
is not found {code}
--
This message was sent by Atlassian Jira
(v8.20.1#820001)
