This is an automated email from the ASF dual-hosted git repository. nixon pushed a commit to branch branch-2.0 in repository https://gitbox.apache.org/repos/asf/atlas.git
commit a954642025094e35bf8f6de105cee31cda611f5b Author: Vlad Glinsky <[email protected]> AuthorDate: Mon Mar 16 09:10:12 2020 +0200 ATLAS-3661 :- Create 'spark_column_lineage' type and relationship definition to add support of column level lineage (#93) (cherry picked from commit 9d6f1f6baec75a393d2effa7359ae7ac028fa6b5) --- addons/models/1000-Hadoop/1100-spark_model.json | 27 +++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/addons/models/1000-Hadoop/1100-spark_model.json b/addons/models/1000-Hadoop/1100-spark_model.json index 28f24d0..ddb9264 100644 --- a/addons/models/1000-Hadoop/1100-spark_model.json +++ b/addons/models/1000-Hadoop/1100-spark_model.json @@ -359,6 +359,14 @@ ] }, { + "name" : "spark_column_lineage", + "superTypes" : [ + "Process" + ], + "serviceType": "spark", + "typeVersion" : "1.0" + }, + { "name": "spark_ml_pipeline", "superTypes": [ "DataSet" @@ -470,6 +478,25 @@ "cardinality": "SINGLE" }, "propagateTags": "NONE" + }, + { + "name": "spark_process_column_lineages", + "serviceType": "spark", + "typeVersion": "1.0", + "relationshipCategory": "AGGREGATION", + "endDef1": { + "type": "spark_column_lineage", + "name": "process", + "isContainer": false, + "cardinality": "SINGLE" + }, + "endDef2": { + "type": "spark_process", + "name": "columnLineages", + "isContainer": true, + "cardinality": "SET" + }, + "propagateTags": "NONE" } ] }
