[atlas] branch master updated: ATLAS-3661 :- Create 'spark_column_lineage' type and relationship definition to add support of column level lineage (#93)

[nixon]

This is an automated email from the ASF dual-hosted git repository.

nixon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git


The following commit(s) were added to refs/heads/master by this push:
     new 9d6f1f6  ATLAS-3661 :- Create 'spark_column_lineage' type and 
relationship definition to add support of column level lineage (#93)
9d6f1f6 is described below

commit 9d6f1f6baec75a393d2effa7359ae7ac028fa6b5
Author: Vlad Glinsky <61428392+vladhlin...@users.noreply.github.com>
AuthorDate: Mon Mar 16 09:10:12 2020 +0200

    ATLAS-3661 :- Create 'spark_column_lineage' type and relationship 
definition to add support of column level lineage (#93)
---
 addons/models/1000-Hadoop/1100-spark_model.json | 27 +++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/addons/models/1000-Hadoop/1100-spark_model.json 
b/addons/models/1000-Hadoop/1100-spark_model.json
index 28f24d0..ddb9264 100644
--- a/addons/models/1000-Hadoop/1100-spark_model.json
+++ b/addons/models/1000-Hadoop/1100-spark_model.json
@@ -359,6 +359,14 @@
       ]
     },
     {
+      "name" : "spark_column_lineage",
+      "superTypes" : [
+        "Process"
+      ],
+      "serviceType": "spark",
+      "typeVersion" : "1.0"
+    },
+    {
       "name": "spark_ml_pipeline",
       "superTypes": [
         "DataSet"
@@ -470,6 +478,25 @@
         "cardinality": "SINGLE"
       },
       "propagateTags": "NONE"
+    },
+    {
+      "name": "spark_process_column_lineages",
+      "serviceType": "spark",
+      "typeVersion": "1.0",
+      "relationshipCategory": "AGGREGATION",
+      "endDef1": {
+        "type": "spark_column_lineage",
+        "name": "process",
+        "isContainer": false,
+        "cardinality": "SINGLE"
+      },
+      "endDef2": {
+        "type": "spark_process",
+        "name": "columnLineages",
+        "isContainer": true,
+        "cardinality": "SET"
+      },
+      "propagateTags": "NONE"
     }
   ]
 }