atlas git commit: ATLAS-2251: Lineage/DataSetLineage (schema fixes #2)

[apoorvnaik]

Repository: atlas
Updated Branches:
  refs/heads/ATLAS-2251 fe4535979 -> f3835b647


ATLAS-2251: Lineage/DataSetLineage (schema fixes #2)


Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/f3835b64
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/f3835b64
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/f3835b64

Branch: refs/heads/ATLAS-2251
Commit: f3835b647d439f7fa224a32a40585e637aa553d8
Parents: fe45359
Author: apoorvnaik 
Authored: Thu Nov 9 16:17:59 2017 -0800
Committer: apoorvnaik 
Committed: Thu Nov 9 16:17:59 2017 -0800

--
 .../java/org/apache/atlas/type/AtlasTypeUtil.java| 10 ++
 .../atlas/v1/model/lineage/SchemaResponse.java   |  9 -
 pom.xml  | 15 ---
 .../apache/atlas/discovery/EntityLineageService.java |  2 --
 4 files changed, 10 insertions(+), 26 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/atlas/blob/f3835b64/intg/src/main/java/org/apache/atlas/type/AtlasTypeUtil.java
--
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasTypeUtil.java 
b/intg/src/main/java/org/apache/atlas/type/AtlasTypeUtil.java
index 5cb74de..80c6a0c 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasTypeUtil.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasTypeUtil.java
@@ -17,6 +17,7 @@
  */
 package org.apache.atlas.type;
 
+import org.apache.atlas.model.instance.AtlasClassification;
 import org.apache.atlas.model.instance.AtlasEntity;
 import org.apache.atlas.model.instance.AtlasEntityHeader;
 import org.apache.atlas.model.instance.AtlasObjectId;
@@ -509,7 +510,7 @@ public class AtlasTypeUtil {
 if (CollectionUtils.isNotEmpty(entity.getClassifications())) {
 Map traitDetails = entity.getClassifications()
   .stream()
-  
.collect(Collectors.toMap(AtlasStruct::getTypeName, c -> 
getNestedTraitDetails(c.getTypeName(;
+  
.collect(Collectors.toMap(AtlasStruct::getTypeName, 
AtlasTypeUtil::getNestedTraitDetails));
 ret.put("$traits$", traitDetails);
 }
 
@@ -534,9 +535,10 @@ public class AtlasTypeUtil {
 return ret;
 }
 
-private static HashMap getNestedTraitDetails(final Object typeName) {
-return new HashMap() {{
-put("$typeName$", typeName);
+private static HashMap getNestedTraitDetails(final AtlasClassification 
atlasClassification) {
+return new HashMap() {{
+put("$typeName$", atlasClassification.getTypeName());
+putAll(atlasClassification.getAttributes());
 }};
 }
 

http://git-wip-us.apache.org/repos/asf/atlas/blob/f3835b64/intg/src/main/java/org/apache/atlas/v1/model/lineage/SchemaResponse.java
--
diff --git 
a/intg/src/main/java/org/apache/atlas/v1/model/lineage/SchemaResponse.java 
b/intg/src/main/java/org/apache/atlas/v1/model/lineage/SchemaResponse.java
index 837668b..56d8226 100644
--- a/intg/src/main/java/org/apache/atlas/v1/model/lineage/SchemaResponse.java
+++ b/intg/src/main/java/org/apache/atlas/v1/model/lineage/SchemaResponse.java
@@ -70,21 +70,12 @@ public class SchemaResponse {
 @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
 @JsonIgnoreProperties(ignoreUnknown = true)
 public static class SchemaDetails {
-private Stringquery;  // The DSL query 
representation for obtaining schema
 private ClassTypeDefinition   dataType;   // Data type of the 
(hive) columns
 private List> rows;   // Column instances for 
the given table
 
 public SchemaDetails() {
 }
 
-public String getQuery() {
-return query;
-}
-
-public void setQuery(final String query) {
-this.query = query;
-}
-
 public ClassTypeDefinition getDataType() {
 return dataType;
 }

http://git-wip-us.apache.org/repos/asf/atlas/blob/f3835b64/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 35b0fec..fdf42a7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -800,7 +800,6 @@
 
 
 
-
 
 com.google.guava
 guava
@@ -1671,6 +1670,10 @@
 org.apache.maven.plugins
 maven-compiler-plugin
 3.7.0
+
+1.8
+

atlas git commit: ATLAS-2251: Lineage/DataSetLineage (schema fixes)

[apoorvnaik]

Repository: atlas
Updated Branches:
  refs/heads/ATLAS-2251 aa85a8ad4 -> fe4535979


ATLAS-2251: Lineage/DataSetLineage (schema fixes)


Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/fe453597
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/fe453597
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/fe453597

Branch: refs/heads/ATLAS-2251
Commit: fe4535979ea83c02fa5c29b038928ee8bc9b457b
Parents: aa85a8a
Author: apoorvnaik 
Authored: Thu Nov 9 14:39:14 2017 -0800
Committer: apoorvnaik 
Committed: Thu Nov 9 14:39:14 2017 -0800

--
 intg/pom.xml|   8 ++
 .../org/apache/atlas/type/AtlasTypeUtil.java| 137 ++-
 .../atlas/v1/model/lineage/LineageResponse.java |   2 +-
 .../atlas/v1/model/lineage/SchemaResponse.java  | 104 ++
 .../v1/model/typedef/AttributeDefinition.java   |  22 ++-
 .../v1/model/typedef/ClassTypeDefinition.java   |   8 +-
 .../atlas/discovery/AtlasLineageService.java|   5 +-
 .../atlas/discovery/EntityLineageService.java   |  91 ++--
 .../web/resources/DataSetLineageResource.java   |  20 +--
 .../atlas/web/resources/LineageResource.java|  16 +--
 10 files changed, 342 insertions(+), 71 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/atlas/blob/fe453597/intg/pom.xml
--
diff --git a/intg/pom.xml b/intg/pom.xml
index 071e6c7..22895df 100644
--- a/intg/pom.xml
+++ b/intg/pom.xml
@@ -86,6 +86,14 @@
 
 
 
+
+org.apache.maven.plugins
+maven-compiler-plugin
+
+1.8
+1.8
+
+
 
 
 

http://git-wip-us.apache.org/repos/asf/atlas/blob/fe453597/intg/src/main/java/org/apache/atlas/type/AtlasTypeUtil.java
--
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasTypeUtil.java 
b/intg/src/main/java/org/apache/atlas/type/AtlasTypeUtil.java
index 0652855..5cb74de 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasTypeUtil.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasTypeUtil.java
@@ -20,13 +20,25 @@ package org.apache.atlas.type;
 import org.apache.atlas.model.instance.AtlasEntity;
 import org.apache.atlas.model.instance.AtlasEntityHeader;
 import org.apache.atlas.model.instance.AtlasObjectId;
-import org.apache.atlas.model.typedef.*;
+import org.apache.atlas.model.instance.AtlasStruct;
+import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
+import org.apache.atlas.model.typedef.AtlasClassificationDef;
+import org.apache.atlas.model.typedef.AtlasEntityDef;
+import org.apache.atlas.model.typedef.AtlasEnumDef;
 import org.apache.atlas.model.typedef.AtlasEnumDef.AtlasEnumElementDef;
+import org.apache.atlas.model.typedef.AtlasRelationshipDef;
 import org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags;
 import 
org.apache.atlas.model.typedef.AtlasRelationshipDef.RelationshipCategory;
+import org.apache.atlas.model.typedef.AtlasRelationshipEndDef;
+import org.apache.atlas.model.typedef.AtlasStructDef;
 import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
 import 
org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality;
 import org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef;
+import org.apache.atlas.model.typedef.AtlasTypeDefHeader;
+import org.apache.atlas.model.typedef.AtlasTypesDef;
+import org.apache.atlas.v1.model.typedef.AttributeDefinition;
+import org.apache.atlas.v1.model.typedef.ClassTypeDefinition;
+import org.apache.atlas.v1.model.typedef.Multiplicity;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.collections.MapUtils;
 import org.apache.commons.lang.StringUtils;
@@ -34,6 +46,7 @@ import org.apache.commons.lang.StringUtils;
 import java.util.*;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 
 import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.*;
 
@@ -405,6 +418,128 @@ public class AtlasTypeUtil {
 return sb.toString();
 }
 
+public static ClassTypeDefinition toClassTypeDefinition(final 
AtlasEntityType entityType) {
+ClassTypeDefinition ret = null;
+
+if (entityType != null) {
+AtlasEntityDef entityDef = entityType.getEntityDef();
+ret = new ClassTypeDefinition();
+ret.setTypeName(entityDef.getName());
+ret.setTypeDescription(entityDef.getDescription());
+ret.setTypeVersion(entityDef.getTypeVersion());
+