[atlas] branch master updated: ATLAS-3035: updated entity-get/delete to retrieve/delete soft-ref entities
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/master by this push:
new 2341d58 ATLAS-3035: updated entity-get/delete to retrieve/delete
soft-ref entities
2341d58 is described below
commit 2341d58a6ef14de410b75abdbc1378c6f81db6fa
Author: Madhan Neethiraj
AuthorDate: Tue Jan 22 19:39:37 2019 -0800
ATLAS-3035: updated entity-get/delete to retrieve/delete soft-ref entities
---
.../org/apache/atlas/utils/AtlasEntityUtil.java| 97 +-
.../repository/store/graph/v1/DeleteHandlerV1.java | 63 ++--
.../store/graph/v2/EntityGraphMapper.java | 26 ++---
.../store/graph/v2/EntityGraphRetriever.java | 111 +++--
4 files changed, 214 insertions(+), 83 deletions(-)
diff --git a/intg/src/main/java/org/apache/atlas/utils/AtlasEntityUtil.java
b/intg/src/main/java/org/apache/atlas/utils/AtlasEntityUtil.java
index 27409c7..316496d 100644
--- a/intg/src/main/java/org/apache/atlas/utils/AtlasEntityUtil.java
+++ b/intg/src/main/java/org/apache/atlas/utils/AtlasEntityUtil.java
@@ -19,24 +19,30 @@ package org.apache.atlas.utils;
import org.apache.atlas.model.instance.AtlasEntity;
-import org.apache.atlas.type.AtlasArrayType;
+import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.type.AtlasEntityType;
-import org.apache.atlas.type.AtlasMapType;
-import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
import org.apache.atlas.type.AtlasType;
+import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
+import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.Collection;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
import java.util.Map;
-import java.util.Objects;
public class AtlasEntityUtil {
private static final Logger LOG =
LoggerFactory.getLogger(AtlasEntityUtil.class);
+private static final String SOFT_REFERENCE_FORMAT_SEPERATOR = ":";
+private static final String SOFT_REF_FORMAT = "%s" +
SOFT_REFERENCE_FORMAT_SEPERATOR + "%s";
+private static final intSOFT_REFERENCE_FORMAT_INDEX_TYPE_NAME = 0;
+private static final intSOFT_REFERENCE_FORMAT_INDEX_GUID = 1;
+
public static boolean hasAnyAttributeUpdate(AtlasEntityType entityType,
AtlasEntity currEntity, AtlasEntity newEntity) {
if (LOG.isDebugEnabled()) {
LOG.debug("==> hasAnyAttributeUpdate(guid={}, typeName={})",
currEntity.getGuid(), currEntity.getTypeName());
@@ -68,4 +74,85 @@ public class AtlasEntityUtil {
return ret;
}
+
+
+public static String formatSoftRefValue(String typeName, String guid) {
+return String.format(SOFT_REF_FORMAT, typeName, guid);
+}
+
+public static String formatSoftRefValue(AtlasObjectId objectId) {
+return formatSoftRefValue(objectId.getTypeName(), objectId.getGuid());
+}
+
+public static List formatSoftRefValue(List objIds) {
+List ret = new ArrayList<>();
+
+for (AtlasObjectId objId : objIds) {
+ret.add(formatSoftRefValue(objId));
+}
+
+return ret;
+}
+
+public static Map formatSoftRefValue(Map objIdMap) {
+Map ret = new HashMap<>();
+
+for (Map.Entry entry : objIdMap.entrySet()) {
+ret.put(entry.getKey(), formatSoftRefValue(entry.getValue()));
+}
+
+return ret;
+}
+
+public static AtlasObjectId parseSoftRefValue(String softRefValue) {
+AtlasObjectId ret = null;
+
+if (StringUtils.isNotEmpty(softRefValue)) {
+String[] objectIdParts = StringUtils.split(softRefValue,
SOFT_REFERENCE_FORMAT_SEPERATOR);
+
+if(objectIdParts.length >= 2) {
+ret = new
AtlasObjectId(objectIdParts[SOFT_REFERENCE_FORMAT_INDEX_GUID],
objectIdParts[SOFT_REFERENCE_FORMAT_INDEX_TYPE_NAME]);
+} else {
+LOG.warn("Invalid soft-ref value: {}", softRefValue);
+}
+}
+
+return ret;
+}
+
+public static List parseSoftRefValue(List
softRefValue) {
+List ret = null;
+
+if (CollectionUtils.isNotEmpty(softRefValue)) {
+ret = new ArrayList<>();
+
+for (String elemValue : softRefValue) {
+AtlasObjectId objId = parseSoftRefValue(elemValue);
+
+if (objId != null) {
+ret.add(objId);
+}
+}
+}
+
+return ret;
+}
+
+public static Map parseSoftRefValue(Map softRefValue) {
+Map ret = null;
+
+if (MapUtils.isNotEmpty(softRefValue)) {
+ret = new HashMap<>();
+
+for
[atlas] branch master updated: ATLAS-3015: Classification Updater tool.
This is an automated email from the ASF dual-hosted git repository.
amestry pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/master by this push:
new 4985b9f ATLAS-3015: Classification Updater tool.
4985b9f is described below
commit 4985b9f2a64a0f16f323bcbac600457215636852
Author: Ashutosh Mestry
AuthorDate: Wed Jan 23 11:13:34 2019 -0800
ATLAS-3015: Classification Updater tool.
---
tools/classification-updater/pom.xml | 114 +
.../org/apache/atlas/tools/BulkFetchAndUpdate.java | 510 +
.../src/main/resources/atlas-log4j.xml | 42 ++
.../src/main/resources/update-classifications.sh | 90
4 files changed, 756 insertions(+)
diff --git a/tools/classification-updater/pom.xml
b/tools/classification-updater/pom.xml
new file mode 100644
index 000..4e23537
--- /dev/null
+++ b/tools/classification-updater/pom.xml
@@ -0,0 +1,114 @@
+
+
+
+http://maven.apache.org/POM/4.0.0;
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
+4.0.0
+
+apache-atlas
+org.apache.atlas
+2.0.0-SNAPSHOT
+../../
+
+atlas-classification-updater
+Apache Atlas classification updater Module
+Apache Atlas classification updater
+jar
+
+
+0.9.2-incubating
+
+
+
+
+
+org.slf4j
+slf4j-api
+
+
+
+org.slf4j
+slf4j-log4j12
+
+
+
+org.apache.atlas
+atlas-client-v1
+
+
+
+org.apache.atlas
+atlas-client-v2
+${project.version}
+
+
+
+org.apache.atlas
+atlas-notification
+
+
+
+org.apache.atlas
+hdfs-model
+
+
+
+
+
+com.fasterxml.jackson.core
+jackson-databind
+${jackson.version}
+
+
+
+org.testng
+testng
+
+
+
+org.mockito
+mockito-all
+
+
+
+
+
+
+
+org.apache.maven.plugins
+maven-jar-plugin
+2.4
+
+
+
+test-jar
+
+
+
+
+
+org.apache.maven.plugins
+maven-compiler-plugin
+
+1.8
+1.8
+
+
+
+
+
diff --git
a/tools/classification-updater/src/main/java/org/apache/atlas/tools/BulkFetchAndUpdate.java
b/tools/classification-updater/src/main/java/org/apache/atlas/tools/BulkFetchAndUpdate.java
new file mode 100644
index 000..1e0b66d
--- /dev/null
+++
b/tools/classification-updater/src/main/java/org/apache/atlas/tools/BulkFetchAndUpdate.java
@@ -0,0 +1,510 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.tools;
+
+import com.sun.jersey.core.util.MultivaluedMapImpl;
+import org.apache.atlas.ApplicationProperties;
+import org.apache.atlas.AtlasClientV2;
+import org.apache.atlas.AtlasException;
+import org.apache.atlas.AtlasServiceException;
+import org.apache.atlas.model.SearchFilter;
+import org.apache.atlas.model.instance.AtlasClassification;
+import org.apache.atlas.model.instance.AtlasEntityHeader;
+import org.apache.atlas.model.instance.AtlasEntityHeaders;
+import org.apache.atlas.model.typedef.AtlasClassificationDef;
+import org.apache.atlas.model.typedef.AtlasTypesDef;
+import org.apache.atlas.type.AtlasType;
+import org.apache.atlas.utils.AtlasJson;
+import org.apache.atlas.utils.AuthenticationUtil;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.DefaultParser;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.configuration.Configuration;
+import
[atlas] branch branch-1.0 updated: ATLAS-3034: Perf enhancement to avoid unnecessary lookup when creating new relationships
This is an automated email from the ASF dual-hosted git repository.
sarath pushed a commit to branch branch-1.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/branch-1.0 by this push:
new 7c471dc ATLAS-3034: Perf enhancement to avoid unnecessary lookup when
creating new relationships
7c471dc is described below
commit 7c471dc313daf7c9e30e77024ac3e2c94a9a5403
Author: Sarath Subramanian
AuthorDate: Wed Jan 23 10:35:35 2019 -0800
ATLAS-3034: Perf enhancement to avoid unnecessary lookup when creating new
relationships
(cherry picked from commit a9aa5b0e1b0a7c2867aef16358e3c3c1977bb140)
---
.../store/graph/AtlasRelationshipStore.java| 4 +
.../store/graph/v2/AtlasRelationshipStoreV2.java | 124 +++--
.../store/graph/v2/EntityGraphMapper.java | 32 ++
3 files changed, 77 insertions(+), 83 deletions(-)
diff --git
a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasRelationshipStore.java
b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasRelationshipStore.java
index 4468951..53e29e1 100644
---
a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasRelationshipStore.java
+++
b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasRelationshipStore.java
@@ -59,6 +59,10 @@ public interface AtlasRelationshipStore {
AtlasEdge getOrCreate(AtlasVertex end1Vertex, AtlasVertex end2Vertex,
AtlasRelationship relationship) throws AtlasBaseException;
+AtlasEdge getRelationship(AtlasVertex end1Vertex, AtlasVertex end2Vertex,
AtlasRelationship relationship);
+
+AtlasEdge createRelationship(AtlasVertex end1Vertex, AtlasVertex
end2Vertex, AtlasRelationship relationship) throws AtlasBaseException;
+
/**
* Retrieve a relationship if it exists or creates a new relationship
instance.
* @param relationship relationship instance definition
diff --git
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java
index 6bc36b3..2791ef3 100644
---
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java
+++
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java
@@ -81,9 +81,8 @@ import static
org.apache.atlas.repository.graph.GraphHelper.getBlockedClassifica
import static
org.apache.atlas.repository.graph.GraphHelper.getClassificationEntityGuid;
import static
org.apache.atlas.repository.graph.GraphHelper.getClassificationName;
import static
org.apache.atlas.repository.graph.GraphHelper.getClassificationVertices;
-import static
org.apache.atlas.repository.graph.GraphHelper.getOutGoingEdgesByLabel;
+import static
org.apache.atlas.repository.graph.GraphHelper.getIncomingEdgesByLabel;
import static org.apache.atlas.repository.graph.GraphHelper.getPropagateTags;
-import static
org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.getIdFromVertex;
import static
org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.getState;
import static
org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.getTypeName;
@@ -117,8 +116,6 @@ public class AtlasRelationshipStoreV2 implements
AtlasRelationshipStore {
AtlasVertex end1Vertex = getVertexFromEndPoint(relationship.getEnd1());
AtlasVertex end2Vertex = getVertexFromEndPoint(relationship.getEnd2());
-validateRelationship(end1Vertex, end2Vertex,
relationship.getTypeName(), relationship.getAttributes());
-
AtlasEdge edge = createRelationship(end1Vertex, end2Vertex,
relationship);
AtlasRelationship ret = edge != null ?
entityRetriever.mapEdgeToAtlasRelationship(edge) : null;
@@ -289,36 +286,37 @@ public class AtlasRelationshipStoreV2 implements
AtlasRelationshipStore {
@Override
public AtlasEdge getOrCreate(AtlasVertex end1Vertex, AtlasVertex
end2Vertex, AtlasRelationship relationship) throws AtlasBaseException {
-AtlasEdge ret = getRelationshipEdge(end1Vertex, end2Vertex,
relationship.getTypeName());
+AtlasEdge ret = getRelationship(end1Vertex, end2Vertex, relationship);
if (ret == null) {
-validateRelationship(end1Vertex, end2Vertex,
relationship.getTypeName(), relationship.getAttributes());
-
-ret = createRelationship(end1Vertex, end2Vertex, relationship);
+ret = createRelationship(end1Vertex, end2Vertex, relationship,
false);
}
return ret;
}
@Override
+public AtlasEdge getRelationship(AtlasVertex fromVertex, AtlasVertex
toVertex, AtlasRelationship relationship) {
+String relationshipLabel = getRelationshipEdgeLabel(fromVertex,
toVertex, relationship.getTypeName());
+
+return getRelationshipEdge(fromVertex, toVertex,
[atlas] branch master updated: ATLAS-3034: Perf enhancement to avoid unnecessary lookup when creating new relationships
This is an automated email from the ASF dual-hosted git repository.
sarath pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/master by this push:
new a9aa5b0 ATLAS-3034: Perf enhancement to avoid unnecessary lookup when
creating new relationships
a9aa5b0 is described below
commit a9aa5b0e1b0a7c2867aef16358e3c3c1977bb140
Author: Sarath Subramanian
AuthorDate: Wed Jan 23 10:35:35 2019 -0800
ATLAS-3034: Perf enhancement to avoid unnecessary lookup when creating new
relationships
---
.../store/graph/AtlasRelationshipStore.java| 4 +
.../store/graph/v2/AtlasRelationshipStoreV2.java | 126 +++--
.../store/graph/v2/EntityGraphMapper.java | 32 ++
3 files changed, 78 insertions(+), 84 deletions(-)
diff --git
a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasRelationshipStore.java
b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasRelationshipStore.java
index 4468951..53e29e1 100644
---
a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasRelationshipStore.java
+++
b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasRelationshipStore.java
@@ -59,6 +59,10 @@ public interface AtlasRelationshipStore {
AtlasEdge getOrCreate(AtlasVertex end1Vertex, AtlasVertex end2Vertex,
AtlasRelationship relationship) throws AtlasBaseException;
+AtlasEdge getRelationship(AtlasVertex end1Vertex, AtlasVertex end2Vertex,
AtlasRelationship relationship);
+
+AtlasEdge createRelationship(AtlasVertex end1Vertex, AtlasVertex
end2Vertex, AtlasRelationship relationship) throws AtlasBaseException;
+
/**
* Retrieve a relationship if it exists or creates a new relationship
instance.
* @param relationship relationship instance definition
diff --git
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java
index 86cc98c..6371f56 100644
---
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java
+++
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java
@@ -82,9 +82,8 @@ import static
org.apache.atlas.repository.graph.GraphHelper.getBlockedClassifica
import static
org.apache.atlas.repository.graph.GraphHelper.getClassificationEntityGuid;
import static
org.apache.atlas.repository.graph.GraphHelper.getClassificationName;
import static
org.apache.atlas.repository.graph.GraphHelper.getClassificationVertices;
-import static
org.apache.atlas.repository.graph.GraphHelper.getOutGoingEdgesByLabel;
+import static
org.apache.atlas.repository.graph.GraphHelper.getIncomingEdgesByLabel;
import static org.apache.atlas.repository.graph.GraphHelper.getPropagateTags;
-import static
org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.getIdFromVertex;
import static
org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.getState;
import static
org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.getTypeName;
@@ -118,8 +117,6 @@ public class AtlasRelationshipStoreV2 implements
AtlasRelationshipStore {
AtlasVertex end1Vertex = getVertexFromEndPoint(relationship.getEnd1());
AtlasVertex end2Vertex = getVertexFromEndPoint(relationship.getEnd2());
-validateRelationship(end1Vertex, end2Vertex,
relationship.getTypeName(), relationship.getAttributes());
-
AtlasEdge edge = createRelationship(end1Vertex, end2Vertex,
relationship);
AtlasRelationship ret = edge != null ?
entityRetriever.mapEdgeToAtlasRelationship(edge) : null;
@@ -290,36 +287,37 @@ public class AtlasRelationshipStoreV2 implements
AtlasRelationshipStore {
@Override
public AtlasEdge getOrCreate(AtlasVertex end1Vertex, AtlasVertex
end2Vertex, AtlasRelationship relationship) throws AtlasBaseException {
-AtlasEdge ret = getRelationshipEdge(end1Vertex, end2Vertex,
relationship.getTypeName());
+AtlasEdge ret = getRelationship(end1Vertex, end2Vertex, relationship);
if (ret == null) {
-validateRelationship(end1Vertex, end2Vertex,
relationship.getTypeName(), relationship.getAttributes());
-
-ret = createRelationship(end1Vertex, end2Vertex, relationship);
+ret = createRelationship(end1Vertex, end2Vertex, relationship,
false);
}
return ret;
}
@Override
+public AtlasEdge getRelationship(AtlasVertex fromVertex, AtlasVertex
toVertex, AtlasRelationship relationship) {
+String relationshipLabel = getRelationshipEdgeLabel(fromVertex,
toVertex, relationship.getTypeName());
+
+return getRelationshipEdge(fromVertex, toVertex, relationshipLabel);
+}
+
+@Override
public AtlasRelationship
