This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch branch-1.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/branch-1.0 by this push:
new ea008cb ATLAS-3065: added type-patch to remove legacy attributes
ea008cb is described below
commit ea008cb6d390d67e72be59ae038c3e34354263b8
Author: Madhan Neethiraj <[email protected]>
AuthorDate: Mon Mar 4 13:32:34 2019 -0800
ATLAS-3065: added type-patch to remove legacy attributes
(cherry picked from commit 5c8445627891f67c806d7dbdd26e8f856c87a106)
---
.../main/java/org/apache/atlas/AtlasErrorCode.java | 1 +
.../atlas/model/typedef/AtlasRelationshipDef.java | 44 ++++++++---
.../bootstrap/AtlasTypeDefStoreInitializer.java | 89 ++++++++++++++++++++++
.../graph/v2/AtlasRelationshipDefStoreV2.java | 13 +++-
.../store/graph/v2/AtlasStructDefStoreV2.java | 27 ++++++-
.../main/java/org/apache/atlas/RequestContext.java | 9 +++
6 files changed, 168 insertions(+), 15 deletions(-)
diff --git a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
index e847014..c04f561 100644
--- a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
+++ b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
@@ -154,6 +154,7 @@ public enum AtlasErrorCode {
INVALID_TIMEBOUNDRY_DATERANGE(400, "ATLAS-400-00-87D", "Invalid dateRange:
startTime {0} must be before endTime {1}"),
PROPAGATED_CLASSIFICATION_REMOVAL_NOT_SUPPORTED(400, "ATLAS-400-00-87E",
"Removal of classification {0}, which is propagated from entity {1}, is not
supported"),
IMPORT_ATTEMPTING_EMPTY_ZIP(400, "ATLAS-400-00-87F", "Attempting to import
empty ZIP file."),
+ PATCH_MISSING_RELATIONSHIP_LABEL(400, "ATLAS-400-00-880", "{0} - must
include relationship label for type {1}"),
UNAUTHORIZED_ACCESS(403, "ATLAS-403-00-001", "{0} is not authorized to
perform {1}"),
diff --git
a/intg/src/main/java/org/apache/atlas/model/typedef/AtlasRelationshipDef.java
b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasRelationshipDef.java
index d76f7fa..7153351 100644
---
a/intg/src/main/java/org/apache/atlas/model/typedef/AtlasRelationshipDef.java
+++
b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasRelationshipDef.java
@@ -19,6 +19,7 @@ package org.apache.atlas.model.typedef;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
@@ -116,6 +117,7 @@ public class AtlasRelationshipDef extends AtlasStructDef
implements java.io.Seri
};
private RelationshipCategory relationshipCategory;
+ private String relationshipLabel;
private PropagateTags propagateTags;
private AtlasRelationshipEndDef endDef1;
private AtlasRelationshipEndDef endDef2;
@@ -193,11 +195,24 @@ public class AtlasRelationshipDef extends AtlasStructDef
implements java.io.Seri
super(TypeCategory.RELATIONSHIP, name, description, typeVersion,
attributeDefs, null);
setRelationshipCategory(relationshipCategory);
+ setRelationshipLabel(getDefaultRelationshipLabel());
setPropagateTags(propagatetags);
setEndDef1(endDef1);
setEndDef2(endDef2);
}
+ public AtlasRelationshipDef(AtlasRelationshipDef other) throws
AtlasBaseException {
+ super(other);
+
+ if (other != null) {
+ setRelationshipCategory(other.getRelationshipCategory());
+ setRelationshipLabel(other.getRelationshipLabel());
+ setPropagateTags(other.getPropagateTags());
+ setEndDef1(other.getEndDef1());
+ setEndDef2(other.getEndDef2());
+ }
+ }
+
public void setRelationshipCategory(RelationshipCategory
relationshipCategory) {
this.relationshipCategory = relationshipCategory;
}
@@ -206,6 +221,14 @@ public class AtlasRelationshipDef extends AtlasStructDef
implements java.io.Seri
return this.relationshipCategory;
}
+ public void setRelationshipLabel(String relationshipLabel) {
+ this.relationshipLabel = relationshipLabel;
+ }
+
+ public String getRelationshipLabel() {
+ return relationshipLabel != null ? relationshipLabel : ("r:" +
super.getName());
+ }
+
public void setPropagateTags(PropagateTags propagateTags) {
this.propagateTags=propagateTags;
}
@@ -230,18 +253,13 @@ public class AtlasRelationshipDef extends AtlasStructDef
implements java.io.Seri
return this.endDef2;
}
- public String getRelationshipLabel() { return "r:" + super.getName(); }
-
- public AtlasRelationshipDef(AtlasRelationshipDef other) throws
AtlasBaseException {
- super(other);
+ @JsonIgnore
+ private String getDefaultRelationshipLabel() {
+ String name = super.getName();
- if (other != null) {
- setRelationshipCategory(other.getRelationshipCategory());
- setPropagateTags(other.getPropagateTags());
- setEndDef1(other.getEndDef1());
- setEndDef2(other.getEndDef2());
- }
+ return name != null ? ("r:" + name) : null;
}
+
@Override
public StringBuilder toString(StringBuilder sb) {
if (sb == null) {
@@ -253,6 +271,8 @@ public class AtlasRelationshipDef extends AtlasStructDef
implements java.io.Seri
sb.append(',');
sb.append(this.relationshipCategory);
sb.append(',');
+ sb.append(this.relationshipLabel);
+ sb.append(',');
sb.append(this.propagateTags);
sb.append(',');
if (this.endDef1 != null) {
@@ -284,6 +304,8 @@ public class AtlasRelationshipDef extends AtlasStructDef
implements java.io.Seri
AtlasRelationshipDef that = (AtlasRelationshipDef) o;
if (!Objects.equals(relationshipCategory,
that.getRelationshipCategory()))
return false;
+ if (!Objects.equals(relationshipLabel, that.getRelationshipLabel()))
+ return false;
if (!Objects.equals(propagateTags, that.getPropagateTags()))
return false;
if (!Objects.equals(endDef1, that.getEndDef1()))
@@ -293,7 +315,7 @@ public class AtlasRelationshipDef extends AtlasStructDef
implements java.io.Seri
@Override
public int hashCode() {
- return Objects.hash(super.hashCode(), relationshipCategory,
propagateTags, endDef1, endDef2);
+ return Objects.hash(super.hashCode(), relationshipCategory,
relationshipLabel, propagateTags, endDef1, endDef2);
}
@Override
diff --git
a/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java
b/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java
index 25dd81d..a86282e 100644
---
a/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java
+++
b/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java
@@ -24,6 +24,7 @@ import
com.fasterxml.jackson.databind.annotation.JsonSerialize;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.AtlasException;
+import org.apache.atlas.RequestContext;
import org.apache.atlas.authorize.AtlasAuthorizerFactory;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.ha.HAConfiguration;
@@ -34,10 +35,13 @@ import org.apache.atlas.model.typedef.AtlasEntityDef;
import org.apache.atlas.model.typedef.AtlasEnumDef;
import org.apache.atlas.model.typedef.AtlasEnumDef.AtlasEnumElementDef;
import org.apache.atlas.model.typedef.AtlasRelationshipDef;
+import org.apache.atlas.model.typedef.AtlasRelationshipEndDef;
import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.store.AtlasTypeDefStore;
+import org.apache.atlas.type.AtlasEntityType;
+import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.type.AtlasTypeUtil;
@@ -59,6 +63,7 @@ import java.io.File;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -73,6 +78,7 @@ import static
com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_
public class AtlasTypeDefStoreInitializer implements ActiveStateChangeHandler {
private static final Logger LOG =
LoggerFactory.getLogger(AtlasTypeDefStoreInitializer.class);
public static final String PATCHES_FOLDER_NAME = "patches";
+ public static final String RELATIONSHIP_LABEL = "relationshipLabel";
private final AtlasTypeDefStore atlasTypeDefStore;
private final AtlasTypeRegistry atlasTypeRegistry;
@@ -406,6 +412,7 @@ public class AtlasTypeDefStoreInitializer implements
ActiveStateChangeHandler {
PatchHandler[] patchHandlers = new PatchHandler[] {
new AddAttributePatchHandler(atlasTypeDefStore,
atlasTypeRegistry),
new UpdateAttributePatchHandler(atlasTypeDefStore,
atlasTypeRegistry),
+ new RemoveLegacyAttributesPatchHandler(atlasTypeDefStore,
atlasTypeRegistry),
new UpdateTypeDefOptionsPatchHandler(atlasTypeDefStore,
atlasTypeRegistry),
new SetServiceTypePatchHandler(atlasTypeDefStore,
atlasTypeRegistry)
};
@@ -701,6 +708,88 @@ public class AtlasTypeDefStoreInitializer implements
ActiveStateChangeHandler {
}
}
+ class RemoveLegacyAttributesPatchHandler extends PatchHandler {
+ public RemoveLegacyAttributesPatchHandler(AtlasTypeDefStore
typeDefStore, AtlasTypeRegistry typeRegistry) {
+ super(typeDefStore, typeRegistry, new String[] {
"REMOVE_LEGACY_ATTRIBUTES" });
+ }
+
+ @Override
+ public void applyPatch(TypeDefPatch patch) throws AtlasBaseException {
+ String typeName = patch.getTypeName();
+ AtlasBaseTypeDef typeDef =
typeRegistry.getTypeDefByName(typeName);
+
+ if (typeDef == null) {
+ throw new
AtlasBaseException(AtlasErrorCode.PATCH_FOR_UNKNOWN_TYPE, patch.getAction(),
typeName);
+ }
+
+ if (isPatchApplicable(patch, typeDef)) {
+ if (typeDef.getClass().equals(AtlasRelationshipDef.class)) {
+ AtlasRelationshipDef relationshipDef =
(AtlasRelationshipDef) typeDef;
+ AtlasRelationshipEndDef end1Def =
relationshipDef.getEndDef1();
+ AtlasRelationshipEndDef end2Def =
relationshipDef.getEndDef2();
+ AtlasEntityType end1Type =
typeRegistry.getEntityTypeByName(end1Def.getType());
+ AtlasEntityType end2Type =
typeRegistry.getEntityTypeByName(end2Def.getType());
+
+ String newRelationshipLabel = null;
+
+ if (patch.getParams() != null) {
+ Object val = patch.getParams().get(RELATIONSHIP_LABEL);
+
+ if (val != null) {
+ newRelationshipLabel = val.toString();
+ }
+ }
+
+ if (StringUtils.isEmpty(newRelationshipLabel)) {
+ if (end1Def.getIsLegacyAttribute()) {
+ if (!end2Def.getIsLegacyAttribute()) {
+ AtlasAttribute legacyAttribute =
end1Type.getAttribute(end1Def.getName());
+
+ newRelationshipLabel = "__" +
legacyAttribute.getQualifiedName();
+ } else { // if both ends are legacy attributes,
RELATIONSHIP_LABEL should be specified in the patch
+ throw new
AtlasBaseException(AtlasErrorCode.PATCH_MISSING_RELATIONSHIP_LABEL,
patch.getAction(), typeName);
+ }
+ } else if (end2Def.getIsLegacyAttribute()) {
+ AtlasAttribute legacyAttribute =
end2Type.getAttribute(end2Def.getName());
+
+ newRelationshipLabel = "__" +
legacyAttribute.getQualifiedName();
+ } else {
+ newRelationshipLabel =
relationshipDef.getRelationshipLabel();
+ }
+ }
+
+ AtlasRelationshipDef updatedDef = new
AtlasRelationshipDef(relationshipDef);
+ AtlasEntityDef updatedEntityDef1 = new
AtlasEntityDef(end1Type.getEntityDef());
+ AtlasEntityDef updatedEntityDef2 = new
AtlasEntityDef(end2Type.getEntityDef());
+
+ updatedDef.setRelationshipLabel(newRelationshipLabel);
+ updatedDef.getEndDef1().setIsLegacyAttribute(false);
+ updatedDef.getEndDef2().setIsLegacyAttribute(false);
+ updatedDef.setTypeVersion(patch.getUpdateToVersion());
+
+ updatedEntityDef1.removeAttribute(end1Def.getName());
+ updatedEntityDef2.removeAttribute(end2Def.getName());
+
+ AtlasTypesDef typesDef = new AtlasTypesDef();
+
+ typesDef.setEntityDefs(Arrays.asList(updatedEntityDef1,
updatedEntityDef2));
+
typesDef.setRelationshipDefs(Collections.singletonList(updatedDef));
+
+ try {
+ RequestContext.get().setInTypePatching(true); // to
allow removal of attributes
+
+ typeDefStore.updateTypesDef(typesDef);
+ } finally {
+ RequestContext.get().setInTypePatching(false);
+ }
+ }
+ } else {
+ LOG.info("patch skipped: typeName={}; applyToVersion={};
updateToVersion={}",
+ patch.getTypeName(), patch.getApplyToVersion(),
patch.getUpdateToVersion());
+ }
+ }
+ }
+
class UpdateTypeDefOptionsPatchHandler extends PatchHandler {
public UpdateTypeDefOptionsPatchHandler(AtlasTypeDefStore
typeDefStore, AtlasTypeRegistry typeRegistry) {
super(typeDefStore, typeRegistry, new String[] {
"UPDATE_TYPEDEF_OPTIONS" });
diff --git
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipDefStoreV2.java
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipDefStoreV2.java
index 3602c80..3b79ee1 100644
---
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipDefStoreV2.java
+++
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipDefStoreV2.java
@@ -43,6 +43,7 @@ import javax.inject.Inject;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
+import java.util.Objects;
/**
* RelationshipDef store in v1 format.
@@ -432,7 +433,7 @@ public class AtlasRelationshipDefStoreV2 extends
AtlasAbstractDefStoreV2<AtlasRe
AtlasRelationshipEndDef existingEnd1 =
existingRelationshipDef.getEndDef1();
AtlasRelationshipEndDef newEnd1 = newRelationshipDef.getEndDef1();
- if ( !newEnd1.equals(existingEnd1) ) {
+ if ( !isValidUpdate(existingEnd1, newEnd1) ) {
throw new
AtlasBaseException(AtlasErrorCode.RELATIONSHIPDEF_INVALID_END1_UPDATE,
newRelationshipDef.getName(),
newEnd1.toString(), existingEnd1.toString());
}
@@ -440,7 +441,7 @@ public class AtlasRelationshipDefStoreV2 extends
AtlasAbstractDefStoreV2<AtlasRe
AtlasRelationshipEndDef existingEnd2 =
existingRelationshipDef.getEndDef2();
AtlasRelationshipEndDef newEnd2 = newRelationshipDef.getEndDef2();
- if ( !newEnd2.equals(existingEnd2) ) {
+ if ( !isValidUpdate(existingEnd2, newEnd2) ) {
throw new
AtlasBaseException(AtlasErrorCode.RELATIONSHIPDEF_INVALID_END2_UPDATE,
newRelationshipDef.getName(),
newEnd2.toString(), existingEnd2.toString());
}
@@ -505,4 +506,12 @@ public class AtlasRelationshipDefStoreV2 extends
AtlasAbstractDefStoreV2<AtlasRe
return ret;
}
+ private static boolean isValidUpdate(AtlasRelationshipEndDef currentDef,
AtlasRelationshipEndDef updatedDef) {
+ // permit updates to description and isLegacyAttribute (ref type-patch
REMOVE_LEGACY_ATTRIBUTES)
+ return Objects.equals(currentDef.getType(), updatedDef.getType()) &&
+ Objects.equals(currentDef.getName(), updatedDef.getName()) &&
+ Objects.equals(currentDef.getIsContainer(),
updatedDef.getIsContainer()) &&
+ Objects.equals(currentDef.getCardinality(),
updatedDef.getCardinality());
+ }
+
}
diff --git
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasStructDefStoreV2.java
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasStructDefStoreV2.java
index 1b67f11..d67468e 100644
---
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasStructDefStoreV2.java
+++
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasStructDefStoreV2.java
@@ -19,6 +19,7 @@ package org.apache.atlas.repository.store.graph.v2;
import com.google.common.annotations.VisibleForTesting;
import org.apache.atlas.AtlasErrorCode;
+import org.apache.atlas.RequestContext;
import org.apache.atlas.authorize.AtlasPrivilege;
import org.apache.atlas.authorize.AtlasTypeAccessRequest;
import org.apache.atlas.authorize.AtlasAuthorizationUtils;
@@ -385,12 +386,34 @@ public class AtlasStructDefStoreV2 extends
AtlasAbstractDefStoreV2<AtlasStructDe
// delete attributes that are not present in updated structDef
if (CollectionUtils.isNotEmpty(currAttrNames)) {
+ List<String> removedAttributes = null;
+
for (String currAttrName : currAttrNames) {
if (!attrNames.contains(currAttrName)) {
- throw new
AtlasBaseException(AtlasErrorCode.ATTRIBUTE_DELETION_NOT_SUPPORTED,
- structDef.getName(), currAttrName);
+ if (RequestContext.get().isInTypePatching()) {
+ String propertyKey =
AtlasGraphUtilsV2.getTypeDefPropertyKey(structDef, currAttrName);
+
+ AtlasGraphUtilsV2.setProperty(vertex, propertyKey,
null);
+
+ if (removedAttributes == null) {
+ removedAttributes = new ArrayList<>();
+ }
+
+ removedAttributes.add(currAttrName);
+
+ LOG.warn("REMOVED ATTRIBUTE: {}.{}",
structDef.getName(), currAttrName);
+ } else {
+ throw new
AtlasBaseException(AtlasErrorCode.ATTRIBUTE_DELETION_NOT_SUPPORTED,
+ structDef.getName(), currAttrName);
+ }
}
}
+
+ if (removedAttributes != null) {
+ currAttrNames.removeAll(removedAttributes);
+
+ vertex.setListProperty(encodedStructDefPropertyKey,
currAttrNames);
+ }
}
typeDefStore.updateTypeVertex(structDef, vertex);
diff --git a/server-api/src/main/java/org/apache/atlas/RequestContext.java
b/server-api/src/main/java/org/apache/atlas/RequestContext.java
index 18a1b7b..62899b6 100644
--- a/server-api/src/main/java/org/apache/atlas/RequestContext.java
+++ b/server-api/src/main/java/org/apache/atlas/RequestContext.java
@@ -56,6 +56,7 @@ public class RequestContext {
private int attemptCount = 1;
private boolean isImportInProgress = false;
private boolean isInNotificationProcessing = false;
+ private boolean isInTypePatching = false;
private RequestContext() {
@@ -173,6 +174,14 @@ public class RequestContext {
isInNotificationProcessing = inNotificationProcessing;
}
+ public boolean isInTypePatching() {
+ return isInTypePatching;
+ }
+
+ public void setInTypePatching(boolean inTypePatching) {
+ isInTypePatching = inTypePatching;
+ }
+
public void recordEntityUpdate(AtlasEntityHeader entity) {
if (entity != null && entity.getGuid() != null) {
