Repository: atlas Updated Branches: refs/heads/master a3374c747 -> 48b2eaa99
ATLAS-2457: updates to support time-boundary in AtlasClassification associations Project: http://git-wip-us.apache.org/repos/asf/atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/48b2eaa9 Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/48b2eaa9 Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/48b2eaa9 Branch: refs/heads/master Commit: 48b2eaa99f91ec615f67fcde3183b262c765d698 Parents: a3374c7 Author: Madhan Neethiraj <mad...@apache.org> Authored: Wed Feb 21 13:53:44 2018 -0800 Committer: Madhan Neethiraj <mad...@apache.org> Committed: Wed Feb 21 18:04:59 2018 -0800 ---------------------------------------------------------------------- .../org/apache/atlas/repository/Constants.java | 7 +- .../org/apache/atlas/model/TimeBoundary.java | 120 +++++++++++++++++++ .../model/instance/AtlasClassification.java | 19 ++- .../store/graph/v1/EntityGraphMapper.java | 8 +- .../store/graph/v1/EntityGraphRetriever.java | 13 +- .../integration/EntityV2JerseyResourceIT.java | 45 ++++++- 6 files changed, 202 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/atlas/blob/48b2eaa9/common/src/main/java/org/apache/atlas/repository/Constants.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java index ae52880..b6c639a 100644 --- a/common/src/main/java/org/apache/atlas/repository/Constants.java +++ b/common/src/main/java/org/apache/atlas/repository/Constants.java @@ -116,8 +116,11 @@ public final class Constants { public static final String ATTRIBUTE_NAME_VERSION = "version"; public static final String TEMP_STRUCT_NAME_PREFIX = "__tempQueryResultStruct"; - public static final String CLASSIFICATION_ENTITY_GUID = INTERNAL_PROPERTY_KEY_PREFIX + "entityGuid"; - public static final String CLASSIFICATION_PROPAGATE_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "propagate"; + public static final String CLASSIFICATION_ENTITY_GUID = INTERNAL_PROPERTY_KEY_PREFIX + "entityGuid"; + public static final String CLASSIFICATION_PROPAGATE_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "propagate"; + public static final String CLASSIFICATION_VALIDITY_PERIOD_STARTTIME_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "vp_startTime"; + public static final String CLASSIFICATION_VALIDITY_PERIOD_ENDTIME_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "vp_endTime"; + public static final String CLASSIFICATION_VALIDITY_PERIOD_TIMEZONE_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "vp_timeZone"; private Constants() { } http://git-wip-us.apache.org/repos/asf/atlas/blob/48b2eaa9/intg/src/main/java/org/apache/atlas/model/TimeBoundary.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/model/TimeBoundary.java b/intg/src/main/java/org/apache/atlas/model/TimeBoundary.java new file mode 100644 index 0000000..882a33c --- /dev/null +++ b/intg/src/main/java/org/apache/atlas/model/TimeBoundary.java @@ -0,0 +1,120 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.model; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +import java.io.Serializable; +import java.util.Objects; + +import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE; +import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY; + +/** + * Captures time-boundary details + */ +@JsonAutoDetect(getterVisibility=PUBLIC_ONLY, setterVisibility=PUBLIC_ONLY, fieldVisibility=NONE) +@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown=true) +@XmlRootElement +@XmlAccessorType(XmlAccessType.PROPERTY) +public class TimeBoundary implements Serializable { + private static final long serialVersionUID = 1L; + + public static final String TIME_FORMAT = "yyyy/MM/dd HH:mm:ss"; + + private String startTime; + private String endTime; + private String timeZone; // null for local-time; or a valid ID for TimeZone.getTimeZone(id) + + public TimeBoundary() { + this(null, null, null); + } + + public TimeBoundary(String startTime) { + this(startTime, null, null); + } + + public TimeBoundary(String startTime, String endTime) { + this(startTime, endTime, null); + } + + public TimeBoundary(String startTime, String endTime, String timeZone) { + this.startTime = startTime; + this.endTime = endTime; + this.timeZone = timeZone; + } + + public String getStartTime() { + return startTime; + } + + public void setStartTime(String startTime) { + this.startTime = startTime; + } + + public String getEndTime() { + return endTime; + } + + public void setEndTime(String endTime) { + this.endTime = endTime; + } + + public String getTimeZone() { + return timeZone; + } + + public void setTimeZone(String timeZone) { + this.timeZone = timeZone; + } + + + @Override + public boolean equals(Object o) { + if (this == o) { return true; } + if (o == null || getClass() != o.getClass()) { return false; } + TimeBoundary that = (TimeBoundary) o; + return Objects.equals(startTime, that.startTime) && + Objects.equals(endTime, that.endTime) && + Objects.equals(timeZone, that.timeZone); + } + + @Override + public int hashCode() { + return Objects.hash(startTime, endTime, timeZone); + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("TimeBoundary{"); + + sb.append("startTime='").append(startTime) + .append("; endTime='").append(endTime) + .append("; timeZone='").append(timeZone) + .append('}'); + + return sb.toString(); + } +} http://git-wip-us.apache.org/repos/asf/atlas/blob/48b2eaa9/intg/src/main/java/org/apache/atlas/model/instance/AtlasClassification.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/model/instance/AtlasClassification.java b/intg/src/main/java/org/apache/atlas/model/instance/AtlasClassification.java index a499f79..3d2ecef 100644 --- a/intg/src/main/java/org/apache/atlas/model/instance/AtlasClassification.java +++ b/intg/src/main/java/org/apache/atlas/model/instance/AtlasClassification.java @@ -34,6 +34,7 @@ import javax.xml.bind.annotation.XmlSeeAlso; import org.apache.atlas.model.PList; import org.apache.atlas.model.SearchFilter.SortType; +import org.apache.atlas.model.TimeBoundary; import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE; import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY; @@ -50,8 +51,10 @@ import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ public class AtlasClassification extends AtlasStruct implements Serializable { private static final long serialVersionUID = 1L; - private String entityGuid = null; - private boolean propagate = true; + private String entityGuid = null; + private boolean propagate = true; + private TimeBoundary validityPeriod = null; + public AtlasClassification() { this(null, null); @@ -96,6 +99,14 @@ public class AtlasClassification extends AtlasStruct implements Serializable { this.propagate = propagate; } + public TimeBoundary getValidityPeriod() { + return validityPeriod; + } + + public void setValidityPeriod(TimeBoundary validityPeriod) { + this.validityPeriod = validityPeriod; + } + @Override public boolean equals(Object o) { if (this == o) { return true; } @@ -103,7 +114,8 @@ public class AtlasClassification extends AtlasStruct implements Serializable { if (!super.equals(o)) { return false; } AtlasClassification that = (AtlasClassification) o; return propagate == that.propagate && - Objects.equals(entityGuid, that.entityGuid); + Objects.equals(entityGuid, that.entityGuid) && + Objects.equals(validityPeriod, that.validityPeriod); } @Override @@ -117,6 +129,7 @@ public class AtlasClassification extends AtlasStruct implements Serializable { super.toString(sb); sb.append("entityGuid='").append(entityGuid).append('\''); sb.append(", propagate=").append(propagate); + sb.append(", validityPeriod=").append(validityPeriod); sb.append('}'); return sb.toString(); } http://git-wip-us.apache.org/repos/asf/atlas/blob/48b2eaa9/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java index 0224bf0..80c64b1 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java @@ -22,6 +22,7 @@ import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.AtlasException; import org.apache.atlas.RequestContextV1; import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.TimeBoundary; import org.apache.atlas.model.TypeCategory; import org.apache.atlas.model.instance.AtlasClassification; import org.apache.atlas.model.instance.AtlasEntity; @@ -53,7 +54,6 @@ import org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdg import org.apache.atlas.type.AtlasType; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.type.AtlasTypeUtil; -import org.apache.atlas.util.AtlasGremlinQueryProvider; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.IteratorUtils; import org.apache.commons.collections.MapUtils; @@ -1592,6 +1592,12 @@ public class EntityGraphMapper { AtlasEntityType entityType, AtlasVertex parentInstanceVertex, AtlasVertex traitInstanceVertex) throws AtlasBaseException { + TimeBoundary validityPeriod = classification.getValidityPeriod(); + + AtlasGraphUtilsV1.setProperty(traitInstanceVertex, Constants.CLASSIFICATION_VALIDITY_PERIOD_STARTTIME_KEY, validityPeriod != null ? validityPeriod.getStartTime() : null); + AtlasGraphUtilsV1.setProperty(traitInstanceVertex, Constants.CLASSIFICATION_VALIDITY_PERIOD_ENDTIME_KEY, validityPeriod != null ? validityPeriod.getEndTime() : null); + AtlasGraphUtilsV1.setProperty(traitInstanceVertex, Constants.CLASSIFICATION_VALIDITY_PERIOD_TIMEZONE_KEY, validityPeriod != null ? validityPeriod.getTimeZone() : null); + // map all the attributes to this newly created AtlasVertex mapAttributes(classification, traitInstanceVertex, operation, context); http://git-wip-us.apache.org/repos/asf/atlas/blob/48b2eaa9/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java index d01fb9f..45c7d39 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java @@ -19,6 +19,7 @@ package org.apache.atlas.repository.store.graph.v1; import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.TimeBoundary; import org.apache.atlas.model.instance.AtlasClassification; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo; @@ -68,9 +69,7 @@ import java.util.Map; import java.util.Set; import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.*; -import static org.apache.atlas.repository.Constants.CLASSIFICATION_ENTITY_GUID; -import static org.apache.atlas.repository.Constants.CLASSIFICATION_PROPAGATE_KEY; -import static org.apache.atlas.repository.Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY; +import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.repository.graph.GraphHelper.EDGE_LABEL_PREFIX; import static org.apache.atlas.repository.graph.GraphHelper.edgeExists; import static org.apache.atlas.repository.graph.GraphHelper.getAdjacentEdgesByLabel; @@ -219,6 +218,14 @@ public final class EntityGraphRetriever { ret.setEntityGuid(AtlasGraphUtilsV1.getProperty(classificationVertex, CLASSIFICATION_ENTITY_GUID, String.class)); ret.setPropagate(AtlasGraphUtilsV1.getProperty(classificationVertex, CLASSIFICATION_PROPAGATE_KEY, Boolean.class)); + String vpStartTime = AtlasGraphUtilsV1.getProperty(classificationVertex, CLASSIFICATION_VALIDITY_PERIOD_STARTTIME_KEY, String.class); + String vpEndTime = AtlasGraphUtilsV1.getProperty(classificationVertex, CLASSIFICATION_VALIDITY_PERIOD_ENDTIME_KEY, String.class); + String vpTimeZone = AtlasGraphUtilsV1.getProperty(classificationVertex, CLASSIFICATION_VALIDITY_PERIOD_TIMEZONE_KEY, String.class); + + if (vpStartTime != null || vpEndTime != null || vpTimeZone != null) { + ret.setValidityPeriod(new TimeBoundary(vpStartTime, vpEndTime, vpTimeZone)); + } + mapAttributes(classificationVertex, ret, null); return ret; http://git-wip-us.apache.org/repos/asf/atlas/blob/48b2eaa9/webapp/src/test/java/org/apache/atlas/web/integration/EntityV2JerseyResourceIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/web/integration/EntityV2JerseyResourceIT.java b/webapp/src/test/java/org/apache/atlas/web/integration/EntityV2JerseyResourceIT.java index dabb2ef..1d666a6 100755 --- a/webapp/src/test/java/org/apache/atlas/web/integration/EntityV2JerseyResourceIT.java +++ b/webapp/src/test/java/org/apache/atlas/web/integration/EntityV2JerseyResourceIT.java @@ -24,6 +24,7 @@ import com.sun.jersey.api.client.ClientResponse; import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasServiceException; import org.apache.atlas.EntityAuditEvent; +import org.apache.atlas.model.TimeBoundary; import org.apache.atlas.model.instance.AtlasClassification; import org.apache.atlas.model.instance.AtlasClassification.AtlasClassifications; import org.apache.atlas.model.instance.AtlasEntity; @@ -38,7 +39,7 @@ import org.apache.atlas.model.typedef.AtlasTypesDef; import org.apache.atlas.type.AtlasTypeUtil; import org.apache.atlas.v1.typesystem.types.utils.TypesUtil; import org.apache.commons.lang.RandomStringUtils; -import org.joda.time.DateTime; +import org.apache.hadoop.util.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.Assert; @@ -448,6 +449,48 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT { assertEntityAudit(createHiveTable().getGuid(), EntityAuditEvent.EntityAuditAction.TAG_ADD); } + @Test(dependsOnMethods = "testGetTraitNames") + public void testAddTraitWithValidityPeriod() throws Exception { + traitName = "PII_Trait" + randomString(); + + AtlasClassificationDef piiTrait = AtlasTypeUtil.createTraitTypeDef(traitName, Collections.<String>emptySet()); + AtlasTypesDef typesDef = new AtlasTypesDef(Collections.emptyList(), Collections.emptyList(), Collections.singletonList(piiTrait), Collections.emptyList()); + + createType(typesDef); + + String tableGuid = createHiveTable().getGuid(); + AtlasClassification classification = new AtlasClassification(piiTrait.getName()); + TimeBoundary validityPeriod = new TimeBoundary("2018/03/01 00:00:00", "2018/04/01 00:00:00", "GMT"); + + classification.setEntityGuid(tableGuid); + classification.setValidityPeriod(validityPeriod); + + atlasClientV2.addClassifications(tableGuid, Collections.singletonList(classification)); + + assertEntityAudit(tableGuid, EntityAuditEvent.EntityAuditAction.TAG_ADD); + + AtlasClassifications classifications = atlasClientV2.getClassifications(tableGuid); + + assertNotNull(classifications); + assertNotNull(classifications.getList()); + assertTrue(classifications.getList().size() > 1); + + boolean foundClassification = false; + for (AtlasClassification entityClassification : classifications.getList()) { + if (StringUtils.equalsIgnoreCase(entityClassification.getTypeName(), piiTrait.getName())) { + foundClassification = true; + + assertEquals(entityClassification.getTypeName(), piiTrait.getName()); + assertEquals(entityClassification.getValidityPeriod(), validityPeriod); + assertEquals(entityClassification, classification); + + break; + } + } + + assertTrue(foundClassification, "classification '" + piiTrait.getName() + "' is missing for entity '" + tableGuid + "'"); + } + @Test(dependsOnMethods = "testSubmitEntity") public void testGetTraitDefinitionForEntity() throws Exception{ traitName = "PII_Trait" + randomString();
