This is an automated email from the ASF dual-hosted git repository. madhan pushed a commit to branch branch-2.0 in repository https://gitbox.apache.org/repos/asf/atlas.git
commit 9e17881ae092141a6656c78c14cab6ea19728b03 Author: Madhan Neethiraj <[email protected]> AuthorDate: Tue Mar 31 10:53:43 2020 -0700 ATLAS-3700: added option to append value to array, map type attributes (cherry picked from commit f5cd728d561a3e58a6b69c8b635e04769103a67f) --- .../model/instance/EntityMutationResponse.java | 5 + .../apache/atlas/model/typedef/AtlasStructDef.java | 16 +++- .../test/java/org/apache/atlas/TestUtilsV2.java | 15 ++- .../store/graph/v2/EntityGraphMapper.java | 52 ++++++++--- .../store/graph/v2/AtlasComplexAttributesTest.java | 101 ++++++++++++++++++++- 5 files changed, 168 insertions(+), 21 deletions(-) diff --git a/intg/src/main/java/org/apache/atlas/model/instance/EntityMutationResponse.java b/intg/src/main/java/org/apache/atlas/model/instance/EntityMutationResponse.java index 1434a24..d83f0e0 100644 --- a/intg/src/main/java/org/apache/atlas/model/instance/EntityMutationResponse.java +++ b/intg/src/main/java/org/apache/atlas/model/instance/EntityMutationResponse.java @@ -209,6 +209,11 @@ public class EntityMutationResponse { } @JsonIgnore + public AtlasEntityHeader getFirstPartialUpdatedEntityByTypeName(String typeName) { + return getFirstEntityByType(getEntitiesByOperation(EntityOperation.PARTIAL_UPDATE), typeName); + } + + @JsonIgnore public void addEntity(EntityOperation op, AtlasEntityHeader header) { // if an entity is already included in CREATE, ignore subsequent UPDATE, PARTIAL_UPDATE if (op == EntityOperation.UPDATE || op == EntityOperation.PARTIAL_UPDATE) { diff --git a/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java index 1d4e37b..a621fb0 100644 --- a/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java +++ b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java @@ -263,10 +263,11 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable { public static final int DEFAULT_SEARCHWEIGHT = -1; - public static final String SEARCH_WEIGHT_ATTR_NAME = "searchWeight"; - public static final String INDEX_TYPE_ATTR_NAME = "indexType"; - public static final String ATTRDEF_OPTION_SOFT_REFERENCE = "isSoftReference"; - private final String STRING_TRUE = "true"; + public static final String SEARCH_WEIGHT_ATTR_NAME = "searchWeight"; + public static final String INDEX_TYPE_ATTR_NAME = "indexType"; + public static final String ATTRDEF_OPTION_SOFT_REFERENCE = "isSoftReference"; + public static final String ATTRDEF_OPTION_APPEND_ON_PARTIAL_UPDATE = "isAppendOnPartialUpdate"; + private final String STRING_TRUE = "true"; /** * single-valued attribute or multi-valued attribute. @@ -520,6 +521,13 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable { } @JsonIgnore + public boolean isAppendOnPartialUpdate() { + String val = getOption(AtlasAttributeDef.ATTRDEF_OPTION_APPEND_ON_PARTIAL_UPDATE); + + return val != null && Boolean.valueOf(val); + } + + @JsonIgnore public void setOption(String name, String value) { if (this.options == null) { this.options = new HashMap<>(); diff --git a/intg/src/test/java/org/apache/atlas/TestUtilsV2.java b/intg/src/test/java/org/apache/atlas/TestUtilsV2.java index 7ec2b87..55497fc 100755 --- a/intg/src/test/java/org/apache/atlas/TestUtilsV2.java +++ b/intg/src/test/java/org/apache/atlas/TestUtilsV2.java @@ -627,6 +627,13 @@ public final class TestUtilsV2 { } public static AtlasTypesDef defineSimpleAttrType() { + AtlasAttributeDef attrPuArray = new AtlasAttributeDef("puArray", "array<string>", true, SINGLE, 1, 1, false, false, false, null); + AtlasAttributeDef attrPuMap = new AtlasAttributeDef("puMap", "map<string,string>", true, SINGLE, 1,1, false, false, false, null); + + attrPuArray.setOption(AtlasAttributeDef.ATTRDEF_OPTION_APPEND_ON_PARTIAL_UPDATE, "true"); + attrPuMap.setOption(AtlasAttributeDef.ATTRDEF_OPTION_APPEND_ON_PARTIAL_UPDATE, "true"); + + AtlasEntityDef simpleAttributesEntityType = createClassTypeDef(ENTITY_TYPE_WITH_SIMPLE_ATTR, ENTITY_TYPE_WITH_SIMPLE_ATTR + "_description", null, createUniqueRequiredAttrDef("name", "string"), @@ -641,7 +648,11 @@ public final class TestUtilsV2 { false, false, false, null), new AtlasAttributeDef("mapOfStrings", "map<string,string>", - true, SINGLE, 1,1, false, false, false, null) + true, SINGLE, 1,1, false, false, false, null), + + attrPuArray, + + attrPuMap ); AtlasTypesDef ret = AtlasTypeUtil.getTypesDef(Collections.<AtlasEnumDef>emptyList(), @@ -659,6 +670,8 @@ public final class TestUtilsV2 { entity.setAttribute("stringAtrr", "DummyThree"); entity.setAttribute("arrayOfStrings", Arrays.asList("DummyOne", "DummyTwo")); entity.setAttribute("mapOfStrings", Collections.singletonMap("one", "DummyString")); + entity.setAttribute("puArray", Arrays.asList("DummyOne", "DummyTwo")); + entity.setAttribute("puMap", Collections.singletonMap("one", "DummyString")); return new AtlasEntityWithExtInfo(entity); } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index fc432b4..b983af3 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -284,6 +284,8 @@ public class EntityGraphMapper { } } + EntityOperation updateType = isPartialUpdate ? PARTIAL_UPDATE : UPDATE; + if (CollectionUtils.isNotEmpty(updatedEntities)) { for (AtlasEntity updatedEntity : updatedEntities) { String guid = updatedEntity.getGuid(); @@ -292,14 +294,10 @@ public class EntityGraphMapper { mapRelationshipAttributes(updatedEntity, entityType, vertex, UPDATE, context); - mapAttributes(updatedEntity, entityType, vertex, UPDATE, context); + mapAttributes(updatedEntity, entityType, vertex, updateType, context); setCustomAttributes(vertex,updatedEntity); - if (isPartialUpdate) { - resp.addEntity(PARTIAL_UPDATE, constructHeader(updatedEntity, entityType, vertex)); - } else { - resp.addEntity(UPDATE, constructHeader(updatedEntity, entityType, vertex)); - } + resp.addEntity(updateType, constructHeader(updatedEntity, entityType, vertex)); if (replaceClassifications) { deleteClassifications(guid); @@ -325,12 +323,7 @@ public class EntityGraphMapper { } for (AtlasEntityHeader entity : req.getUpdatedEntities()) { - if (isPartialUpdate) { - resp.addEntity(PARTIAL_UPDATE, entity); - } - else { - resp.addEntity(UPDATE, entity); - } + resp.addEntity(updateType, entity); } RequestContext.get().endMetricRecord(metric); @@ -623,7 +616,7 @@ public class EntityGraphMapper { mapAttribute(attribute, attrValue, vertex, op, context); } - } else if (op.equals(UPDATE)) { + } else if (op.equals(UPDATE) || op.equals(PARTIAL_UPDATE)) { for (String attrName : struct.getAttributes().keySet()) { AtlasAttribute attribute = structType.getAttribute(attrName); @@ -665,7 +658,7 @@ public class EntityGraphMapper { mapAttribute(attribute, attrValue, vertex, op, context); } - } else if (op.equals(UPDATE)) { + } else if (op.equals(UPDATE) || op.equals(PARTIAL_UPDATE)) { // relationship attributes mapping for (String attrName : entityType.getRelationshipAttributes().keySet()) { if (entity.hasRelationshipAttribute(attrName)) { @@ -1251,6 +1244,22 @@ public class EntityGraphMapper { boolean isReference = isReference(mapType.getValueType()); boolean isSoftReference = ctx.getAttribute().getAttributeDef().isSoftReferenced(); + if (PARTIAL_UPDATE.equals(ctx.getOp()) && attribute.getAttributeDef().isAppendOnPartialUpdate() && MapUtils.isNotEmpty(currentMap)) { + if (MapUtils.isEmpty(newVal)) { + newVal = new HashMap<>(currentMap); + } else { + Map<Object, Object> mergedVal = new HashMap<>(currentMap); + + for (Map.Entry<Object, Object> entry : newVal.entrySet()) { + String newKey = entry.getKey().toString(); + + mergedVal.put(newKey, entry.getValue()); + } + + newVal = mergedVal; + } + } + boolean isNewValNull = newVal == null; if (isNewValNull) { @@ -1334,7 +1343,6 @@ public class EntityGraphMapper { Cardinality cardinality = attribute.getAttributeDef().getCardinality(); List<Object> newElementsCreated = new ArrayList<>(); List<Object> currentElements; - boolean isNewElementsNull = newElements == null; if (isReference && !isSoftReference) { currentElements = (List) getCollectionElementsUsingRelationship(ctx.getReferringVertex(), attribute); @@ -1342,6 +1350,20 @@ public class EntityGraphMapper { currentElements = (List) getArrayElementsProperty(elementType, isSoftReference, ctx.getReferringVertex(), ctx.getVertexProperty()); } + if (PARTIAL_UPDATE.equals(ctx.getOp()) && attribute.getAttributeDef().isAppendOnPartialUpdate() && CollectionUtils.isNotEmpty(currentElements)) { + if (CollectionUtils.isEmpty(newElements)) { + newElements = new ArrayList<>(currentElements); + } else { + List<Object> mergedVal = new ArrayList<>(currentElements); + + mergedVal.addAll(newElements); + + newElements = mergedVal; + } + } + + boolean isNewElementsNull = newElements == null; + if (isNewElementsNull) { newElements = new ArrayList(); } diff --git a/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasComplexAttributesTest.java b/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasComplexAttributesTest.java index b682e86..ad5fa92 100644 --- a/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasComplexAttributesTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasComplexAttributesTest.java @@ -32,6 +32,7 @@ import org.apache.atlas.repository.graphdb.AtlasEdgeDirection; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.type.AtlasEntityType; import org.apache.commons.lang.time.DateUtils; +import org.locationtech.jts.util.Assert; import org.testng.annotations.BeforeClass; import org.testng.annotations.Guice; import org.testng.annotations.Test; @@ -212,6 +213,8 @@ public class AtlasComplexAttributesTest extends AtlasEntityTestBase { createdSimpleEntity.setAttribute("stringAtrr", null); createdSimpleEntity.setAttribute("mapOfStrings", Collections.emptyMap()); createdSimpleEntity.setAttribute("arrayOfStrings", Collections.emptyList()); + createdSimpleEntity.setAttribute("puArray", Collections.emptyList()); + createdSimpleEntity.setAttribute("puMap", Collections.emptyMap()); EntityMutationResponse responseUpdated = entityStore.createOrUpdate(new AtlasEntityStream(createdSimpleEntity), false); AtlasEntityHeader simpleEntityUpdatedHeader = responseUpdated.getFirstUpdatedEntityByTypeName(ENTITY_TYPE_WITH_SIMPLE_ATTR); AtlasEntity updatedSimpleEntity = getEntityFromStore(simpleEntityUpdatedHeader); @@ -219,10 +222,14 @@ public class AtlasComplexAttributesTest extends AtlasEntityTestBase { assertNull(updatedSimpleEntity.getAttribute("stringAtrr")); assertEquals(updatedSimpleEntity.getAttribute("mapOfStrings"), Collections.emptyMap()); assertEquals(updatedSimpleEntity.getAttribute("arrayOfStrings"), Collections.emptyList()); + assertEquals(updatedSimpleEntity.getAttribute("puArray"), Collections.emptyList()); + assertEquals(updatedSimpleEntity.getAttribute("puMap"), Collections.emptyMap()); updatedSimpleEntity.setAttribute("stringAtrr", ""); updatedSimpleEntity.setAttribute("mapOfStrings", null); updatedSimpleEntity.setAttribute("arrayOfStrings", null); + updatedSimpleEntity.setAttribute("puArray", null); + updatedSimpleEntity.setAttribute("puMap", null); EntityMutationResponse responseUpdatedAgain = entityStore.createOrUpdate(new AtlasEntityStream(updatedSimpleEntity), false); AtlasEntityHeader simpleEntityUpdatedAgainHeader = responseUpdatedAgain.getFirstUpdatedEntityByTypeName(ENTITY_TYPE_WITH_SIMPLE_ATTR); AtlasEntity updatedAgainSimpleEntity = getEntityFromStore(simpleEntityUpdatedAgainHeader); @@ -230,10 +237,14 @@ public class AtlasComplexAttributesTest extends AtlasEntityTestBase { assertEquals(updatedAgainSimpleEntity.getAttribute("stringAtrr"), ""); assertNull(updatedAgainSimpleEntity.getAttribute("arrayOfStrings")); assertNull(updatedAgainSimpleEntity.getAttribute("mapOfStrings")); + assertNull(updatedAgainSimpleEntity.getAttribute("puArray")); + assertNull(updatedAgainSimpleEntity.getAttribute("puMap")); updatedAgainSimpleEntity.setAttribute("stringAtrr", "Dummy String Test 3"); updatedAgainSimpleEntity.setAttribute("mapOfStrings", Collections.singletonMap("key1", "val1")); updatedAgainSimpleEntity.setAttribute("arrayOfStrings", Arrays.asList("DummyTest3", "DummyTest4")); + updatedAgainSimpleEntity.setAttribute("puArray", Arrays.asList("1")); + updatedAgainSimpleEntity.setAttribute("puMap", Collections.singletonMap("1", "1")); EntityMutationResponse updateRes = entityStore.createOrUpdate(new AtlasEntityStream(updatedAgainSimpleEntity), false); AtlasEntityHeader updateHeader = updateRes.getFirstUpdatedEntityByTypeName(ENTITY_TYPE_WITH_SIMPLE_ATTR); AtlasEntity updateEntity = getEntityFromStore(updateHeader); @@ -241,6 +252,94 @@ public class AtlasComplexAttributesTest extends AtlasEntityTestBase { assertEquals(updateEntity.getAttribute("stringAtrr"), "Dummy String Test 3"); assertEquals(updateEntity.getAttribute("arrayOfStrings"), Arrays.asList("DummyTest3", "DummyTest4")); assertEquals(updateEntity.getAttribute("mapOfStrings"), Collections.singletonMap("key1", "val1")); + assertEquals(updateEntity.getAttribute("puArray"), Arrays.asList("1")); + assertEquals(updateEntity.getAttribute("puMap"), Collections.singletonMap("1", "1")); + + // full-update puArray and puMap; existing values should be replaced + updatedAgainSimpleEntity.setAttribute("stringAtrr", "Dummy String Test 3"); + updatedAgainSimpleEntity.setAttribute("mapOfStrings", Collections.singletonMap("key1", "val1")); + updatedAgainSimpleEntity.setAttribute("arrayOfStrings", Arrays.asList("DummyTest3", "DummyTest4")); + updatedAgainSimpleEntity.setAttribute("puArray", Arrays.asList("10")); + updatedAgainSimpleEntity.setAttribute("puMap", Collections.singletonMap("10", "10")); + updateRes = entityStore.createOrUpdate(new AtlasEntityStream(updatedAgainSimpleEntity), false); + updateHeader = updateRes.getFirstUpdatedEntityByTypeName(ENTITY_TYPE_WITH_SIMPLE_ATTR); + updateEntity = getEntityFromStore(updateHeader); + + assertEquals(updateEntity.getAttribute("stringAtrr"), "Dummy String Test 3"); + assertEquals(updateEntity.getAttribute("arrayOfStrings"), Arrays.asList("DummyTest3", "DummyTest4")); + assertEquals(updateEntity.getAttribute("mapOfStrings"), Collections.singletonMap("key1", "val1")); + assertEquals(updateEntity.getAttribute("puArray"), Arrays.asList("10")); + assertEquals(updateEntity.getAttribute("puMap"), Collections.singletonMap("10", "10")); + + // partial-update tests + // set puArray and puMap to null + updatedAgainSimpleEntity.setAttribute("stringAtrr", "Dummy String Test 3"); + updatedAgainSimpleEntity.setAttribute("mapOfStrings", Collections.singletonMap("key1", "val1")); + updatedAgainSimpleEntity.setAttribute("arrayOfStrings", Arrays.asList("DummyTest3", "DummyTest4")); + updatedAgainSimpleEntity.setAttribute("puArray", null); + updatedAgainSimpleEntity.setAttribute("puMap", null); + updateRes = entityStore.createOrUpdate(new AtlasEntityStream(updatedAgainSimpleEntity), false); + updateHeader = updateRes.getFirstUpdatedEntityByTypeName(ENTITY_TYPE_WITH_SIMPLE_ATTR); + updateEntity = getEntityFromStore(updateHeader); + + assertEquals(updateEntity.getAttribute("stringAtrr"), "Dummy String Test 3"); + assertEquals(updateEntity.getAttribute("arrayOfStrings"), Arrays.asList("DummyTest3", "DummyTest4")); + assertEquals(updateEntity.getAttribute("mapOfStrings"), Collections.singletonMap("key1", "val1")); + assertNull(updateEntity.getAttribute("puArray")); + assertNull(updateEntity.getAttribute("puMap")); + + List<String> puArray = new ArrayList<>(); + Map<String, String> puMap = new HashMap<>(); + + // partial-update: current value as null + updatedAgainSimpleEntity.getAttributes().clear(); + updatedAgainSimpleEntity.setAttribute("puArray", Collections.singletonList("1")); + updatedAgainSimpleEntity.setAttribute("puMap", Collections.singletonMap("1", "1")); + updateRes = entityStore.createOrUpdate(new AtlasEntityStream(updatedAgainSimpleEntity), true); + updateHeader = updateRes.getFirstPartialUpdatedEntityByTypeName(ENTITY_TYPE_WITH_SIMPLE_ATTR); + updateEntity = getEntityFromStore(updateHeader); + + puArray.addAll(Collections.singletonList("1")); + puMap.putAll(Collections.singletonMap("1", "1")); + + Assert.equals(updateEntity.getAttribute("puArray"), puArray); + Assert.equals(updateEntity.getAttribute("puMap"), puMap); + + // partial-update: append to existing value + updatedAgainSimpleEntity.getAttributes().clear(); + updatedAgainSimpleEntity.setAttribute("puArray", Collections.singletonList("2")); + updatedAgainSimpleEntity.setAttribute("puMap", Collections.singletonMap("2", "2")); + updateRes = entityStore.createOrUpdate(new AtlasEntityStream(updatedAgainSimpleEntity), true); + updateHeader = updateRes.getFirstPartialUpdatedEntityByTypeName(ENTITY_TYPE_WITH_SIMPLE_ATTR); + updateEntity = getEntityFromStore(updateHeader); + + puArray.addAll(Collections.singletonList("2")); + puMap.putAll(Collections.singletonMap("2", "2")); + + Assert.equals(updateEntity.getAttribute("puArray"), puArray); + Assert.equals(updateEntity.getAttribute("puMap"), puMap); + + // partial-update: with null value; existing value should be retained + updatedAgainSimpleEntity.getAttributes().clear(); + updatedAgainSimpleEntity.setAttribute("puArray", null); + updatedAgainSimpleEntity.setAttribute("puMap", null); + updateRes = entityStore.createOrUpdate(new AtlasEntityStream(updatedAgainSimpleEntity), true); + updateHeader = updateRes.getFirstPartialUpdatedEntityByTypeName(ENTITY_TYPE_WITH_SIMPLE_ATTR); + updateEntity = getEntityFromStore(updateHeader); + + Assert.equals(updateEntity.getAttribute("puArray"), puArray); + Assert.equals(updateEntity.getAttribute("puMap"), puMap); + + // partial-update: with empty value; existing value should be retained + updatedAgainSimpleEntity.getAttributes().clear(); + updatedAgainSimpleEntity.setAttribute("puArray", Collections.emptyList()); + updatedAgainSimpleEntity.setAttribute("puMap", Collections.emptyMap()); + updateRes = entityStore.createOrUpdate(new AtlasEntityStream(updatedAgainSimpleEntity), true); + updateHeader = updateRes.getFirstPartialUpdatedEntityByTypeName(ENTITY_TYPE_WITH_SIMPLE_ATTR); + updateEntity = getEntityFromStore(updateHeader); + + Assert.equals(updateEntity.getAttribute("puArray"), puArray); + Assert.equals(updateEntity.getAttribute("puMap"), puMap); } @Test(dependsOnMethods = "testCreateComplexAttributeEntity") @@ -570,4 +669,4 @@ public class AtlasComplexAttributesTest extends AtlasEntityTestBase { assertEquals(entity.getStatus(), AtlasEntity.Status.DELETED); } } -} \ No newline at end of file +}
