ATLAS-2875: Implement clear attribute value transformer for Atlas Entity Transformer
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/6c4d3990 Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/6c4d3990 Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/6c4d3990 Branch: refs/heads/branch-1.0 Commit: 6c4d399043314d17ff2348b4a7f5bd5d6f9f18fb Parents: e33b8bf Author: Sarath Subramanian <ssubraman...@hortonworks.com> Authored: Thu Sep 20 11:20:39 2018 -0700 Committer: Ashutosh Mestry <ames...@hortonworks.com> Committed: Thu Nov 1 15:42:55 2018 -0700 ---------------------------------------------------------------------- .../apache/atlas/entitytransform/Action.java | 18 ++++ .../entitytransform/BaseEntityHandler.java | 8 ++ .../apache/atlas/entitytransform/Condition.java | 22 +++++ .../TransformationHandlerTest.java | 99 ++++++++++++++++++++ 4 files changed, 147 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/atlas/blob/6c4d3990/intg/src/main/java/org/apache/atlas/entitytransform/Action.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/entitytransform/Action.java b/intg/src/main/java/org/apache/atlas/entitytransform/Action.java index ca5f3a8..f01c6ce 100644 --- a/intg/src/main/java/org/apache/atlas/entitytransform/Action.java +++ b/intg/src/main/java/org/apache/atlas/entitytransform/Action.java @@ -31,6 +31,7 @@ public abstract class Action { private static final String ACTION_NAME_REPLACE_PREFIX = "REPLACE_PREFIX"; private static final String ACTION_NAME_TO_LOWER = "TO_LOWER"; private static final String ACTION_NAME_TO_UPPER = "TO_UPPER"; + private static final String ACTION_NAME_CLEAR = "CLEAR"; protected final String attributeName; @@ -80,6 +81,10 @@ public abstract class Action { ret = new SetAction(key, actionValue); break; + case ACTION_NAME_CLEAR: + ret = new ClearAction(key); + break; + default: ret = new SetAction(key, value); // treat unspecified/unknown action as 'SET' break; @@ -196,4 +201,17 @@ public abstract class Action { } } } + + public static class ClearAction extends Action { + public ClearAction(String attributeName) { + super(attributeName); + } + + @Override + public void apply(AtlasTransformableEntity entity) { + if (isValid() && entity.hasAttribute(attributeName)) { + entity.setAttribute(attributeName, null); + } + } + } } http://git-wip-us.apache.org/repos/asf/atlas/blob/6c4d3990/intg/src/main/java/org/apache/atlas/entitytransform/BaseEntityHandler.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/entitytransform/BaseEntityHandler.java b/intg/src/main/java/org/apache/atlas/entitytransform/BaseEntityHandler.java index c1f2869..9d44043 100644 --- a/intg/src/main/java/org/apache/atlas/entitytransform/BaseEntityHandler.java +++ b/intg/src/main/java/org/apache/atlas/entitytransform/BaseEntityHandler.java @@ -95,6 +95,10 @@ public class BaseEntityHandler { } } + if (CollectionUtils.isEmpty(ret)) { + ret.add(new BaseEntityHandler(transformers)); + } + if (LOG.isDebugEnabled()) { LOG.debug("<== BaseEntityHandler.createEntityHandlers(transforms={}): ret.size={}", transforms, ret.size()); } @@ -158,6 +162,10 @@ public class BaseEntityHandler { } } + public boolean hasAttribute(String attributeName) { + return getAttribute(attributeName) != null; + } + public void transformComplete() { // implementations can override to set value of computed-attributes } http://git-wip-us.apache.org/repos/asf/atlas/blob/6c4d3990/intg/src/main/java/org/apache/atlas/entitytransform/Condition.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/entitytransform/Condition.java b/intg/src/main/java/org/apache/atlas/entitytransform/Condition.java index d44f575..bc63079 100644 --- a/intg/src/main/java/org/apache/atlas/entitytransform/Condition.java +++ b/intg/src/main/java/org/apache/atlas/entitytransform/Condition.java @@ -31,6 +31,7 @@ public abstract class Condition { private static final String CONDITION_NAME_EQUALS_IGNORE_CASE = "EQUALS_IGNORE_CASE"; private static final String CONDITION_NAME_STARTS_WITH = "STARTS_WITH"; private static final String CONDITION_NAME_STARTS_WITH_IGNORE_CASE = "STARTS_WITH_IGNORE_CASE"; + private static final String CONDITION_NAME_HAS_VALUE = "HAS_VALUE"; protected final String attributeName; @@ -75,6 +76,10 @@ public abstract class Condition { ret = new StartsWithIgnoreCaseCondition(key, conditionValue); break; + case CONDITION_NAME_HAS_VALUE: + ret = new HasValueCondition(key, conditionValue); + break; + default: ret = new EqualsCondition(key, value); // treat unspecified/unknown condition as 'EQUALS' break; @@ -158,4 +163,21 @@ public abstract class Condition { return attributeValue != null && StringUtils.startsWithIgnoreCase(attributeValue.toString(), this.prefix); } } + + public static class HasValueCondition extends Condition { + protected final String attributeValue; + + public HasValueCondition(String attributeName, String attributeValue) { + super(attributeName); + + this.attributeValue = attributeValue; + } + + @Override + public boolean matches(AtlasTransformableEntity entity) { + Object attributeValue = entity != null ? entity.getAttribute(attributeName) : null; + + return attributeValue != null ? StringUtils.isNotEmpty(attributeValue.toString()) : false; + } + } } http://git-wip-us.apache.org/repos/asf/atlas/blob/6c4d3990/intg/src/test/java/org/apache/atlas/entitytransform/TransformationHandlerTest.java ---------------------------------------------------------------------- diff --git a/intg/src/test/java/org/apache/atlas/entitytransform/TransformationHandlerTest.java b/intg/src/test/java/org/apache/atlas/entitytransform/TransformationHandlerTest.java index 69fba1e..a0ebe59 100644 --- a/intg/src/test/java/org/apache/atlas/entitytransform/TransformationHandlerTest.java +++ b/intg/src/test/java/org/apache/atlas/entitytransform/TransformationHandlerTest.java @@ -25,9 +25,12 @@ import org.testng.annotations.Test; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import static org.apache.atlas.entitytransform.TransformationConstants.HDFS_PATH; +import static org.apache.atlas.entitytransform.TransformationConstants.HIVE_TABLE; public class TransformationHandlerTest { @Test @@ -102,6 +105,100 @@ public class TransformationHandlerTest { } @Test + public void testHiveTableClearAttributeHandler() { + // clear replicatedTo attribute for hive_table entities + AttributeTransform p1 = new AttributeTransform(Collections.singletonMap("hive_table.replicatedTo", "HAS_VALUE:"), + Collections.singletonMap("hive_table.replicatedTo", "CLEAR:")); + + List<BaseEntityHandler> handlers = initializeHandlers(Collections.singletonList(p1)); + + List<AtlasEntity> entities = getAllEntities(); + + for (AtlasEntity entity : entities) { + String replicatedTo = (String) entity.getAttribute("replicatedTo"); + + if (entity.getTypeName() == HIVE_TABLE) { + Assert.assertTrue(StringUtils.isNotEmpty(replicatedTo)); + } + + applyTransforms(entity, handlers); + + String transformedValue = (String) entity.getAttribute("replicatedTo"); + + if (entity.getTypeName() == HIVE_TABLE) { + Assert.assertTrue(StringUtils.isEmpty(transformedValue)); + } + } + } + + @Test + public void testEntityClearAttributesActionWithNoCondition() { + // clear replicatedFrom attribute for hive_table entities without any condition + Map<String, String> actions = new HashMap<String, String>() {{ put("__entity.replicatedTo", "CLEAR:"); + put("__entity.replicatedFrom", "CLEAR:"); }}; + + AttributeTransform transform = new AttributeTransform(null, actions); + + List<BaseEntityHandler> handlers = initializeHandlers(Collections.singletonList(transform)); + + + List<AtlasEntity> entities = getAllEntities(); + + for (AtlasEntity entity : entities) { + String replicatedTo = (String) entity.getAttribute("replicatedTo"); + String replicatedFrom = (String) entity.getAttribute("replicatedFrom"); + + if (entity.getTypeName() == HIVE_TABLE) { + Assert.assertTrue(StringUtils.isNotEmpty(replicatedTo)); + Assert.assertTrue(StringUtils.isNotEmpty(replicatedFrom)); + } + + applyTransforms(entity, handlers); + + replicatedTo = (String) entity.getAttribute("replicatedTo"); + replicatedFrom = (String) entity.getAttribute("replicatedFrom"); + + if (entity.getTypeName() == HIVE_TABLE) { + Assert.assertTrue(StringUtils.isEmpty(replicatedTo)); + Assert.assertTrue(StringUtils.isEmpty(replicatedFrom)); + } + } + } + + @Test + public void testEntityClearAttributesActionWithNoTypeNameAndNoCondition() { + // clear replicatedFrom attribute for hive_table entities without any condition + Map<String, String> actions = new HashMap<String, String>() {{ put("replicatedTo", "CLEAR:"); + put("replicatedFrom", "CLEAR:"); }}; + + AttributeTransform transform = new AttributeTransform(null, actions); + + List<BaseEntityHandler> handlers = initializeHandlers(Collections.singletonList(transform)); + + List<AtlasEntity> entities = getAllEntities(); + + for (AtlasEntity entity : entities) { + String replicatedTo = (String) entity.getAttribute("replicatedTo"); + String replicatedFrom = (String) entity.getAttribute("replicatedFrom"); + + if (entity.getTypeName() == HIVE_TABLE) { + Assert.assertTrue(StringUtils.isNotEmpty(replicatedTo)); + Assert.assertTrue(StringUtils.isNotEmpty(replicatedFrom)); + } + + applyTransforms(entity, handlers); + + replicatedTo = (String) entity.getAttribute("replicatedTo"); + replicatedFrom = (String) entity.getAttribute("replicatedFrom"); + + if (entity.getTypeName() == HIVE_TABLE) { + Assert.assertTrue(StringUtils.isEmpty(replicatedTo)); + Assert.assertTrue(StringUtils.isEmpty(replicatedFrom)); + } + } + } + + @Test public void testHdfsPathNameReplacePrefixHandler() { // Prefix replace hdfs_path name from /aa/bb/ to /xx/yy/ AttributeTransform p1 = new AttributeTransform(Collections.singletonMap("hdfs_path.name", "STARTS_WITH: /aa/bb/"), @@ -338,6 +435,8 @@ public class TransformationHandlerTest { entity.setAttribute("tableType", "EXTERNAL_TABLE"); entity.setAttribute("createTime", "1535656355000"); entity.setAttribute("retention", 0); + entity.setAttribute("replicatedTo", "[{\"guid\":\"f378cfa5-c4aa-4699-a733-8f11d2f089cd\",\"typeName\":\"AtlasServer\"},{\"guid\":\"58e42789-ea3e-4eaa-a0c4-d38d8632e548\",\"typeName\":\"AtlasServer\"}]"); + entity.setAttribute("replicatedFrom", "[{\"guid\":\"f378cfa5-c4aa-4699-a733-8f11d2f089cd\",\"typeName\":\"AtlasServer\"},{\"guid\":\"58e42789-ea3e-4eaa-a0c4-d38d8632e548\",\"typeName\":\"AtlasServer\"}]"); return entity; }
