Repository: atlas Updated Branches: refs/heads/branch-0.8 ada74d10e -> f7b58111e
ATLAS-2799: Import Transforms: Additional transforms. Project: http://git-wip-us.apache.org/repos/asf/atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/f7b58111 Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/f7b58111 Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/f7b58111 Branch: refs/heads/branch-0.8 Commit: f7b58111e19234ba13b490d17d274d9ec477bb57 Parents: ada74d1 Author: Ashutosh Mestry <ames...@hortonworks.com> Authored: Wed Aug 1 17:41:46 2018 -0700 Committer: Ashutosh Mestry <ames...@hortonworks.com> Committed: Wed Aug 1 17:41:46 2018 -0700 ---------------------------------------------------------------------- .../repository/impexp/ImportTransformer.java | 186 ++++++++++++++++++- .../repository/impexp/ImportTransforms.java | 162 +++++++++------- .../impexp/ImportTransformerTest.java | 4 +- .../repository/impexp/ImportTransformsTest.java | 119 ++++++++++-- 4 files changed, 374 insertions(+), 97 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/atlas/blob/f7b58111/repository/src/main/java/org/apache/atlas/repository/impexp/ImportTransformer.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/ImportTransformer.java b/repository/src/main/java/org/apache/atlas/repository/impexp/ImportTransformer.java index 1b9305c..348bcd2 100644 --- a/repository/src/main/java/org/apache/atlas/repository/impexp/ImportTransformer.java +++ b/repository/src/main/java/org/apache/atlas/repository/impexp/ImportTransformer.java @@ -19,14 +19,25 @@ package org.apache.atlas.repository.impexp; import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.instance.AtlasClassification; +import org.apache.atlas.model.instance.AtlasEntity; import org.apache.commons.lang.StringUtils; +import java.util.ArrayList; +import java.util.List; + public abstract class ImportTransformer { private static final String TRANSFORMER_PARAMETER_SEPARATOR = "\\:"; - private final String transformType; + private static final String TRANSFORMER_NAME_ADD = "add"; + private static final String TRANSFORMER_NAME_CLEAR_ATTR = "clearAttrValue"; + private static final String TRANSFORMER_NAME_LOWERCASE = "lowercase"; + private static final String TRANSFORMER_NAME_UPPERCASE = "uppercase"; + private static final String TRANSFORMER_NAME_REMOVE_CLASSIFICATION = "removeClassification"; + private static final String TRANSFORMER_NAME_REPLACE = "replace"; + private final String transformType; public static ImportTransformer getTransformer(String transformerSpec) throws AtlasBaseException { String[] params = StringUtils.split(transformerSpec, TRANSFORMER_PARAMETER_SEPARATOR); @@ -36,15 +47,24 @@ public abstract class ImportTransformer { if (StringUtils.isEmpty(key)) { throw new AtlasBaseException(AtlasErrorCode.INVALID_VALUE, "Error creating ImportTransformer. Invalid transformer-specification: {}.", transformerSpec); - } else if (key.equals("replace")) { + } else if (key.equals(TRANSFORMER_NAME_REPLACE)) { String toFindStr = (params == null || params.length < 2) ? "" : params[1]; String replaceStr = (params == null || params.length < 3) ? "" : params[2]; ret = new Replace(toFindStr, replaceStr); - } else if (key.equals("lowercase")) { + } else if (key.equals(TRANSFORMER_NAME_LOWERCASE)) { ret = new Lowercase(); - } else if (key.equals("uppercase")) { + } else if (key.equals(TRANSFORMER_NAME_UPPERCASE)) { ret = new Uppercase(); + } else if (key.equals(TRANSFORMER_NAME_REMOVE_CLASSIFICATION)) { + String name = (params == null || params.length < 1) ? "" : StringUtils.join(params, ":", 1, params.length); + ret = new RemoveClassification(name); + } else if (key.equals(TRANSFORMER_NAME_ADD)) { + String name = (params == null || params.length < 1) ? "" : StringUtils.join(params, ":", 1, params.length); + ret = new AddValueToAttribute(name); + } else if (key.equals(TRANSFORMER_NAME_CLEAR_ATTR)) { + String name = (params == null || params.length < 1) ? "" : StringUtils.join(params, ":", 1, params.length); + ret = new ClearAttributes(name); } else { throw new AtlasBaseException(AtlasErrorCode.INVALID_VALUE, "Error creating ImportTransformer. Unknown transformer: {}.", transformerSpec); } @@ -66,7 +86,7 @@ public abstract class ImportTransformer { private final String replaceStr; public Replace(String toFindStr, String replaceStr) { - super("replace"); + super(TRANSFORMER_NAME_REPLACE); this.toFindStr = toFindStr; this.replaceStr = replaceStr; @@ -77,7 +97,7 @@ public abstract class ImportTransformer { public String getReplaceStr() { return replaceStr; } @Override - public Object apply(Object o) throws AtlasBaseException { + public Object apply(Object o) { Object ret = o; if(o instanceof String) { @@ -90,7 +110,7 @@ public abstract class ImportTransformer { static class Lowercase extends ImportTransformer { public Lowercase() { - super("lowercase"); + super(TRANSFORMER_NAME_LOWERCASE); } @Override @@ -107,7 +127,7 @@ public abstract class ImportTransformer { static class Uppercase extends ImportTransformer { public Uppercase() { - super("uppercase"); + super(TRANSFORMER_NAME_UPPERCASE); } @Override @@ -121,4 +141,154 @@ public abstract class ImportTransformer { return ret; } } + + static class RemoveClassification extends ImportTransformer { + private final String classificationToBeRemoved; + + public RemoveClassification(String name) { + super(TRANSFORMER_NAME_REMOVE_CLASSIFICATION); + + this.classificationToBeRemoved = name; + } + + @Override + public Object apply(Object o) { + if (!(o instanceof AtlasEntity)) { + return o; + } + + AtlasEntity entity = (AtlasEntity) o; + if(entity.getClassifications().size() == 0) { + return o; + } + + List<AtlasClassification> toRemove = null; + for (AtlasClassification classification : entity.getClassifications()) { + if (classification.getTypeName().equals(classificationToBeRemoved)) { + if (toRemove == null) { + toRemove = new ArrayList<AtlasClassification>(); + } + + + toRemove.add(classification); + + } + } + + if (toRemove != null) { + entity.getClassifications().removeAll(toRemove); + } + + return entity; + } + + @Override + public String toString() { + return String.format("%s=%s", "RemoveClassification", classificationToBeRemoved); + } + } + + static class AddValueToAttribute extends ImportTransformer { + private final String nameValuePair; + private String attrName; + private String attrValueRaw; + private Object attrValue; + + protected AddValueToAttribute(String nameValuePair) { + super(TRANSFORMER_NAME_ADD); + + this.nameValuePair = nameValuePair; + setAttrNameValue(this.nameValuePair); + } + + private void setAttrNameValue(String nameValuePair) { + String SEPARATOR_EQUALS = "="; + if(!nameValuePair.contains(SEPARATOR_EQUALS)) return; + + String splits[] = StringUtils.split(nameValuePair, SEPARATOR_EQUALS); + if(splits.length == 0) { + return; + } + + if(splits.length >= 1) { + attrName = splits[0]; + } + + if(splits.length >= 1) { + attrValueRaw = splits[1]; + } + + setAttrValue(attrValueRaw); + } + + private void setAttrValue(String attrValueRaw) { + final String type_prefix = "list:"; + + if(attrValueRaw.startsWith(type_prefix)) { + final String item = StringUtils.remove(attrValueRaw, type_prefix); + attrValue = new ArrayList<String>() {{ + add(item); + }}; + } else { + attrValue = attrValueRaw; + } + } + + @Override + public Object apply(Object o) { + if(o == null) { + return o; + } + + if(!(o instanceof AtlasEntity)) { + return o; + } + + AtlasEntity entity = (AtlasEntity) o; + Object attrExistingValue = entity.getAttribute(attrName); + if(attrExistingValue == null) { + entity.setAttribute(attrName, attrValue); + } else if(attrExistingValue instanceof List) { + List list = (List) attrExistingValue; + + if(attrValue instanceof List) { + list.addAll((List) attrValue); + } else { + list.add(attrValue); + } + } else { + entity.setAttribute(attrName, attrValueRaw); + } + + return entity; + } + } + + static class ClearAttributes extends ImportTransformer { + private String[] attrNames; + + protected ClearAttributes(String attrNames) { + super(TRANSFORMER_NAME_CLEAR_ATTR); + + this.attrNames = StringUtils.split(attrNames, ","); + } + + @Override + public Object apply(Object o) { + if (o == null) { + return o; + } + + if (!(o instanceof AtlasEntity)) { + return o; + } + + AtlasEntity entity = (AtlasEntity) o; + for (String attrName : attrNames) { + entity.setAttribute(attrName, null); + } + + return entity; + } + } } http://git-wip-us.apache.org/repos/asf/atlas/blob/f7b58111/repository/src/main/java/org/apache/atlas/repository/impexp/ImportTransforms.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/ImportTransforms.java b/repository/src/main/java/org/apache/atlas/repository/impexp/ImportTransforms.java index 2f27448..24c5d7b 100644 --- a/repository/src/main/java/org/apache/atlas/repository/impexp/ImportTransforms.java +++ b/repository/src/main/java/org/apache/atlas/repository/impexp/ImportTransforms.java @@ -35,17 +35,16 @@ import java.util.Set; public class ImportTransforms { private static final Logger LOG = LoggerFactory.getLogger(ImportTransforms.class); - private Map<String, Map<String, List<ImportTransformer>>> transforms; + private static final String ALL_ATTRIBUTES = "*"; + private Map<String, Map<String, List<ImportTransformer>>> transforms; public static ImportTransforms fromJson(String jsonString) { - ImportTransforms ret = null; - - if (StringUtils.isNotBlank(jsonString)) { - ret = new ImportTransforms(jsonString); + if (StringUtils.isEmpty(jsonString)) { + return null; } - return ret; + return new ImportTransforms(jsonString); } public Map<String, Map<String, List<ImportTransformer>>> getTransforms() { @@ -72,13 +71,15 @@ public class ImportTransforms { } public AtlasEntity.AtlasEntityWithExtInfo apply(AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo) throws AtlasBaseException { - if (entityWithExtInfo != null) { - apply(entityWithExtInfo.getEntity()); + if (entityWithExtInfo == null) { + return entityWithExtInfo; + } - if(MapUtils.isNotEmpty(entityWithExtInfo.getReferredEntities())) { - for (AtlasEntity e : entityWithExtInfo.getReferredEntities().values()) { - apply(e); - } + apply(entityWithExtInfo.getEntity()); + + if(MapUtils.isNotEmpty(entityWithExtInfo.getReferredEntities())) { + for (AtlasEntity e : entityWithExtInfo.getReferredEntities().values()) { + apply(e); } } @@ -86,27 +87,37 @@ public class ImportTransforms { } public AtlasEntity apply(AtlasEntity entity) throws AtlasBaseException { - if(entity != null) { - Map<String, List<ImportTransformer>> entityTransforms = getTransforms(entity.getTypeName()); + if (entity == null) { + return entity; + } - if (MapUtils.isNotEmpty(entityTransforms)) { - for (Map.Entry<String, List<ImportTransformer>> entry : entityTransforms.entrySet()) { - String attributeName = entry.getKey(); - List<ImportTransformer> attrTransforms = entry.getValue(); + Map<String, List<ImportTransformer>> entityTransforms = getTransforms(entity.getTypeName()); + if (MapUtils.isEmpty(entityTransforms)) { + return entity; + } - if (!entity.hasAttribute(attributeName)) { - continue; - } + for (Map.Entry<String, List<ImportTransformer>> entry : entityTransforms.entrySet()) { + String attributeName = entry.getKey(); + List<ImportTransformer> attrTransforms = entry.getValue(); - Object transformedValue = entity.getAttribute(attributeName); + if(attributeName.equals(ALL_ATTRIBUTES)) { + for (ImportTransformer attrTransform : attrTransforms) { + attrTransform.apply(entity); + } - for (ImportTransformer attrTransform : attrTransforms) { - transformedValue = attrTransform.apply(transformedValue); - } + continue; + } - entity.setAttribute(attributeName, transformedValue); - } + if (!entity.hasAttribute(attributeName)) { + continue; + } + + Object transformedValue = entity.getAttribute(attributeName); + for (ImportTransformer attrTransform : attrTransforms) { + transformedValue = attrTransform.apply(transformedValue); } + + entity.setAttribute(attributeName, transformedValue); } return entity; @@ -119,38 +130,58 @@ public class ImportTransforms { private ImportTransforms(String jsonString) { this(); - if(jsonString != null) { - Map typeTransforms = AtlasType.fromJson(jsonString, Map.class); - - if (MapUtils.isNotEmpty(typeTransforms)) { - for (Object key : typeTransforms.keySet()) { - Object value = typeTransforms.get(key); - String entityType = (String) key; - Map<String, Object> attributeTransforms = (Map<String, Object>)value; - - if (MapUtils.isNotEmpty(attributeTransforms)) { - for (Map.Entry<String, Object> e : attributeTransforms.entrySet()) { - String attributeName = e.getKey(); - List<String> transforms = (List<String>)e.getValue(); - - if (CollectionUtils.isNotEmpty(transforms)) { - for (String transform : transforms) { - ImportTransformer transformers = null; - - try { - transformers = ImportTransformer.getTransformer(transform); - } catch (AtlasBaseException ex) { - LOG.error("Error converting string to ImportTransformer: {}", transform, ex); - } - - if (transformers != null) { - add(entityType, attributeName, transformers); - } - } - } - } - } + if (StringUtils.isEmpty(jsonString)) { + return; + } + + Map typeTransforms = AtlasType.fromJson(jsonString, Map.class); + if (MapUtils.isEmpty(typeTransforms)) { + return; + } + + addOuterMap(typeTransforms); + } + + private void addOuterMap(Map typeTransforms) { + for (Object key : typeTransforms.keySet()) { + Object value = typeTransforms.get(key); + String entityType = (String) key; + Map<String, Object> attributeTransforms = (Map<String, Object>)value; + + if (MapUtils.isEmpty(attributeTransforms)) { + continue; + } + + addInnerMap(entityType, attributeTransforms); + } + } + + private void addInnerMap(String entityType, Map<String, Object> attributeTransforms) { + for (Map.Entry<String, Object> e : attributeTransforms.entrySet()) { + String attributeName = e.getKey(); + List<String> transforms = (List<String>)e.getValue(); + + if (CollectionUtils.isEmpty(transforms)) { + continue; + } + + addTransforms(entityType, attributeName, transforms); + } + } + + private void addTransforms(String entityType, String attributeName, List<String> transforms) { + for (String transform : transforms) { + ImportTransformer transformers = null; + + try { + transformers = ImportTransformer.getTransformer(transform); + if (transformers == null) { + continue; } + + add(entityType, attributeName, transformers); + } catch (AtlasBaseException ex) { + LOG.error("Error converting string to ImportTransformer: {}", transform, ex); } } } @@ -158,21 +189,16 @@ public class ImportTransforms { private void add(String typeName, String attributeName, ImportTransformer transformer) { Map<String, List<ImportTransformer>> attrMap; - if(transforms.containsKey(typeName)) { - attrMap = transforms.get(typeName); - } else { + if(!transforms.containsKey(typeName)) { attrMap = new HashMap<>(); transforms.put(typeName, attrMap); } - List<ImportTransformer> list; - if(attrMap.containsKey(attributeName)) { - list = attrMap.get(attributeName); - } else { - list = new ArrayList<>(); - attrMap.put(attributeName, list); + attrMap = transforms.get(typeName); + if(!attrMap.containsKey(attributeName)) { + attrMap.put(attributeName, new ArrayList<ImportTransformer>()); } - list.add(transformer); + attrMap.get(attributeName).add(transformer); } } http://git-wip-us.apache.org/repos/asf/atlas/blob/f7b58111/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformerTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformerTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformerTest.java index 7ce34c8..8be5ed0 100644 --- a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformerTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformerTest.java @@ -26,7 +26,7 @@ import static org.testng.Assert.assertTrue; public class ImportTransformerTest { @Test - public void createWithCorrectParameters() throws AtlasBaseException, IllegalAccessException { + public void createWithCorrectParameters() throws AtlasBaseException { String param1 = "@cl1"; String param2 = "@cl2"; @@ -38,7 +38,7 @@ public class ImportTransformerTest { } @Test - public void createSeveralWithCorrectParameters() throws AtlasBaseException, IllegalAccessException { + public void createSeveralWithCorrectParameters() throws AtlasBaseException { String param1 = "@cl1"; String param2 = "@cl2"; http://git-wip-us.apache.org/repos/asf/atlas/blob/f7b58111/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformsTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformsTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformsTest.java index a73abcd..9b38922 100644 --- a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformsTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformsTest.java @@ -18,44 +18,56 @@ package org.apache.atlas.repository.impexp; import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.instance.AtlasClassification; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo; +import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; -import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadModelFromJson; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; public class ImportTransformsTest { - private final String qualifiedName = "qualifiedName"; + private final String ATTR_NAME_QUALIFIED_NAME = "qualifiedName"; private final String lowerCaseCL1 = "@cl1"; private final String lowerCaseCL2 = "@cl2"; - private final String jsonTransforms = "{ \"hive_table\": { \"qualifiedName\":[ \"lowercase\", \"replace:@cl1:@cl2\" ] } }"; - private final String jsonTransforms2 = "{ \"Asset\": { \"qualifiedName\":[ \"replace:@cl1:@cl2\" ] }, \"hive_table\": { \"qualifiedName\":[ \"lowercase\", \"replace:@cl1:@cl2\" ] } }"; + private final String jsonLowerCaseReplace = "{ \"hive_table\": { \"qualifiedName\":[ \"lowercase\", \"replace:@cl1:@cl2\" ] } }"; + private final String jsonReplaceLowerCase = "{ \"Asset\": { \"qualifiedName\":[ \"replace:@cl1:@cl2\" ] }, \"hive_table\": { \"qualifiedName\":[ \"lowercase\", \"replace:@cl1:@cl2\" ] } }"; + private final String jsonReplaceRemoveClassification = "{ \"hive_table\": { \"qualifiedName\":[ \"replace:@%s:@%s\"], \"*\":[ \"removeClassification:%s_to_%s\" ] } }"; + private final String jsonReplaceAndAddAttrValue = "{ \"hive_table\": { \"qualifiedName\":[ \"replace:@%s:@%s\"], \"*\":[ \"add:%s=list:%s\" ] } }"; + private final String jsonSingleClearAttrValue = "{ \"hive_table\": { \"*\":[ \"clearAttrValue:replicatedToCluster\", \"clearAttrValue:replicatedFromCluster\" ] } }"; + private final String jsonMultipleClearAttrValue = "{ \"hive_table\": { \"*\":[ \"clearAttrValue:replicatedToCluster,replicatedFromCluster\" ] } }"; private ImportTransforms transform; + private String HIVE_TABLE_ATTR_SYNC_INFO = "hive_table.syncInfo"; + private String HIVE_TABLE_ATTR_REPLICATED_FROM = "replicatedFromCluster"; + private String HIVE_TABLE_ATTR_REPLICATED_TO = "replicatedToCluster"; @BeforeTest - public void setup() throws AtlasBaseException { - transform = ImportTransforms.fromJson(jsonTransforms); + public void setup() { + transform = ImportTransforms.fromJson(jsonLowerCaseReplace); + } + + @BeforeMethod + public void setUp() { } @Test public void transformEntityWith2Transforms() throws AtlasBaseException { AtlasEntity entity = getHiveTableAtlasEntity(); - String attrValue = (String) entity.getAttribute(qualifiedName); + String attrValue = (String) entity.getAttribute(ATTR_NAME_QUALIFIED_NAME); transform.apply(entity); - assertEquals(entity.getAttribute(qualifiedName), applyDefaultTransform(attrValue)); + assertEquals(entity.getAttribute(ATTR_NAME_QUALIFIED_NAME), applyDefaultTransform(attrValue)); } @Test @@ -64,15 +76,15 @@ public class ImportTransformsTest { AtlasEntityWithExtInfo entityWithExtInfo = getAtlasEntityWithExtInfo(); AtlasEntity entity = entityWithExtInfo.getEntity(); - String attrValue = (String) entity.getAttribute(qualifiedName); + String attrValue = (String) entity.getAttribute(ATTR_NAME_QUALIFIED_NAME); String[] expectedValues = getExtEntityExpectedValues(entityWithExtInfo); transform.apply(entityWithExtInfo); - assertEquals(entityWithExtInfo.getEntity().getAttribute(qualifiedName), applyDefaultTransform(attrValue)); + assertEquals(entityWithExtInfo.getEntity().getAttribute(ATTR_NAME_QUALIFIED_NAME), applyDefaultTransform(attrValue)); for (int i = 0; i < expectedValues.length; i++) { - assertEquals(entityWithExtInfo.getReferredEntities().get(Integer.toString(i)).getAttribute(qualifiedName), expectedValues[i]); + assertEquals(entityWithExtInfo.getReferredEntities().get(Integer.toString(i)).getAttribute(ATTR_NAME_QUALIFIED_NAME), expectedValues[i]); } } @@ -92,17 +104,85 @@ public class ImportTransformsTest { @Test public void transformFromJsonWithMultipleEntries() { - ImportTransforms t = ImportTransforms.fromJson(jsonTransforms2); + ImportTransforms t = ImportTransforms.fromJson(jsonReplaceLowerCase); assertNotNull(t); assertEquals(t.getTransforms().size(), 2); } + @Test + public void removeClassificationTransform_RemovesSpecifiedClassification() throws AtlasBaseException { + List<AtlasClassification> classifications = new ArrayList<>(); + classifications.add(new AtlasClassification("cl2_to_cl1")); + + String s = String.format(jsonReplaceRemoveClassification, "cl1", "cl2", "cl2", "cl1"); + ImportTransforms t = ImportTransforms.fromJson(s); + + AtlasEntity entity = getHiveTableAtlasEntity(); + String expected_qualifiedName = entity.getAttribute(ATTR_NAME_QUALIFIED_NAME).toString().replace("@cl1", "@cl2"); + entity.setClassifications(classifications); + assertEquals(entity.getClassifications().size(), 1); + + t.apply(entity); + + assertEquals(entity.getClassifications().size(), 0); + assertNotNull(t); + assertEquals(entity.getAttribute(ATTR_NAME_QUALIFIED_NAME), expected_qualifiedName); + } + + @Test + public void add_setsValueOfAttribute() throws AtlasBaseException { + final String expected_syncInfo = "cl1:import"; + String s = String.format(jsonReplaceAndAddAttrValue, "cl1", "cl2", HIVE_TABLE_ATTR_SYNC_INFO, expected_syncInfo); + ImportTransforms t = ImportTransforms.fromJson(s); + + AtlasEntity entity = getHiveTableAtlasEntity(); + String expected_qualifiedName = entity.getAttribute(ATTR_NAME_QUALIFIED_NAME).toString().replace("@cl1", "@cl2"); + + t.apply(entity); + + assertNotNull(t); + assertEquals(entity.getAttribute(ATTR_NAME_QUALIFIED_NAME), expected_qualifiedName); + assertEquals(entity.getAttribute(HIVE_TABLE_ATTR_SYNC_INFO), new ArrayList<String>() {{ add(expected_syncInfo); }}); + } + + + @Test + public void clearAttrValue_removesValueOfAttribute() throws AtlasBaseException { + AtlasEntity entity = getHiveTableAtlasEntity(); + assertNotNull(entity.getAttribute(HIVE_TABLE_ATTR_REPLICATED_FROM)); + assertNotNull(entity.getAttribute(HIVE_TABLE_ATTR_REPLICATED_TO)); + + ImportTransforms t = ImportTransforms.fromJson(jsonSingleClearAttrValue); + + assertTrue(t.getTransforms().size() > 0); + + t.apply(entity); + + assertNotNull(t); + assertNull(entity.getAttribute(HIVE_TABLE_ATTR_REPLICATED_FROM)); + assertNull(entity.getAttribute(HIVE_TABLE_ATTR_REPLICATED_TO)); + } + + @Test + public void clearAttrValueForMultipleAttributes_removesValueOfAttribute() throws AtlasBaseException { + AtlasEntity entity = getHiveTableAtlasEntity(); + ImportTransforms t = ImportTransforms.fromJson(jsonMultipleClearAttrValue); + + assertTrue(t.getTransforms().size() > 0); + + t.apply(entity); + + assertNotNull(t); + assertNull(entity.getAttribute(HIVE_TABLE_ATTR_REPLICATED_FROM)); + assertNull(entity.getAttribute(HIVE_TABLE_ATTR_REPLICATED_TO)); + } + private String[] getExtEntityExpectedValues(AtlasEntityWithExtInfo entityWithExtInfo) { String[] ret = new String[entityWithExtInfo.getReferredEntities().size()]; for (int i = 0; i < ret.length; i++) { - String attrValue = (String) entityWithExtInfo.getReferredEntities().get(Integer.toString(i)).getAttribute(qualifiedName); + String attrValue = (String) entityWithExtInfo.getReferredEntities().get(Integer.toString(i)).getAttribute(ATTR_NAME_QUALIFIED_NAME); ret[i] = attrValue.replace(lowerCaseCL1, lowerCaseCL2); } @@ -115,9 +195,7 @@ public class ImportTransformsTest { List<ImportTransformer> trList = new ArrayList<>(); trList.add(ImportTransformer.getTransformer(String.format("replace:%s:%s", lowerCaseCL1, lowerCaseCL2))); - - tr.put(qualifiedName, trList); - + tr.put(ATTR_NAME_QUALIFIED_NAME, trList); transform.getTransforms().put("hive_column", tr); } @@ -129,9 +207,12 @@ public class ImportTransformsTest { AtlasEntity entity = new AtlasEntity("hive_table"); Map<String, Object> attributes = new HashMap<>(); - attributes.put(qualifiedName, "TABLE1.default" + lowerCaseCL1); + attributes.put(ATTR_NAME_QUALIFIED_NAME, "TABLE1.default" + lowerCaseCL1); attributes.put("dbname", "someDB"); attributes.put("name", "somename"); + attributes.put(HIVE_TABLE_ATTR_SYNC_INFO, null); + attributes.put(HIVE_TABLE_ATTR_REPLICATED_FROM, "cl1"); + attributes.put(HIVE_TABLE_ATTR_REPLICATED_TO, "clx"); entity.setAttributes(attributes); return entity; @@ -141,7 +222,7 @@ public class ImportTransformsTest { AtlasEntity entity = new AtlasEntity("hive_column"); Map<String, Object> attributes = new HashMap<>(); - attributes.put(qualifiedName, String.format("col%s.TABLE1.default@cl1", index)); + attributes.put(ATTR_NAME_QUALIFIED_NAME, String.format("col%s.TABLE1.default@cl1", index)); attributes.put("name", "col" + index); entity.setAttributes(attributes);
