This is an automated email from the ASF dual-hosted git repository. madhan pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/master by this push: new ac808af ATLAS-3613: updated search to support namespace attributes ac808af is described below commit ac808af07c69191e34a279be169b8f01e5d35240 Author: Madhan Neethiraj <mad...@apache.org> AuthorDate: Mon Feb 10 17:53:38 2020 -0800 ATLAS-3613: updated search to support namespace attributes --- .../apache/atlas/type/AtlasClassificationType.java | 5 + .../org/apache/atlas/type/AtlasEntityType.java | 154 ++++++++++++++------- .../org/apache/atlas/type/AtlasStructType.java | 16 ++- .../apache/atlas/query/GremlinQueryComposer.java | 4 +- .../org/apache/atlas/query/IdentifierHelper.java | 27 ++-- .../apache/atlas/query/RegistryBasedLookup.java | 43 ++---- .../store/graph/v2/AtlasEntityStoreV2.java | 10 +- .../store/graph/v2/EntityGraphMapper.java | 34 ++--- .../store/graph/v2/EntityGraphRetriever.java | 33 +++-- .../graph/v2/AtlasNamespaceDefStoreV2Test.java | 6 +- 10 files changed, 199 insertions(+), 133 deletions(-) diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java b/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java index 1518bbc..4d0179c 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java @@ -238,6 +238,11 @@ public class AtlasClassificationType extends AtlasStructType { classificationDef.setSubTypes(subTypes); } + @Override + public AtlasAttribute getSystemAttribute(String attributeName) { + return AtlasClassificationType.CLASSIFICATION_ROOT.allAttributes.get(attributeName); + } + private void addSubType(AtlasClassificationType subType) { subTypes.add(subType.getTypeName()); } diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java index 7594153..2824feb 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java @@ -55,7 +55,8 @@ import java.util.Set; public class AtlasEntityType extends AtlasStructType { private static final Logger LOG = LoggerFactory.getLogger(AtlasEntityType.class); - public static final AtlasEntityType ENTITY_ROOT = initRootEntityType(); + public static final AtlasEntityType ENTITY_ROOT = new AtlasRootEntityType(); + private static final String NAME = "name"; private static final String DESCRIPTION = "description"; private static final String OWNER = "owner"; @@ -64,6 +65,8 @@ public class AtlasEntityType extends AtlasStructType { private static final String OPTION_SCHEMA_ATTRIBUTES = "schemaAttributes"; private static final String INTERNAL_TYPENAME = "__internal"; + private static final char NS_ATTRIBUTE_NAME_SEPARATOR = '.'; + private static final char DYN_ATTRIBUTE_NAME_SEPARATOR = '.'; private static final char DYN_ATTRIBUTE_OPEN_DELIM = '{'; private static final char DYN_ATTRIBUTE_CLOSE_DELIM = '}'; @@ -90,7 +93,7 @@ public class AtlasEntityType extends AtlasStructType { private List<AtlasAttribute> dynEvalTriggerAttributes = Collections.emptyList(); private Map<String,List<TemplateToken>> parsedTemplates = Collections.emptyMap(); private Set<String> tagPropagationEdges = Collections.emptySet(); - private Map<String, List<AtlasNamespaceAttribute>> namespaceAttributes = Collections.emptyMap(); + private Map<String, Map<String, AtlasNamespaceAttribute>> namespaceAttributes = Collections.emptyMap(); public AtlasEntityType(AtlasEntityDef entityDef) { super(entityDef); @@ -237,24 +240,22 @@ public class AtlasEntityType extends AtlasStructType { } } - Map<String, List<AtlasNamespaceAttribute>> superTypeNamespaces = superType.getNamespaceAttributes(); + Map<String, Map<String, AtlasNamespaceAttribute>> superTypeNamespaces = superType.getNamespaceAttributes(); if (MapUtils.isNotEmpty(superTypeNamespaces)) { - for (Map.Entry<String, List<AtlasNamespaceAttribute>> entry : superTypeNamespaces.entrySet()) { - String nsName = entry.getKey(); - List<AtlasNamespaceAttribute> superTypeNsAttrs = entry.getValue(); - List<AtlasNamespaceAttribute> nsAttrs = namespaceAttributes.get(nsName); + for (Map.Entry<String, Map<String, AtlasNamespaceAttribute>> entry : superTypeNamespaces.entrySet()) { + String nsName = entry.getKey(); + Map<String, AtlasNamespaceAttribute> superTypeNsAttrs = entry.getValue(); + Map<String, AtlasNamespaceAttribute> nsAttrs = namespaceAttributes.get(nsName); if (nsAttrs == null) { - nsAttrs = new ArrayList<>(); + nsAttrs = new HashMap<>(); namespaceAttributes.put(nsName, nsAttrs); } - for (AtlasNamespaceAttribute superTypeNsAttr : superTypeNsAttrs) { - if (!nsAttrs.contains(superTypeNsAttr)) { - nsAttrs.add(superTypeNsAttr); - } + for (Map.Entry<String, AtlasNamespaceAttribute> nsAttrEntry : superTypeNsAttrs.entrySet()) { + nsAttrs.put(nsAttrEntry.getKey(), nsAttrEntry.getValue()); } } } @@ -306,12 +307,12 @@ public class AtlasEntityType extends AtlasStructType { Map<String, List<AtlasAttributeDef>> namespaceAttributeDefs = new HashMap<>(); - for (Map.Entry<String, List<AtlasNamespaceAttribute>> entry : namespaceAttributes.entrySet()) { - String nsName = entry.getKey(); - List<AtlasNamespaceAttribute> nsAttrs = entry.getValue(); - List<AtlasAttributeDef> nsAttrDefs = new ArrayList<>(); + for (Map.Entry<String, Map<String, AtlasNamespaceAttribute>> entry : namespaceAttributes.entrySet()) { + String nsName = entry.getKey(); + Map<String, AtlasNamespaceAttribute> nsAttrs = entry.getValue(); + List<AtlasAttributeDef> nsAttrDefs = new ArrayList<>(); - for (AtlasNamespaceAttribute nsAttr : nsAttrs) { + for (AtlasNamespaceAttribute nsAttr : nsAttrs.values()) { nsAttrDefs.add(nsAttr.getAttributeDef()); } @@ -329,6 +330,29 @@ public class AtlasEntityType extends AtlasStructType { } } + @Override + public AtlasAttribute getSystemAttribute(String attributeName) { + return AtlasEntityType.ENTITY_ROOT.allAttributes.get(attributeName); + } + + @Override + public AtlasNamespaceAttribute getNamespaceAttribute(String nsAttrQualifiedName) { + AtlasNamespaceAttribute ret = null; + + if (nsAttrQualifiedName != null) { + int idxSep = nsAttrQualifiedName.indexOf(AtlasEntityType.NS_ATTRIBUTE_NAME_SEPARATOR); + + if (idxSep != -1) { + String nsName = nsAttrQualifiedName.substring(0, idxSep); + String nsAttrName = nsAttrQualifiedName.substring(idxSep + 1); + + ret = getNamespaceAttribute(nsName, nsAttrName); + } + } + + return ret; + } + public Set<String> getSuperTypes() { return entityDef.getSuperTypes(); } @@ -407,14 +431,21 @@ public class AtlasEntityType extends AtlasStructType { return CollectionUtils.isNotEmpty(tagPropagationEdges) ? tagPropagationEdges.toArray(new String[tagPropagationEdges.size()]) : null; } - public Map<String, List<AtlasNamespaceAttribute>> getNamespaceAttributes() { + public Map<String, Map<String, AtlasNamespaceAttribute>> getNamespaceAttributes() { return namespaceAttributes; } - public List<AtlasNamespaceAttribute> getNamespaceAttributes(String nsName) { + public Map<String, AtlasNamespaceAttribute> getNamespaceAttributes(String nsName) { return namespaceAttributes.get(nsName); } + public AtlasNamespaceAttribute getNamespaceAttribute(String nsName, String nsAttrName) { + Map<String, AtlasNamespaceAttribute> nsAttrs = namespaceAttributes.get(nsName); + AtlasNamespaceAttribute ret = nsAttrs != null ? nsAttrs.get(nsAttrName) : null; + + return ret; + } + public Map<String,List<TemplateToken>> getParsedTemplates() { return parsedTemplates; } public AtlasAttribute getRelationshipAttribute(String attributeName, String relationshipType) { @@ -506,16 +537,16 @@ public class AtlasEntityType extends AtlasStructType { } public void addNamespaceAttribute(AtlasNamespaceAttribute attribute) { - String nsName = attribute.getDefinedInType().getTypeName(); - List<AtlasNamespaceAttribute> attributes = namespaceAttributes.get(nsName); + String nsName = attribute.getDefinedInType().getTypeName(); + Map<String, AtlasNamespaceAttribute> nsAttrs = namespaceAttributes.get(nsName); - if (attributes == null) { - attributes = new ArrayList<>(); + if (nsAttrs == null) { + nsAttrs = new HashMap<>(); - namespaceAttributes.put(nsName, attributes); + namespaceAttributes.put(nsName, nsAttrs); } - attributes.add(attribute); + nsAttrs.put(attribute.getName(), attribute); } public String getQualifiedAttributeName(String attrName) throws AtlasBaseException { @@ -739,30 +770,6 @@ public class AtlasEntityType extends AtlasStructType { } } - private static AtlasEntityType initRootEntityType() { - List<AtlasAttributeDef> attributeDefs = new ArrayList<AtlasAttributeDef>() {{ - add(new AtlasAttributeDef(TIMESTAMP_PROPERTY_KEY, ATLAS_TYPE_DATE, false, true)); - add(new AtlasAttributeDef(MODIFICATION_TIMESTAMP_PROPERTY_KEY, ATLAS_TYPE_DATE, false, true)); - add(new AtlasAttributeDef(MODIFIED_BY_KEY, ATLAS_TYPE_STRING, false, true)); - add(new AtlasAttributeDef(CREATED_BY_KEY, ATLAS_TYPE_STRING, false, true)); - add(new AtlasAttributeDef(STATE_PROPERTY_KEY, ATLAS_TYPE_STRING, false, true)); - - add(new AtlasAttributeDef(GUID_PROPERTY_KEY, ATLAS_TYPE_STRING, true, true)); - add(new AtlasAttributeDef(HISTORICAL_GUID_PROPERTY_KEY, ATLAS_TYPE_STRING, true, true)); - add(new AtlasAttributeDef(TYPE_NAME_PROPERTY_KEY, ATLAS_TYPE_STRING, false, true)); - add(new AtlasAttributeDef(CLASSIFICATION_TEXT_KEY, ATLAS_TYPE_STRING, false, true)); - add(new AtlasAttributeDef(CLASSIFICATION_NAMES_KEY, ATLAS_TYPE_STRING, false, true)); - add(new AtlasAttributeDef(PROPAGATED_CLASSIFICATION_NAMES_KEY, ATLAS_TYPE_STRING, false, true)); - add(new AtlasAttributeDef(IS_INCOMPLETE_PROPERTY_KEY, ATLAS_TYPE_INT, false, true)); - add(new AtlasAttributeDef(LABELS_PROPERTY_KEY, ATLAS_TYPE_STRING, false, true)); - add(new AtlasAttributeDef(CUSTOM_ATTRIBUTES_PROPERTY_KEY, ATLAS_TYPE_STRING, false, true)); - }}; - - AtlasEntityDef entityDef = new AtlasEntityDef(ENTITY_ROOT_NAME, "Root entity for system attributes", "1.0", attributeDefs); - - return new AtlasEntityType(entityDef); - } - private void addSubType(AtlasEntityType subType) { subTypes.add(subType.getTypeName()); } @@ -1198,4 +1205,53 @@ public class AtlasEntityType extends AtlasStructType { return adj; } + + /* this class provides abstractions that help basic-search and dsl-search to deal with + * system-attributes and namespace-attributes + */ + private static class AtlasRootEntityType extends AtlasEntityType { + private AtlasTypeRegistry typeRegistry = null; + + public AtlasRootEntityType() { + super(getRootEntityDef()); + } + + @Override + void resolveReferences(AtlasTypeRegistry typeRegistry) throws AtlasBaseException { + super.resolveReferences(typeRegistry); + + // save typeRegistry for use in getNamespaceAttribute() + this.typeRegistry = typeRegistry; + } + + @Override + public AtlasNamespaceAttribute getNamespaceAttribute(String nsName, String nsAttrName) { + AtlasNamespaceType nsType = typeRegistry != null ? typeRegistry.getNamespaceTypeByName(nsName) : null; + AtlasAttribute nsAttr = nsType != null ? nsType.getAttribute(nsAttrName) : null; + + return (nsAttr instanceof AtlasNamespaceAttribute) ? (AtlasNamespaceAttribute) nsAttr : null; + } + + private static AtlasEntityDef getRootEntityDef() { + List<AtlasAttributeDef> attributeDefs = new ArrayList<AtlasAttributeDef>() {{ + add(new AtlasAttributeDef(TIMESTAMP_PROPERTY_KEY, ATLAS_TYPE_DATE, false, true)); + add(new AtlasAttributeDef(MODIFICATION_TIMESTAMP_PROPERTY_KEY, ATLAS_TYPE_DATE, false, true)); + add(new AtlasAttributeDef(MODIFIED_BY_KEY, ATLAS_TYPE_STRING, false, true)); + add(new AtlasAttributeDef(CREATED_BY_KEY, ATLAS_TYPE_STRING, false, true)); + add(new AtlasAttributeDef(STATE_PROPERTY_KEY, ATLAS_TYPE_STRING, false, true)); + + add(new AtlasAttributeDef(GUID_PROPERTY_KEY, ATLAS_TYPE_STRING, true, true)); + add(new AtlasAttributeDef(HISTORICAL_GUID_PROPERTY_KEY, ATLAS_TYPE_STRING, true, true)); + add(new AtlasAttributeDef(TYPE_NAME_PROPERTY_KEY, ATLAS_TYPE_STRING, false, true)); + add(new AtlasAttributeDef(CLASSIFICATION_TEXT_KEY, ATLAS_TYPE_STRING, false, true)); + add(new AtlasAttributeDef(CLASSIFICATION_NAMES_KEY, ATLAS_TYPE_STRING, false, true)); + add(new AtlasAttributeDef(PROPAGATED_CLASSIFICATION_NAMES_KEY, ATLAS_TYPE_STRING, false, true)); + add(new AtlasAttributeDef(IS_INCOMPLETE_PROPERTY_KEY, ATLAS_TYPE_INT, false, true)); + add(new AtlasAttributeDef(LABELS_PROPERTY_KEY, ATLAS_TYPE_STRING, false, true)); + add(new AtlasAttributeDef(CUSTOM_ATTRIBUTES_PROPERTY_KEY, ATLAS_TYPE_STRING, false, true)); + }}; + + return new AtlasEntityDef(ENTITY_ROOT_NAME, "Root entity for system attributes", "1.0", attributeDefs); + } + } } diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java index 6bd3a11..5c62a24 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java @@ -226,17 +226,19 @@ public class AtlasStructType extends AtlasType { ret = getSystemAttribute(attributeName); } + if (ret == null) { + ret = getNamespaceAttribute(attributeName); + } + return ret; } public AtlasAttribute getSystemAttribute(String attributeName) { - AtlasAttribute ret = null; - if (this instanceof AtlasEntityType) { - ret = AtlasEntityType.ENTITY_ROOT.allAttributes.get(attributeName); - } else if (this instanceof AtlasClassificationType) { - ret = AtlasClassificationType.CLASSIFICATION_ROOT.allAttributes.get(attributeName); - } - return ret; + return null; + } + + public AtlasAttribute getNamespaceAttribute(String attributeName) { + return null; } @Override diff --git a/repository/src/main/java/org/apache/atlas/query/GremlinQueryComposer.java b/repository/src/main/java/org/apache/atlas/query/GremlinQueryComposer.java index 8256d25..04b1775 100644 --- a/repository/src/main/java/org/apache/atlas/query/GremlinQueryComposer.java +++ b/repository/src/main/java/org/apache/atlas/query/GremlinQueryComposer.java @@ -119,9 +119,9 @@ public class GremlinQueryComposer { if (ia.isTrait()) { String traitName = ia.get(); - if (traitName.equalsIgnoreCase(ALL_CLASSIFICATIONS)) { + if (traitName.equals(ALL_CLASSIFICATIONS)) { addTrait(GremlinClause.ANY_TRAIT, ia); - } else if (traitName.equalsIgnoreCase(NO_CLASSIFICATIONS)) { + } else if (traitName.equals(NO_CLASSIFICATIONS)) { addTrait(GremlinClause.NO_TRAIT, ia); } else { addTrait(GremlinClause.TRAIT, ia); diff --git a/repository/src/main/java/org/apache/atlas/query/IdentifierHelper.java b/repository/src/main/java/org/apache/atlas/query/IdentifierHelper.java index 801ec02..c443652 100644 --- a/repository/src/main/java/org/apache/atlas/query/IdentifierHelper.java +++ b/repository/src/main/java/org/apache/atlas/query/IdentifierHelper.java @@ -20,6 +20,8 @@ package org.apache.atlas.query; import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.type.AtlasNamespaceType; +import org.apache.atlas.type.AtlasType; import org.apache.commons.lang.StringUtils; import java.util.regex.Matcher; @@ -148,7 +150,6 @@ public class IdentifierHelper { raw = context.getTypeNameFromAlias(this.raw); } - updateParts(); updateTypeInfo(lookup, context); setIsTrait(context, lookup, attributeName); updateEdgeInfo(lookup, context); @@ -183,15 +184,29 @@ public class IdentifierHelper { } private void updateTypeInfo(org.apache.atlas.query.Lookup lookup, GremlinQueryComposer.Context context) { + parts = StringUtils.split(raw, "."); + + // check if this is a namespace attribute + if (parts.length == 2) { + try { + AtlasType type = lookup.getType(parts[0]); + + if (type instanceof AtlasNamespaceType) { + parts = new String[1]; + parts[0] = raw; + } + } catch (AtlasBaseException excp) { + // ignore + } + } + if (parts.length == 1) { typeName = context.hasAlias(parts[0]) ? context.getTypeNameFromAlias(parts[0]) : context.getActiveTypeName(); qualifiedName = getDefaultQualifiedNameForSinglePartName(context, parts[0]); attributeName = parts[0]; - } - - if (parts.length == 2) { + } else if (parts.length == 2) { boolean isAttrOfActiveType = lookup.hasAttribute(context, parts[0]); if (isAttrOfActiveType) { attributeName = parts[0]; @@ -242,10 +257,6 @@ public class IdentifierHelper { } } - private void updateParts() { - parts = StringUtils.split(raw, "."); - } - public String getQualifiedName() { return qualifiedName; } diff --git a/repository/src/main/java/org/apache/atlas/query/RegistryBasedLookup.java b/repository/src/main/java/org/apache/atlas/query/RegistryBasedLookup.java index 2878a74..2b34abf 100644 --- a/repository/src/main/java/org/apache/atlas/query/RegistryBasedLookup.java +++ b/repository/src/main/java/org/apache/atlas/query/RegistryBasedLookup.java @@ -21,7 +21,6 @@ package org.apache.atlas.query; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.TypeCategory; import org.apache.atlas.model.typedef.AtlasBaseTypeDef; -import org.apache.atlas.repository.Constants; import org.apache.atlas.type.*; import org.apache.commons.lang.StringUtils; @@ -29,20 +28,12 @@ import java.util.*; import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFIED; import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_NOT_CLASSIFIED; +import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_ENTITY_TYPES; import static org.apache.atlas.model.discovery.SearchParameters.ALL_CLASSIFICATIONS; import static org.apache.atlas.model.discovery.SearchParameters.NO_CLASSIFICATIONS; +import static org.apache.atlas.model.discovery.SearchParameters.ALL_ENTITY_TYPES; class RegistryBasedLookup implements Lookup { - private static final Set<String> SYSTEM_ATTRIBUTES = new HashSet<>( - Arrays.asList(Constants.GUID_PROPERTY_KEY, - Constants.MODIFIED_BY_KEY, - Constants.CREATED_BY_KEY, - Constants.STATE_PROPERTY_KEY, - Constants.TIMESTAMP_PROPERTY_KEY, - Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, - Constants.HOME_ID_KEY - )); - private static final Map<String, String> NUMERIC_ATTRIBUTES = new HashMap<String, String>() {{ put(AtlasBaseTypeDef.ATLAS_TYPE_SHORT, ""); put(AtlasBaseTypeDef.ATLAS_TYPE_INT, ""); @@ -63,10 +54,12 @@ class RegistryBasedLookup implements Lookup { public AtlasType getType(String typeName) throws AtlasBaseException { AtlasType ret; - if (typeName.equalsIgnoreCase(ALL_CLASSIFICATIONS)) { + if (typeName.equals(ALL_CLASSIFICATIONS)) { ret = MATCH_ALL_CLASSIFIED; - } else if (typeName.equalsIgnoreCase(NO_CLASSIFICATIONS)) { + } else if (typeName.equals(NO_CLASSIFICATIONS)) { ret = MATCH_ALL_NOT_CLASSIFIED; + } else if (typeName.equals(ALL_ENTITY_TYPES)) { + ret = MATCH_ALL_ENTITY_TYPES; } else { ret = typeRegistry.getType(typeName); } @@ -81,15 +74,7 @@ class RegistryBasedLookup implements Lookup { return ""; } - if(isSystemAttribute(name)) { - return name; - } else { - return et.getQualifiedAttributeName(name); - } - } - - private boolean isSystemAttribute(String s) { - return SYSTEM_ATTRIBUTES.contains(s); + return et.getQualifiedAttributeName(name); } @Override @@ -99,10 +84,6 @@ class RegistryBasedLookup implements Lookup { return false; } - if(isSystemAttribute(attributeName)) { - return true; - } - AtlasType at = getAttributeType(et, attributeName); if(at == null) { return false; @@ -144,8 +125,7 @@ class RegistryBasedLookup implements Lookup { public boolean hasAttribute(GremlinQueryComposer.Context context, String typeName) { AtlasEntityType entityType = context.getActiveEntityType(); - return (entityType != null) && - (isSystemAttribute(typeName) || entityType.hasAttribute(typeName) || entityType.hasRelationshipAttribute(typeName)); + return getAttribute(entityType, typeName) != null; } @Override @@ -178,6 +158,8 @@ class RegistryBasedLookup implements Lookup { t = MATCH_ALL_CLASSIFIED; } else if (typeName.equalsIgnoreCase(NO_CLASSIFICATIONS)) { t = MATCH_ALL_NOT_CLASSIFIED; + } else if (typeName.equalsIgnoreCase(ALL_ENTITY_TYPES)) { + t = MATCH_ALL_ENTITY_TYPES; } else { t = typeRegistry.getType(typeName); } @@ -257,6 +239,11 @@ class RegistryBasedLookup implements Lookup { @Override public String getVertexPropertyName(String typeName, String attrName) { AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName); + + if (entityType == null && StringUtils.equals(typeName, ALL_ENTITY_TYPES)) { + entityType = MATCH_ALL_ENTITY_TYPES; + } + AtlasStructType.AtlasAttribute attribute = getAttribute(entityType, attrName); if (attribute == null) { return null; diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index f511e2f..c536f3b 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -1377,19 +1377,19 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore { private void validateNamespaceAttributes(AtlasVertex entityVertex, AtlasEntityType entityType, Map<String, Map<String, Object>> entityNamespaces, boolean isOverwrite) throws AtlasBaseException { List<String> messages = new ArrayList<>(); - Map<String, List<AtlasNamespaceAttribute>> entityTypeNamespaces = entityType.getNamespaceAttributes(); + Map<String, Map<String, AtlasNamespaceAttribute>> entityTypeNamespaces = entityType.getNamespaceAttributes(); for (String nsName : entityNamespaces.keySet()) { - if (!entityNamespaces.containsKey(nsName)) { + if (!entityTypeNamespaces.containsKey(nsName)) { messages.add(nsName + ": invalid namespace for entity type " + entityType.getTypeName()); continue; } - List<AtlasNamespaceAttribute> entityTypeNsAttributes = entityTypeNamespaces.get(nsName); - Map<String, Object> entityNsAttributes = entityNamespaces.get(nsName); + Map<String, AtlasNamespaceAttribute> entityTypeNsAttributes = entityTypeNamespaces.get(nsName); + Map<String, Object> entityNsAttributes = entityNamespaces.get(nsName); - for (AtlasNamespaceAttribute nsAttribute : entityTypeNsAttributes) { + for (AtlasNamespaceAttribute nsAttribute : entityTypeNsAttributes.values()) { AtlasType attrType = nsAttribute.getAttributeType(); String attrName = nsAttribute.getName(); Object attrValue = entityNsAttributes.get(attrName); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index 113325d..7461931 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -427,14 +427,14 @@ public class EntityGraphMapper { LOG.debug("==> setNamespaceAttributes(entityVertex={}, entityType={}, entityNamespaces={}", entityVertex, entityType.getTypeName(), entityNamespaces); } - Map<String, List<AtlasNamespaceAttribute>> entityTypeNamespaces = entityType.getNamespaceAttributes(); + Map<String, Map<String, AtlasNamespaceAttribute>> entityTypeNamespaces = entityType.getNamespaceAttributes(); - for (Map.Entry<String, List<AtlasNamespaceAttribute>> entry : entityTypeNamespaces.entrySet()) { - String nsName = entry.getKey(); - List<AtlasNamespaceAttribute> entityTypeNsAttributes = entry.getValue(); - Map<String, Object> entityNsAttributes = MapUtils.isEmpty(entityNamespaces) ? null : entityNamespaces.get(nsName); + for (Map.Entry<String, Map<String, AtlasNamespaceAttribute>> entry : entityTypeNamespaces.entrySet()) { + String nsName = entry.getKey(); + Map<String, AtlasNamespaceAttribute> entityTypeNsAttributes = entry.getValue(); + Map<String, Object> entityNsAttributes = MapUtils.isEmpty(entityNamespaces) ? null : entityNamespaces.get(nsName); - for (AtlasNamespaceAttribute nsAttribute : entityTypeNsAttributes) { + for (AtlasNamespaceAttribute nsAttribute : entityTypeNsAttributes.values()) { String nsAttrName = nsAttribute.getName(); Object nsAttrExistingValue = entityVertex.getProperty(nsAttribute.getVertexPropertyName(), Object.class); Object nsAttrNewValue = MapUtils.isEmpty(entityNsAttributes) ? null : entityNsAttributes.get(nsAttrName); @@ -480,19 +480,19 @@ public class EntityGraphMapper { LOG.debug("==> addOrUpdateNamespaceAttributes(entityVertex={}, entityType={}, entityNamespaces={}", entityVertex, entityType.getTypeName(), entityNamespaces); } - Map<String, List<AtlasNamespaceAttribute>> entityTypeNamespaces = entityType.getNamespaceAttributes(); + Map<String, Map<String, AtlasNamespaceAttribute>> entityTypeNamespaces = entityType.getNamespaceAttributes(); if (MapUtils.isNotEmpty(entityTypeNamespaces) && MapUtils.isNotEmpty(entityNamespaces)) { - for (Map.Entry<String, List<AtlasNamespaceAttribute>> entry : entityTypeNamespaces.entrySet()) { - String nsName = entry.getKey(); - List<AtlasNamespaceAttribute> entityTypeNsAttributes = entry.getValue(); - Map<String, Object> entityNsAttributes = entityNamespaces.get(nsName); + for (Map.Entry<String, Map<String, AtlasNamespaceAttribute>> entry : entityTypeNamespaces.entrySet()) { + String nsName = entry.getKey(); + Map<String, AtlasNamespaceAttribute> entityTypeNsAttributes = entry.getValue(); + Map<String, Object> entityNsAttributes = entityNamespaces.get(nsName); if (MapUtils.isEmpty(entityNsAttributes)) { continue; } - for (AtlasNamespaceAttribute nsAttribute : entityTypeNsAttributes) { + for (AtlasNamespaceAttribute nsAttribute : entityTypeNsAttributes.values()) { String nsAttrName = nsAttribute.getName(); if (!entityNsAttributes.containsKey(nsAttrName)) { @@ -528,12 +528,12 @@ public class EntityGraphMapper { LOG.debug("==> removeNamespaceAttributes(entityVertex={}, entityType={}, entityNamespaces={}", entityVertex, entityType.getTypeName(), entityNamespaces); } - Map<String, List<AtlasNamespaceAttribute>> entityTypeNamespaces = entityType.getNamespaceAttributes(); + Map<String, Map<String, AtlasNamespaceAttribute>> entityTypeNamespaces = entityType.getNamespaceAttributes(); if (MapUtils.isNotEmpty(entityTypeNamespaces) && MapUtils.isNotEmpty(entityNamespaces)) { - for (Map.Entry<String, List<AtlasNamespaceAttribute>> entry : entityTypeNamespaces.entrySet()) { - String nsName = entry.getKey(); - List<AtlasNamespaceAttribute> entityTypeNsAttributes = entry.getValue(); + for (Map.Entry<String, Map<String, AtlasNamespaceAttribute>> entry : entityTypeNamespaces.entrySet()) { + String nsName = entry.getKey(); + Map<String, AtlasNamespaceAttribute> entityTypeNsAttributes = entry.getValue(); if (!entityNamespaces.containsKey(nsName)) { // nothing to remove for this namespace continue; @@ -541,7 +541,7 @@ public class EntityGraphMapper { Map<String, Object> entityNsAttributes = entityNamespaces.get(nsName); - for (AtlasNamespaceAttribute nsAttribute : entityTypeNsAttributes) { + for (AtlasNamespaceAttribute nsAttribute : entityTypeNsAttributes.values()) { // if (entityNsAttributes is empty) remove all attributes in this namespace // else remove the attribute only if its given in entityNsAttributes if (MapUtils.isEmpty(entityNsAttributes) || entityNsAttributes.containsKey(nsAttribute.getName())) { diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 55c1cac..3c22d9a 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -678,21 +678,26 @@ public class EntityGraphRetriever { if (CollectionUtils.isNotEmpty(attributes)) { for (String attrName : attributes) { - String nonQualifiedAttrName = toNonQualifiedName(attrName); - if (ret.hasAttribute(attrName)) { - continue; - } - - AtlasAttribute attribute = entityType.getAttribute(nonQualifiedAttrName); + AtlasAttribute attribute = entityType.getAttribute(attrName); if (attribute == null) { - attribute = entityType.getRelationshipAttribute(nonQualifiedAttrName, null); + attrName = toNonQualifiedName(attrName); + + if (ret.hasAttribute(attrName)) { + continue; + } + + attribute = entityType.getAttribute(attrName); + + if (attribute == null) { + attribute = entityType.getRelationshipAttribute(attrName, null); + } } Object attrValue = getVertexAttribute(entityVertex, attribute); if (attrValue != null) { - ret.setAttribute(nonQualifiedAttrName, attrValue); + ret.setAttribute(attrName, attrValue); } } } @@ -765,15 +770,15 @@ public class EntityGraphRetriever { } private void mapNamespaceAttributes(AtlasVertex entityVertex, AtlasEntity entity) throws AtlasBaseException { - AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName()); - Map<String, List<AtlasNamespaceAttribute>> entityTypeNamespaces = entityType != null ? entityType.getNamespaceAttributes() : null; + AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName()); + Map<String, Map<String, AtlasNamespaceAttribute>> entityTypeNamespaces = entityType != null ? entityType.getNamespaceAttributes() : null; if (MapUtils.isNotEmpty(entityTypeNamespaces)) { - for (Map.Entry<String, List<AtlasNamespaceAttribute>> entry : entityTypeNamespaces.entrySet()) { - String nsName = entry.getKey(); - List<AtlasNamespaceAttribute> nsAttributes = entry.getValue(); + for (Map.Entry<String, Map<String, AtlasNamespaceAttribute>> entry : entityTypeNamespaces.entrySet()) { + String nsName = entry.getKey(); + Map<String, AtlasNamespaceAttribute> nsAttributes = entry.getValue(); - for (AtlasNamespaceAttribute nsAttribute : nsAttributes) { + for (AtlasNamespaceAttribute nsAttribute : nsAttributes.values()) { Object nsAttrValue = mapVertexToAttribute(entityVertex, nsAttribute, null, false, false); if (nsAttrValue != null) { diff --git a/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasNamespaceDefStoreV2Test.java b/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasNamespaceDefStoreV2Test.java index 8f6afc7..cbac18c 100644 --- a/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasNamespaceDefStoreV2Test.java +++ b/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasNamespaceDefStoreV2Test.java @@ -25,7 +25,7 @@ import org.apache.atlas.model.typedef.AtlasNamespaceDef; import org.apache.atlas.model.typedef.AtlasStructDef; import org.apache.atlas.model.typedef.AtlasTypesDef; import org.apache.atlas.type.AtlasEntityType; -import org.apache.atlas.type.AtlasNamespaceType; +import org.apache.atlas.type.AtlasNamespaceType.AtlasNamespaceAttribute; import org.apache.atlas.type.AtlasType; import org.apache.atlas.type.AtlasTypeRegistry; import org.testng.Assert; @@ -84,7 +84,7 @@ public class AtlasNamespaceDefStoreV2Test { createNamespaceTypes(namespaceName); Assert.assertEquals(typeRegistry.getAllNamespaceDefs().size(), 1); AtlasEntityType entityType = typeRegistry.getEntityTypeByName("hive_table"); - Map<String, List<AtlasNamespaceType.AtlasNamespaceAttribute>> m1 = entityType.getNamespaceAttributes(); + Map<String, Map<String, AtlasNamespaceAttribute>> m1 = entityType.getNamespaceAttributes(); Assert.assertEquals(m1.get(namespaceName).size(), 2); } @@ -117,7 +117,7 @@ public class AtlasNamespaceDefStoreV2Test { updateNamespaceTypeDefs(namespaceDef); typeDefStore.updateTypesDef(typesDefs); AtlasEntityType entityType = typeRegistry.getEntityTypeByName("hive_table"); - Map<String, List<AtlasNamespaceType.AtlasNamespaceAttribute>> m1 = entityType.getNamespaceAttributes(); + Map<String, Map<String, AtlasNamespaceAttribute>> m1 = entityType.getNamespaceAttributes(); Assert.assertEquals(m1.get(namespaceName).size(), 3); }